diff options
Diffstat (limited to '')
217 files changed, 63204 insertions, 0 deletions
diff --git a/services/sync/SyncComponents.manifest b/services/sync/SyncComponents.manifest new file mode 100644 index 0000000000..68c36cd80f --- /dev/null +++ b/services/sync/SyncComponents.manifest @@ -0,0 +1,3 @@ +# Register resource aliases +# (Note, for tests these are also set up in addResourceAlias) +resource services-sync resource://gre/modules/services-sync/ diff --git a/services/sync/Weave.sys.mjs b/services/sync/Weave.sys.mjs new file mode 100644 index 0000000000..05a7031a73 --- /dev/null +++ b/services/sync/Weave.sys.mjs @@ -0,0 +1,190 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + CLIENT_NOT_CONFIGURED: "resource://services-sync/constants.sys.mjs", + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "syncUsername", + "services.sync.username" +); + +/** + * Sync's XPCOM service. + * + * It is named "Weave" for historical reasons. + * + * It's worth noting how Sync is lazily loaded. We register a timer that + * loads Sync a few seconds after app startup. This is so Sync does not + * adversely affect application start time. + * + * If Sync is not configured, no extra Sync code is loaded. If an + * external component (say the UI) needs to interact with Sync, it + * should use the promise-base function whenLoaded() - something like the + * following: + * + * // 1. Grab a handle to the Sync XPCOM service. + * let service = Cc["@mozilla.org/weave/service;1"] + * .getService(Components.interfaces.nsISupports) + * .wrappedJSObject; + * + * // 2. Use the .then method of the promise. + * service.whenLoaded().then(() => { + * // You are free to interact with "Weave." objects. + * return; + * }); + * + * And that's it! However, if you really want to avoid promises and do it + * old-school, then + * + * // 1. Get a reference to the service as done in (1) above. + * + * // 2. Check if the service has been initialized. + * if (service.ready) { + * // You are free to interact with "Weave." objects. + * return; + * } + * + * // 3. Install "ready" listener. + * Services.obs.addObserver(function onReady() { + * Services.obs.removeObserver(onReady, "weave:service:ready"); + * + * // You are free to interact with "Weave." objects. + * }, "weave:service:ready", false); + * + * // 4. Trigger loading of Sync. + * service.ensureLoaded(); + */ +export function WeaveService() { + this.wrappedJSObject = this; + this.ready = false; +} + +WeaveService.prototype = { + classID: Components.ID("{74b89fb0-f200-4ae8-a3ec-dd164117f6de}"), + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + get Weave() { + const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" + ); + return Weave; + }, + + ensureLoaded() { + // Side-effect of accessing the service is that it is instantiated. + this.Weave.Service; + }, + + whenLoaded() { + if (this.ready) { + return Promise.resolve(); + } + let onReadyPromise = new Promise(resolve => { + Services.obs.addObserver(function onReady() { + Services.obs.removeObserver(onReady, "weave:service:ready"); + resolve(); + }, "weave:service:ready"); + }); + this.ensureLoaded(); + return onReadyPromise; + }, + + init() { + // Force Weave service to load if it hasn't triggered from overlays + this.timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + this.timer.initWithCallback( + { + notify: () => { + let isConfigured = false; + // We only load more if it looks like Sync is configured. + if (this.enabled) { + // We have an associated FxAccount. So, do a more thorough check. + // This will import a number of modules and thus increase memory + // accordingly. We could potentially copy code performed by + // this check into this file if our above code is yielding too + // many false positives. + var { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" + ); + isConfigured = + Weave.Status.checkSetup() != lazy.CLIENT_NOT_CONFIGURED; + } + if (isConfigured) { + this.ensureLoaded(); + } + }, + }, + 10000, + Ci.nsITimer.TYPE_ONE_SHOT + ); + }, + + /** + * Whether Sync appears to be enabled. + * + * This returns true if we have an associated FxA account and Sync is enabled. + * + * It does *not* perform a robust check to see if the client is working. + * For that, you'll want to check Weave.Status.checkSetup(). + */ + get enabled() { + return ( + !!lazy.syncUsername && + Services.prefs.getBoolPref("identity.fxaccounts.enabled") + ); + }, +}; + +export function AboutWeaveLog() {} +AboutWeaveLog.prototype = { + classID: Components.ID("{d28f8a0b-95da-48f4-b712-caf37097be41}"), + + QueryInterface: ChromeUtils.generateQI([ + "nsIAboutModule", + "nsISupportsWeakReference", + ]), + + getURIFlags(aURI) { + return 0; + }, + + newChannel(aURI, aLoadInfo) { + let dir = lazy.FileUtils.getDir("ProfD", ["weave", "logs"]); + try { + dir.create(Ci.nsIFile.DIRECTORY_TYPE, lazy.FileUtils.PERMS_DIRECTORY); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_FILE_ALREADY_EXISTS) { + throw ex; + } + // Ignore the exception due to a directory that already exists. + } + let uri = Services.io.newFileURI(dir); + let channel = Services.io.newChannelFromURIWithLoadInfo(uri, aLoadInfo); + + channel.originalURI = aURI; + + // Ensure that the about page has the same privileges as a regular directory + // view. That way links to files can be opened. make sure we use the correct + // origin attributes when creating the principal for accessing the + // about:sync-log data. + let principal = Services.scriptSecurityManager.createContentPrincipal( + uri, + aLoadInfo.originAttributes + ); + + channel.owner = principal; + return channel; + }, +}; diff --git a/services/sync/components.conf b/services/sync/components.conf new file mode 100644 index 0000000000..e4f82b35b1 --- /dev/null +++ b/services/sync/components.conf @@ -0,0 +1,20 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'cid': '{74b89fb0-f200-4ae8-a3ec-dd164117f6de}', + 'contract_ids': ['@mozilla.org/weave/service;1'], + 'esModule': 'resource://services-sync/Weave.sys.mjs', + 'constructor': 'WeaveService', + }, + { + 'cid': '{d28f8a0b-95da-48f4-b712-caf37097be41}', + 'contract_ids': ['@mozilla.org/network/protocol/about;1?what=sync-log'], + 'esModule': 'resource://services-sync/Weave.sys.mjs', + 'constructor': 'AboutWeaveLog', + }, +] diff --git a/services/sync/docs/engines.rst b/services/sync/docs/engines.rst new file mode 100644 index 0000000000..7a4fa721af --- /dev/null +++ b/services/sync/docs/engines.rst @@ -0,0 +1,133 @@ +============================ +The Sync engines in the tree +============================ + +Unless otherwise specified, the engine implementations can be found +`here <https://searchfox.org/mozilla-central/source/services/sync/modules/engines>`_ + +Please read the :doc:`overview`. + +Clients +======= + +The ``clients`` engine is a special engine in that it's invisible to the +user and can not be disabled - think of it as a "meta" engine. As such, it +doesn't really have a sensible concept of ``store`` or ``tracker``. + +The engine is mainly responsible for keeping its own record current in the +``clients`` collection. Some parts of Sync use this collection to know what +other clients exist and when they last synced (although alot of this is moving +to using the Firefox Accounts devices). + +Clients also has the ability to handle ``commands`` - in short, some other +client can write to this client's ``commands``, and when this client notices, +it will execute the command. Commands aren't arbitrary, so commands must be +understood by both sides for them to work. There are commands to "wipe" +collections etc. In practice, this is used only by ``bookmarks`` when a device +restores bookmarks - in that case, the restoring device will send a ``wipe`` +command to all other clients so that they take the new bookmarks instead of +merging them. + +If not for this somewhat limited ``commands`` functionality, this engine could +be considered deprecated and subsumed by FxA devices - but because we +can't just remove support for commands and also do not have a plan for +replacing them, the clients engine remains important. + +Bookmarks +========= + +The ``bookmarks`` engine has changed so that it's tightly integrated with the +``places`` database. Instead of an external ``tracker``, the tracking is +integrated into Places. Each bookmark has a `syncStatus` and a +`syncChangeCounter` and these are managed internally by places. Sync then just +queries for changed bookmarks by looking for these fields. + +Bookmarks is somewhat unique in that it needs to maintain a tree structure, +which makes merging a challenge. The `dogear <https://github.com/mozilla/dogear>`_ +component (written in Rust and also used by the +`application-services bookmarks component <https://github.com/mozilla/application-services/tree/main/components/places>`_) +performs this merging. + +Bookmarks also pioneered the concept of a "mirror" - this is a database table +which tracks exactly what is on the server. Because each sync only fetches +changes from the server since the last sync, each sync does not supply every +record on the server. However, the merging code does need to know what's on +the server - so the mirror tracks this. + +History +======= + +History is similar to bookmarks described above - it's closely integrated with +places - but is less complex because there's no tree structure involved. + +One unique characteristic of history is that the engine takes steps to *not* +upload everything - old profiles tend to have too much history to reasonably +store and upload, so typically uploads are limited to the last 5000 visits. + +Logins +====== + +Logins has also been upgraded to be closely integrated with `Services.logins` - +the logins component itself manages the metadata. + +Tabs +==== + +Tabs is a special engine in that there's no underlying storage at all - it +both saves the currently open tabs from this device (which are enumerated +every time it's updated) and also lets other parts of Firefox know which tabs +are open on other devices. There's no database - if we haven't synced yet we +don't know what other tabs are open, and when we do know, the list is just +stored in memory. + +The `SyncedTabs module <https://searchfox.org/mozilla-central/source/services/sync/modules/SyncedTabs.jsm>`_ +is the main interface the browser uses to get the list of tabs from other +devices. + +Add-ons +======= + +Addons is still an "old school" engine, with a tracker and store which aren't +closely integrated with the addon manager. As a result it's fairly complex and +error prone - eg, it persists the "last known" state so it can know what to +sync, where a better model would be for the addon manager to track the changes +on Sync's behalf. + +It also attempts to sync themes etc. The future of this engine isn't clear given +it doesn't work on mobile platforms. + +Addresses / Credit-Cards +======================== + +Addresses and Credit-cards have Sync functionality tightly bound with the +store. Unlike other engines above, this engine has always been tightly bound, +because it was written after we realized this tight-binding was a feature and +not a bug. + +Technically these are 2 separate engines and collections. However, because the +underlying storage uses a shared implementation, the syncing also uses a +shared implementation - ie, the same logic is used for both - so we tend to +treat them as a single engine in practice. + +As a result, only a shim is in the `services/sync/modules/engines/` directory, +while the actual logic is +`next to the storage implementation <https://searchfox.org/mozilla-central/source/toolkit/components/formautofill/FormAutofillSync.jsm>`_. + +This engine has a unique twist on the "mirror" concept described above - +whenever a change is made to a fields, the original value of the field is +stored directly in the storage. This means that on the next sync, the value +of the record on the server can be deduced, meaning a "3-way" merge can be +done, so it can better tell the difference between local only, remote only, or +conflicting changes. + +WebExt-Storage +============== + +webext-storage is implemented in Rust and lives in +`application services <https://github.com/mozilla/application-services/tree/main/components/webext-storage>`_ +and is vendored into the `addons code <https://searchfox.org/mozilla-central/source/toolkit/components/extensions/storage/webext_storage_bridge>`_ - +note that this includes the storage *and* Sync code. The Sync engine itself +is a shim in the sync directory. + +See the :doc:`rust-engines` document for more about how rust engines are +integrated. diff --git a/services/sync/docs/external.rst b/services/sync/docs/external.rst new file mode 100644 index 0000000000..f7cebde32d --- /dev/null +++ b/services/sync/docs/external.rst @@ -0,0 +1,8 @@ +============== +External Links +============== + +Some external links that might be of interest: + +* `Information about the server APIs <https://mozilla-services.readthedocs.io/en/latest/index.html>`_ +* `Some external Sync Client docs <https://mozilla-services.readthedocs.io/en/latest/sync/index.html>`_ diff --git a/services/sync/docs/index.rst b/services/sync/docs/index.rst new file mode 100644 index 0000000000..37ce3c19a0 --- /dev/null +++ b/services/sync/docs/index.rst @@ -0,0 +1,17 @@ +==== +Sync +==== + +This documents the sync implementation inside mozilla-central. It assumes +a general understanding of what Sync is and how it works at a high level - you +can find `some external docs <https://mozilla-services.readthedocs.io/en/latest/sync/>`_ +which can help with this. + +.. toctree:: + :maxdepth: 1 + + overview + engines + rust-engines + payload-evolution + external diff --git a/services/sync/docs/overview.rst b/services/sync/docs/overview.rst new file mode 100644 index 0000000000..e956090d70 --- /dev/null +++ b/services/sync/docs/overview.rst @@ -0,0 +1,81 @@ +==================== +Introduction to Sync +==================== + +This document is a brief introduction to how Sync is implemented in desktop Firefox. + +General, Historical, Anatomy of a Sync Engine +============================================= + +This section describes how Sync used to work - and indeed, how much of it still +does. While we discuss how this is slowly changing, this context is valuable. + +For any datatype which syncs, there tends to be 3 parts: + +Store +----- + +The sync ``store`` interfaces with the actual Firefox desktop store. For example, +in the ``passwords`` engine, the "store" is that layer that talks to +``Services.logins`` + +Tracker +------- + +The ``tracker`` is what knows that something should be synced. For example, +when the user creates or updates a password, it is the tracker that knows +we should sync now, and what particular password(s) should be updated. + +This is typically done via "observer" notifications - ``Services.logins``, +``places`` etc all send specific notifications when certain events happen +(and indeed, some of these were added for Sync's benefit) + +Engine +------ + +The ``engine`` ties it all together. It works with the ``store`` and +``tracker`` and tracks its own metadata (eg, the timestamp of the passwords on +the server, so it knows how to grab just changed records and how to pass them +off to the ``store`` so the actual underlying storage can be updated. + +All of the above parts were typically in the +`services/sync/modules/engines directory <https://searchfox.org/mozilla-central/source/services/sync/modules/engines>`_ +directory and decoupled from the data they were syncing. + + +The Future of Desktop-Specific Sync Engines +=========================================== + +The system described above reflects the fact that Sync was "bolted on" to +Desktop Firefox relatively late - eg, the Sync ``store`` is decoupled from the +actual ``store``. This has causes a number of problems - particularly around +the ``tracker`` and the metadata used by the engine, and the fact that changes +to the backing store would often forget that Sync existed. + +Over the last few years, the Sync team has come to the conclusion that Sync +support must be integrated much closer to the store itself. For example, +``Services.logins`` should track when something has changed that would cause +an item to be synced. It should also track the metadata for the store so that +if (say) a corrupt database is recovered by creating a new, empty one, the +metadata should also vanish so Sync knows something bad has happened and can +recover. + +However, this is a slow process - currently the ``bookmarks``, ``history`` and +``passwords`` legacy engines have been improved so more responsibility is taken +by the stores. In all cases, for implementation reasons, the Sync +implementation still has a ``store``, but it tends to be a thin wrapper around +the actual underlying store. + +The Future of Cross-Platform Sync Engines +========================================= + +There are a number of Sync engines implemented in Rust and which live in the +application-services repository. While these were often done for mobile +platforms, the longer term hope is that they can be reused on Desktop. +:doc:`engines` has more details on these. + +While no existing engines have been replaced with Rust implemented engines, +the webext-storage engine is implemented in Rust via application-services, so +doesn't tend to use any of the infrastructure described above. + +Hopefully over time we will find more Rust-implemented engines in Desktop. diff --git a/services/sync/docs/payload-evolution.md b/services/sync/docs/payload-evolution.md new file mode 100644 index 0000000000..e195ee545d --- /dev/null +++ b/services/sync/docs/payload-evolution.md @@ -0,0 +1,168 @@ +# Handling the evolution of Sync payloads + +(Note that this document has been written in the format of an [application-services ADR](https://github.com/mozilla/application-services/blob/main/docs/adr/0000-use-markdown-architectural-decision-records.md) +but the relelvant teams decided that ultimately the best home for this doc is in mozilla-central) + +* Status: Accepted +* Deciders: sync team, credentials management team +* Date: 2023-03-15 + +Technical Story: +* https://github.com/mozilla/application-services/pull/5434 +* https://docs.google.com/document/d/1ToLOERA5HKzEzRVZNv6Ohv_2wZaujW69pVb1Kef2jNY + +## Context and Problem Statement + +Sync exists on all platforms (Desktop, Android, iOS), all channels (Nightly, Beta, Release, ESR) and is heavily used across all Firefox features. +Whenever there are feature changes or requests that potentially involve schema changes, there are not a lot of good options to ensure sync doesn’t break for any specific client. +Since sync data is synced from all channels, we need to make sure each client can handle the new data and that all channels can support the new schema. +Issues like [credit card failing on android and desktop release channels due to schema change on desktop Nightly](https://bugzilla.mozilla.org/show_bug.cgi?id=1812235) +are examples of such cases we can run into. +This document describes our decision on how we will support payload evolution over time. + +Note that even though this document exists in the application-services repository, it should +be considered to apply to all sync implementations, whether in this repository, in mozilla-central, +or anywhere else. + +## Definitions + +* A "new" Firefox installation is a version of Firefox which has a change to a Sync payload which + is not yet understood by "recent" versions. The most common example would be a Nightly version + of Firefox with a new feature not yet on the release channel. + +* A "recent" Firefox installation is a version older than a "new" version, which does not understand + or have support for new features in "new" versions, but which we still want to support without + breakage and without the user perceiving data-loss. This is typically accepted to mean the + current ESR version or later, but taking into account the slow update when new ESRs are released. + +* An "old" version is any version before what we consider "recent". + + +## Decision Drivers + +* It must be possible to change what data is carried by Sync to meet future product requirements. +* Both desktop and mobile platforms must be considered. +* We must not break "recent" Firefox installations when a "new" Firefox installation syncs, and vice-versa. +* Round-tripping data from a "new" Firefox installation through a "recent" Firefox installation must not discard any of the new data, and vice-versa. +* Some degree of breakage for "old" Firefox installations when "new" or "recent" firefoxes sync + might be considered acceptable if absolutely necessary. +* However, breakage of "new" or "recent" Firefoxes when an "old" version syncs is *not* acceptable. +* Because such evolution should be rare, we do not want to set an up-front policy about locking out + "old" versions just because they might have a problem in the future. That is, we want to avoid + a policy that dictates versions more than (say) 2 years old will break when syncing "just in case" +* Any solution to this must be achievable in a relatively short timeframe as we know of product + asks coming down the line which require this capability. + +## Considered Options + +* A backwards compatible schema policy, consisting of (a) having engines "round trip" data they + do not know about and (b) never changing the semantics of existing data. +* A policy which prevents "recent" clients from syncing, or editing data, or other restrictions. +* A formal schema-driven process. +* Consider the sync payloads frozen and never change them. +* Use separate collections for new data + +## Decision Outcome + +Chosen option: A backwards compatible schema policy because it is very flexible and the only option +meeting the decision drivers. + +## Pros and Cons of the Options + +### A backwards compatible schema policy + +A summary of this option is a policy by which: + +* Every sync engine must arrange to persist any fields from the payload which it + does not understand. The next time that engine needs to upload that record to the storage server, + it must arrange to add all such "unknown" fields back into the payload. + +* Different engines must identify different locations where this might happen. For example, the + `passwords` engine would identify the "root" of the payload, `addresses` and `creditcards` would + identify the `entry` sub-object in the payload, while the history engine would probably identify + *both* the root of the payload and the `visits` array. + +* Fields can not change type, nor be removed for a significant amount of time. This might mean + that "new" clients must support both new fields *and* fields which are considered deprecated + by these "new" clients because they are still used by "recent" versions. + +The pros and cons: + +* Good, because it meets the requirements. + +* Good, because the initial set of work identified is relatively simple to implement (that work + specifically is to support the round-tripping of "unknown" fields, in the hope that by the + time actual schema changes are proposed, this round-trip capability will then be on all "recent" + versions) + +* Bad, because the inability to deprecate or change existing fields means that + some evolution tasks become complicated. For example, consider a hypothetical change where + we wanted to change from "street/city/state" fields into a free-form "address" field. New + Firefox versions would need to populate *both* new and old fields when writing to the server, + and handle the fact that only the old versions might be updated when it sees an incoming + record written by a "recent" or "old" versions of Firefox. However, this should be rare. + +* Bad, because it's not possible to prove a proposed change meets the requirements - the policy + is informal and requires good judgement as changes are proposed. + +### A policy which prevents "recent" clients from syncing, or editing data + +Proposals which fit into this category might have been implemented by (say) adding +a version number to the schema, and if clients did not fully understand the schema it would +either prevent syncing the record, or sync it but not allow editing it, or similar. + +This was rejected because: + +* The user would certainly perceive data-loss if we ignored the incoming data entirely. +* If we still wanted older versions to "partially" see the record (eg, but disallow editing) we'd + still need most of the chosen option anyway - specifically, we could still never + deprecate fields etc. +* The UI/UX of trying to explain to the user why they can't edit a record was deemed impossible + to do in a satisfactory way. +* This would effectively penalize users who chose to use Nightly Firefoxes in any way. Simply + allowing a Nightly to sync would effectively break Release/Mobile Firefox versions. + +### A formal schema-driven process. + +Ideally we could formally describe schemas, but we can't come up with anything here which +works with the constraints of supporting older clients - we simply can't update older released +Firefoxes so they know how to work with the new schemas. We also couldn't come up with a solution +where a schema is downloaded dynamically which also allowed the *semantics* (as opposed to simply +validity) of new fields to be described. + +### Consider the sync payloads frozen and never change them. + +A process where payloads are frozen was rejected because: + +* The most naive approach here would not meet the needs of Firefox in the future. + +* A complicated system where we started creating new payload and new collections + (ie, freezing "old" schemas but then creating "new" schemas only understood by + newer clients) could not be conceived in a way that still met the requirements, + particularly around data-loss for older clients. For example, adding a credit-card + on a Nightly version but having it be completely unavailable on a release firefox + isn't acceptable. + +### Use separate collections for new data + +We could store the new data in a separate collection. For example define a +bookmarks2 collection where each record has the same guid as one in bookmarks alongside any new fields. +Newer clients use both collections to sync. + +The pros and cons: + +* Good, because it allows newer clients to sync new data without affecting recent or older clients +* Bad, because sync writes would lose atomicity without server changes. + We can currently write to a single collection in an atomic way, but don't have a way to write to multiple collections. +* Bad because this number of collections grows each time we want to add fields. +* Bad because it potentially leaks extra information to an attacker that gets access to the encrypted server records. + For example if we added a new collection for a single field, then the attacker could guess if that + field was set or not based on the size of the encrypted record. +* Bad because it's difficult to handle nested data with this approach, + for example adding a field to a history record visit. +* Bad because it has the same issue with dependent data as the chosen solution. + +## Links <!-- optional --> + +* This document was originally [brain-stormed in this google docs document](https://docs.google.com/document/d/1ToLOERA5HKzEzRVZNv6Ohv_2wZaujW69pVb1Kef2jNY), + which may be of interest for historical context, but should not be considered part of this ADR. diff --git a/services/sync/docs/rust-engines.rst b/services/sync/docs/rust-engines.rst new file mode 100644 index 0000000000..af00fd6619 --- /dev/null +++ b/services/sync/docs/rust-engines.rst @@ -0,0 +1,37 @@ +================================ +How Rust Engines are implemented +================================ + +There are 2 main components to engines implemented in Rust + +The bridged-engine +================== + +Because Rust engines still need to work with the existing Sync infrastructure, +there's the concept of a `bridged-engine <https://searchfox.org/mozilla-central/source/services/sync/modules/bridged_engine.js>`_. +In short, this is just a shim between the existing +`Sync Service <https://searchfox.org/mozilla-central/source/services/sync/modules/service.js>`_ +and the Rust code. + +The bridge +========== + +`"Golden Gate" <https://searchfox.org/mozilla-central/source/services/sync/golden_gate>`_ +is a utility to help bridge any Rust implemented Sync engines with desktop. In +other words, it's a "rusty bridge" - get it? Get it? Yet another of Lina's puns +that live on! + +One of the key challenges with integrating a Rust Sync component with desktop +is the different threading models. The Rust code tends to be synchronous - +most functions block the calling thread to do the disk or network IO necessary +to work - it assumes that the consumer will delegate this to some other thread. + +So golden_gate is this background thread delegation for a Rust Sync engine - +gecko calls golden-gate on the main thread, it marshalls the call to a worker +thread, and the result is marshalled back to the main thread. + +It's worth noting that golden_gate is just for the Sync engine part - other +parts of the component (ie, the part that provides the functionality that's not +sync related) will have its own mechanism for this. For example, the +`webext-storage bridge <https://searchfox.org/mozilla-central/source/toolkit/components/extensions/storage/webext_storage_bridge/src>`_ +uses a similar technique `which has some in-depth documentation <../../toolkit/components/extensions/webextensions/webext-storage.html>`_. diff --git a/services/sync/golden_gate/Cargo.toml b/services/sync/golden_gate/Cargo.toml new file mode 100644 index 0000000000..3f94e1a1e9 --- /dev/null +++ b/services/sync/golden_gate/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "golden_gate" +description = "A bridge for wiring up Sync engines implemented in Rust" +version = "0.1.0" +authors = ["The Firefox Sync Developers <sync-team@mozilla.com>"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +anyhow = "1" +atomic_refcell = "0.1" +cstr = "0.2" +interrupt-support = "0.1" +log = "0.4" +moz_task = { path = "../../../xpcom/rust/moz_task" } +nserror = { path = "../../../xpcom/rust/nserror" } +nsstring = { path = "../../../xpcom/rust/nsstring" } +serde_json = "1" +storage_variant = { path = "../../../storage/variant" } +sync15 = "0.1" +xpcom = { path = "../../../xpcom/rust/xpcom" } + +[dependencies.thin-vec] +version = "0.2.1" +features = ["gecko-ffi"] diff --git a/services/sync/golden_gate/src/error.rs b/services/sync/golden_gate/src/error.rs new file mode 100644 index 0000000000..373d20756e --- /dev/null +++ b/services/sync/golden_gate/src/error.rs @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{error, fmt, result, str::Utf8Error}; + +use nserror::{nsresult, NS_ERROR_INVALID_ARG, NS_ERROR_UNEXPECTED}; +use serde_json::Error as JsonError; + +/// A specialized `Result` type for Golden Gate. +pub type Result<T> = result::Result<T, Error>; + +/// The error type for Golden Gate errors. +#[derive(Debug)] +pub enum Error { + /// A wrapped XPCOM error. + Nsresult(nsresult), + + /// A ferry didn't run on the background task queue. + DidNotRun(&'static str), + + /// A string contains invalid UTF-8 or JSON. + MalformedString(Box<dyn error::Error + Send + Sync + 'static>), +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + Error::MalformedString(error) => Some(error.as_ref()), + _ => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::Nsresult(result) => write!(f, "Operation failed with {}", result.error_name()), + Error::DidNotRun(what) => write!(f, "Failed to run `{what}` on background thread"), + Error::MalformedString(error) => error.fmt(f), + } + } +} + +impl From<nsresult> for Error { + fn from(result: nsresult) -> Error { + Error::Nsresult(result) + } +} + +impl From<Utf8Error> for Error { + fn from(error: Utf8Error) -> Error { + Error::MalformedString(error.into()) + } +} + +impl From<JsonError> for Error { + fn from(error: JsonError) -> Error { + Error::MalformedString(error.into()) + } +} + +impl From<Error> for nsresult { + fn from(error: Error) -> nsresult { + match error { + Error::DidNotRun(_) => NS_ERROR_UNEXPECTED, + Error::Nsresult(result) => result, + Error::MalformedString(_) => NS_ERROR_INVALID_ARG, + } + } +} diff --git a/services/sync/golden_gate/src/ferry.rs b/services/sync/golden_gate/src/ferry.rs new file mode 100644 index 0000000000..99994811ab --- /dev/null +++ b/services/sync/golden_gate/src/ferry.rs @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use nsstring::nsCString; +use storage_variant::VariantType; +use sync15::Guid; +use xpcom::{interfaces::nsIVariant, RefPtr}; + +/// An operation that runs on the background thread, and optionally passes a +/// result to its callback. +pub enum Ferry { + LastSync, + SetLastSync(i64), + SyncId, + ResetSyncId, + EnsureCurrentSyncId(String), + SyncStarted, + StoreIncoming(Vec<nsCString>), + SetUploaded(i64, Vec<Guid>), + SyncFinished, + Reset, + Wipe, +} + +impl Ferry { + /// Returns the operation name for debugging and labeling the task + /// runnable. + pub fn name(&self) -> &'static str { + match self { + Ferry::LastSync => concat!(module_path!(), "getLastSync"), + Ferry::SetLastSync(_) => concat!(module_path!(), "setLastSync"), + Ferry::SyncId => concat!(module_path!(), "getSyncId"), + Ferry::ResetSyncId => concat!(module_path!(), "resetSyncId"), + Ferry::EnsureCurrentSyncId(_) => concat!(module_path!(), "ensureCurrentSyncId"), + Ferry::SyncStarted => concat!(module_path!(), "syncStarted"), + Ferry::StoreIncoming { .. } => concat!(module_path!(), "storeIncoming"), + Ferry::SetUploaded { .. } => concat!(module_path!(), "setUploaded"), + Ferry::SyncFinished => concat!(module_path!(), "syncFinished"), + Ferry::Reset => concat!(module_path!(), "reset"), + Ferry::Wipe => concat!(module_path!(), "wipe"), + } + } +} + +/// The result of a ferry task, sent from the background thread back to the +/// main thread. Results are converted to variants, and passed as arguments to +/// `mozIBridgedSyncEngineCallback`s. +pub enum FerryResult { + LastSync(i64), + SyncId(Option<String>), + AssignedSyncId(String), + Null, +} + +impl Default for FerryResult { + fn default() -> Self { + FerryResult::Null + } +} + +impl FerryResult { + /// Converts the result to an `nsIVariant` that can be passed as an + /// argument to `callback.handleResult()`. + pub fn into_variant(self) -> RefPtr<nsIVariant> { + match self { + FerryResult::LastSync(v) => v.into_variant(), + FerryResult::SyncId(Some(v)) => nsCString::from(v).into_variant(), + FerryResult::SyncId(None) => ().into_variant(), + FerryResult::AssignedSyncId(v) => nsCString::from(v).into_variant(), + FerryResult::Null => ().into_variant(), + } + } +} diff --git a/services/sync/golden_gate/src/lib.rs b/services/sync/golden_gate/src/lib.rs new file mode 100644 index 0000000000..8da6524bd7 --- /dev/null +++ b/services/sync/golden_gate/src/lib.rs @@ -0,0 +1,119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! **Golden Gate** 🌉 is a crate for bridging Desktop Sync to our suite of +//! Rust sync and storage components. It connects Sync's `BridgedEngine` class +//! to the Rust `BridgedEngine` trait via the `mozIBridgedSyncEngine` XPCOM +//! interface. +//! +//! Due to limitations in implementing XPCOM interfaces for generic types, +//! Golden Gate doesn't implement `mozIBridgedSyncEngine` directly. Instead, +//! it provides helpers, called "ferries", for passing Sync records between +//! JavaScript and Rust. The ferries also handle threading and type +//! conversions. +//! +//! Here's a step-by-step guide for adding a new Rust Sync engine to Firefox. +//! +//! ## Step 1: Create your (XPCOM) bridge +//! +//! In your consuming crate, define a type for your `mozIBridgedSyncEngine` +//! implementation. We'll call this type the **brige**. The bridge is +//! responsible for exposing your Sync engine to XPIDL [^1], in a way that lets +//! JavaScript call it. +//! +//! For your bridge type, you'll need to implement an xpcom interface with the +//! `#[xpcom(implement(mozIBridgedSyncEngine), nonatomic)]` attribute then +//! define `xpcom_method!()` stubs for the `mozIBridgedSyncEngine` methods. For +//! more details about implementing XPCOM methods in Rust, check out the docs in +//! `xpcom/rust/xpcom/src/method.rs`. +//! +//! You'll also need to add an entry for your bridge type to `components.conf`, +//! and define C++ and Rust constructors for it, so that JavaScript code can +//! create instances of it. Check out `NS_NewWebExtStorage` (and, in C++, +//! `mozilla::extensions::storageapi::NewWebExtStorage`) and +//! `NS_NewSyncedBookmarksMerger` (`mozilla::places::NewSyncedBookmarksMerger` +//! in C++) for how to do this. +//! +//! [^1]: You can think of XPIDL as a souped-up C FFI, with richer types and a +//! degree of type safety. +//! +//! ## Step 2: Add a background task queue to your bridge +//! +//! A task queue lets your engine do I/O, merging, and other syncing tasks on a +//! background thread pool. This is important because database reads and writes +//! can take an unpredictable amount of time. Doing these on the main thread can +//! cause jank, and, in the worst case, lock up the browser UI for seconds at a +//! time. +//! +//! The `moz_task` crate provides a `create_background_task_queue` function to +//! do this. Once you have a queue, you can use it to call into your Rust +//! engine. Golden Gate takes care of ferrying arguments back and forth across +//! the thread boundary. +//! +//! Since it's a queue, ferries arrive in the order they're scheduled, so +//! your engine's `store_incoming` method will always be called before `apply`, +//! which is likewise called before `set_uploaded`. The thread manager scales +//! the pool for you; you don't need to create or manage your own threads. +//! +//! ## Step 3: Create your Rust engine +//! +//! Next, you'll need to implement the Rust side of the bridge. This is a type +//! that implements the `BridgedEngine` trait. +//! +//! Bridged engines handle storing incoming Sync records, merging changes, +//! resolving conflicts, and fetching outgoing records for upload. Under the +//! hood, your engine will hold either a database connection directly, or +//! another object that does. +//! +//! Although outside the scope of Golden Gate, your engine will also likely +//! expose a data storage API, for fetching, updating, and deleting items +//! locally. Golden Gate provides the syncing layer on top of this local store. +//! +//! A `BridgedEngine` itself doesn't need to be `Send` or `Sync`, but the +//! ferries require both, since they're calling into your bridge on the +//! background task queue. +//! +//! In practice, this means your bridge will need to hold a thread-safe owned +//! reference to the engine, via `Arc<Mutex<BridgedEngine>>`. In fact, this +//! pattern is so common that Golden Gate implements `BridgedEngine` for any +//! `Mutex<BridgedEngine>`, which automatically locks the mutex before calling +//! into the engine. +//! +//! ## Step 4: Connect the bridge to the JavaScript and Rust sides +//! +//! On the JavaScript side, you'll need to subclass Sync's `BridgedEngine` +//! class, and give it a handle to your XPCOM bridge. The base class has all the +//! machinery for hooking up any `mozIBridgedSyncEngine` implementation so that +//! Sync can drive it. +//! +//! On the Rust side, each `mozIBridgedSyncEngine` method should create a +//! Golden Gate ferry, and dispatch it to the background task queue. The +//! ferries correspond to the method names. For example, `ensureCurrentSyncId` +//! should create a `Ferry::ensure_current_sync_id(...)`; `storeIncoming`, a +//! `Ferry::store_incoming(...)`; and so on. This is mostly boilerplate. +//! +//! And that's it! Each ferry will, in turn, call into your Rust +//! `BridgedEngine`, and send the results back to JavaScript. +//! +//! For an example of how all this works, including exposing a storage (not +//! just syncing!) API to JS via XPIDL, check out `webext_storage::Bridge` for +//! the `storage.sync` API! + +#[macro_use] +extern crate cstr; + +pub mod error; +mod ferry; +pub mod log; +pub mod task; + +pub use crate::log::LogSink; +pub use error::{Error, Result}; +// Re-export items from `interrupt-support` and `sync15`, so that +// consumers of `golden_gate` don't have to depend on them. +pub use interrupt_support::{Interrupted, Interruptee}; +pub use sync15::bso::{IncomingBso, OutgoingBso}; +pub use sync15::engine::{ApplyResults, BridgedEngine}; +pub use sync15::Guid; +pub use task::{ApplyTask, FerryTask}; diff --git a/services/sync/golden_gate/src/log.rs b/services/sync/golden_gate/src/log.rs new file mode 100644 index 0000000000..de7fd0dfc3 --- /dev/null +++ b/services/sync/golden_gate/src/log.rs @@ -0,0 +1,161 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::fmt::{self, Write}; + +use log::{Level, LevelFilter, Log, Metadata, Record}; +use moz_task::{Task, TaskRunnable, ThreadPtrHandle, ThreadPtrHolder}; +use nserror::nsresult; +use nsstring::nsString; +use xpcom::{interfaces::mozIServicesLogSink, RefPtr}; + +pub struct LogSink { + pub max_level: LevelFilter, + logger: Option<ThreadPtrHandle<mozIServicesLogSink>>, +} + +impl Default for LogSink { + fn default() -> Self { + LogSink { + max_level: LevelFilter::Off, + logger: None, + } + } +} + +impl LogSink { + /// Creates a log sink that adapts the Rust `log` crate to the Sync + /// `Log.sys.mjs` logger. + /// + /// This is copied from `bookmark_sync::Logger`. It would be nice to share + /// these, but, for now, we've just duplicated it to make prototyping + /// easier. + #[inline] + pub fn new(max_level: LevelFilter, logger: ThreadPtrHandle<mozIServicesLogSink>) -> LogSink { + LogSink { + max_level, + logger: Some(logger), + } + } + + /// Creates a log sink using the given Services `logger` as the + /// underlying implementation. The `logger` will always be called + /// asynchronously on its owning thread; it doesn't need to be + /// thread-safe. + pub fn with_logger(logger: Option<&mozIServicesLogSink>) -> Result<LogSink, nsresult> { + Ok(if let Some(logger) = logger { + // Fetch the maximum log level while we're on the main thread, so + // that `LogSink::enabled()` can check it while on the background + // thread. Otherwise, we'd need to dispatch a `LogTask` for every + // log message, only to discard most of them when the task calls + // into the logger on the main thread. + let mut raw_max_level = 0i16; + let rv = unsafe { logger.GetMaxLevel(&mut raw_max_level) }; + let max_level = if rv.succeeded() { + match raw_max_level { + mozIServicesLogSink::LEVEL_ERROR => LevelFilter::Error, + mozIServicesLogSink::LEVEL_WARN => LevelFilter::Warn, + mozIServicesLogSink::LEVEL_DEBUG => LevelFilter::Debug, + mozIServicesLogSink::LEVEL_TRACE => LevelFilter::Trace, + mozIServicesLogSink::LEVEL_INFO => LevelFilter::Info, + _ => LevelFilter::Off, + } + } else { + LevelFilter::Off + }; + LogSink::new( + max_level, + ThreadPtrHolder::new(cstr!("mozIServicesLogSink"), RefPtr::new(logger))?, + ) + } else { + LogSink::default() + }) + } + + /// Returns a reference to the underlying `mozIServicesLogSink`. + pub fn logger(&self) -> Option<&mozIServicesLogSink> { + self.logger.as_ref().and_then(|l| l.get()) + } + + /// Logs a message to the Sync logger, if one is set. This would be better + /// implemented as a macro, as Dogear does, so that we can pass variadic + /// arguments without manually invoking `fmt_args!()` every time we want + /// to log a message. + /// + /// The `log` crate's macros aren't suitable here, because those log to the + /// global logger. However, we don't want to set the global logger in our + /// crate, because that will log _everything_ that uses the Rust `log` crate + /// to the Sync logs, including WebRender and audio logging. + pub fn debug(&self, args: fmt::Arguments) { + let meta = Metadata::builder() + .level(Level::Debug) + .target(module_path!()) + .build(); + if self.enabled(&meta) { + self.log(&Record::builder().args(args).metadata(meta).build()); + } + } +} + +impl Log for LogSink { + #[inline] + fn enabled(&self, meta: &Metadata) -> bool { + self.logger.is_some() && meta.level() <= self.max_level + } + + fn log(&self, record: &Record) { + if !self.enabled(record.metadata()) { + return; + } + if let Some(logger) = &self.logger { + let mut message = nsString::new(); + if write!(message, "{}", record.args()).is_ok() { + let task = LogTask { + logger: logger.clone(), + level: record.metadata().level(), + message, + }; + let _ = TaskRunnable::new("extension_storage_sync::Logger::log", Box::new(task)) + .and_then(|r| TaskRunnable::dispatch(r, logger.owning_thread())); + } + } + } + + fn flush(&self) {} +} + +/// Logs a message to the mirror logger. This task is created on the background +/// thread queue, and dispatched to the main thread. +struct LogTask { + logger: ThreadPtrHandle<mozIServicesLogSink>, + level: Level, + message: nsString, +} + +impl Task for LogTask { + fn run(&self) { + let logger = self.logger.get().unwrap(); + match self.level { + Level::Error => unsafe { + logger.Error(&*self.message); + }, + Level::Warn => unsafe { + logger.Warn(&*self.message); + }, + Level::Debug => unsafe { + logger.Debug(&*self.message); + }, + Level::Trace => unsafe { + logger.Trace(&*self.message); + }, + Level::Info => unsafe { + logger.Info(&*self.message); + }, + } + } + + fn done(&self) -> Result<(), nsresult> { + Ok(()) + } +} diff --git a/services/sync/golden_gate/src/task.rs b/services/sync/golden_gate/src/task.rs new file mode 100644 index 0000000000..8cab21830b --- /dev/null +++ b/services/sync/golden_gate/src/task.rs @@ -0,0 +1,355 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{fmt::Write, mem, result}; + +use atomic_refcell::AtomicRefCell; +use moz_task::{DispatchOptions, Task, TaskRunnable, ThreadPtrHandle, ThreadPtrHolder}; +use nserror::{nsresult, NS_ERROR_FAILURE}; +use nsstring::{nsACString, nsCString}; +use sync15::engine::{ApplyResults, BridgedEngine}; +use sync15::Guid; +use thin_vec::ThinVec; +use xpcom::{ + interfaces::{ + mozIBridgedSyncEngineApplyCallback, mozIBridgedSyncEngineCallback, nsIEventTarget, + }, + RefPtr, +}; + +use crate::error::{Error, Result}; +use crate::ferry::{Ferry, FerryResult}; + +/// A ferry task sends (or ferries) an operation to a bridged engine on a +/// background thread or task queue, and ferries back an optional result to +/// a callback. +pub struct FerryTask { + /// We want to ensure scheduled ferries can't block finalization of the underlying + /// store - we want a degree of confidence that closing the database will happen when + /// we want even if tasks are queued up to run on another thread. + /// We rely on the semantics of our BridgedEngines to help here: + /// * A bridged engine is expected to hold a weak reference to its store. + /// * Our LazyStore is the only thing holding a reference to the "real" store. + /// Thus, when our LazyStore asks our "real" store to close, we can be confident + /// a close will happen (ie, we assume that the real store will be able to unwrapp + /// the underlying sqlite `Connection` (using `Arc::try_unwrap`) and close it. + /// However, note that if an operation on the bridged engine is currently running, + /// we will block waiting for that operation to complete, so while this isn't + /// guaranteed to happen immediately, it should happen "soon enough". + engine: Box<dyn BridgedEngine>, + ferry: Ferry, + callback: ThreadPtrHandle<mozIBridgedSyncEngineCallback>, + result: AtomicRefCell<anyhow::Result<FerryResult>>, +} + +impl FerryTask { + /// Creates a task to fetch the engine's last sync time, in milliseconds. + #[inline] + pub fn for_last_sync( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::LastSync, callback) + } + + /// Creates a task to set the engine's last sync time, in milliseconds. + #[inline] + pub fn for_set_last_sync( + engine: Box<dyn BridgedEngine>, + last_sync_millis: i64, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::SetLastSync(last_sync_millis), callback) + } + + /// Creates a task to fetch the engine's sync ID. + #[inline] + pub fn for_sync_id( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::SyncId, callback) + } + + /// Creates a task to reset the engine's sync ID and all its local Sync + /// metadata. + #[inline] + pub fn for_reset_sync_id( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::ResetSyncId, callback) + } + + /// Creates a task to compare the bridged engine's local sync ID with + /// the `new_sync_id` from `meta/global`, and ferry back the final sync ID + /// to use. + #[inline] + pub fn for_ensure_current_sync_id( + engine: Box<dyn BridgedEngine>, + new_sync_id: &nsACString, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry( + engine, + Ferry::EnsureCurrentSyncId(std::str::from_utf8(new_sync_id)?.into()), + callback, + ) + } + + /// Creates a task to signal that the engine is about to sync. + #[inline] + pub fn for_sync_started( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::SyncStarted, callback) + } + + /// Creates a task to store incoming records. + pub fn for_store_incoming( + engine: Box<dyn BridgedEngine>, + incoming_envelopes_json: &[nsCString], + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry( + engine, + Ferry::StoreIncoming(incoming_envelopes_json.to_vec()), + callback, + ) + } + + /// Creates a task to mark a subset of outgoing records as uploaded. This + /// may be called multiple times per sync, or not at all if there are no + /// records to upload. + pub fn for_set_uploaded( + engine: Box<dyn BridgedEngine>, + server_modified_millis: i64, + uploaded_ids: &[nsCString], + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + let uploaded_ids = uploaded_ids.iter().map(|id| Guid::from_slice(id)).collect(); + Self::with_ferry( + engine, + Ferry::SetUploaded(server_modified_millis, uploaded_ids), + callback, + ) + } + + /// Creates a task to signal that all records have been uploaded, and + /// the engine has been synced. This is called even if there were no + /// records uploaded. + #[inline] + pub fn for_sync_finished( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::SyncFinished, callback) + } + + /// Creates a task to reset all local Sync state for the engine, without + /// erasing user data. + #[inline] + pub fn for_reset( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::Reset, callback) + } + + /// Creates a task to erase all local user data for the engine. + #[inline] + pub fn for_wipe( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + Self::with_ferry(engine, Ferry::Wipe, callback) + } + + /// Creates a task for a ferry. The `callback` is bound to the current + /// thread, and will be called once, after the ferry returns from the + /// background thread. + fn with_ferry( + engine: Box<dyn BridgedEngine>, + ferry: Ferry, + callback: &mozIBridgedSyncEngineCallback, + ) -> Result<FerryTask> { + let name = ferry.name(); + Ok(FerryTask { + engine, + ferry, + callback: ThreadPtrHolder::new( + cstr!("mozIBridgedSyncEngineCallback"), + RefPtr::new(callback), + )?, + result: AtomicRefCell::new(Err(Error::DidNotRun(name).into())), + }) + } + + /// Dispatches the task to the given thread `target`. + pub fn dispatch(self, target: &nsIEventTarget) -> Result<()> { + let runnable = TaskRunnable::new(self.ferry.name(), Box::new(self))?; + // `may_block` schedules the task on the I/O thread pool, since we + // expect most operations to wait on I/O. + TaskRunnable::dispatch_with_options( + runnable, + target, + DispatchOptions::default().may_block(true), + )?; + Ok(()) + } + + /// Runs the task on the background thread. This is split out into its own + /// method to make error handling easier. + fn inner_run(&self) -> anyhow::Result<FerryResult> { + let engine = &self.engine; + Ok(match &self.ferry { + Ferry::LastSync => FerryResult::LastSync(engine.last_sync()?), + Ferry::SetLastSync(last_sync_millis) => { + engine.set_last_sync(*last_sync_millis)?; + FerryResult::default() + } + Ferry::SyncId => FerryResult::SyncId(engine.sync_id()?), + Ferry::ResetSyncId => FerryResult::AssignedSyncId(engine.reset_sync_id()?), + Ferry::EnsureCurrentSyncId(new_sync_id) => { + FerryResult::AssignedSyncId(engine.ensure_current_sync_id(new_sync_id)?) + } + Ferry::SyncStarted => { + engine.sync_started()?; + FerryResult::default() + } + Ferry::StoreIncoming(incoming_envelopes_json) => { + let incoming_envelopes = incoming_envelopes_json + .iter() + .map(|envelope| Ok(serde_json::from_slice(envelope)?)) + .collect::<Result<_>>()?; + + engine.store_incoming(incoming_envelopes)?; + FerryResult::default() + } + Ferry::SetUploaded(server_modified_millis, uploaded_ids) => { + engine.set_uploaded(*server_modified_millis, uploaded_ids.as_slice())?; + FerryResult::default() + } + Ferry::SyncFinished => { + engine.sync_finished()?; + FerryResult::default() + } + Ferry::Reset => { + engine.reset()?; + FerryResult::default() + } + Ferry::Wipe => { + engine.wipe()?; + FerryResult::default() + } + }) + } +} + +impl Task for FerryTask { + fn run(&self) { + *self.result.borrow_mut() = self.inner_run(); + } + + fn done(&self) -> result::Result<(), nsresult> { + let callback = self.callback.get().unwrap(); + match mem::replace( + &mut *self.result.borrow_mut(), + Err(Error::DidNotRun(self.ferry.name()).into()), + ) { + Ok(result) => unsafe { callback.HandleSuccess(result.into_variant().coerce()) }, + Err(err) => { + let mut message = nsCString::new(); + write!(message, "{err}").unwrap(); + unsafe { callback.HandleError(NS_ERROR_FAILURE, &*message) } + } + } + .to_result() + } +} + +/// An apply task ferries incoming records to an engine on a background +/// thread, and ferries back records to upload. It's separate from +/// `FerryTask` because its callback type is different. +pub struct ApplyTask { + engine: Box<dyn BridgedEngine>, + callback: ThreadPtrHandle<mozIBridgedSyncEngineApplyCallback>, + result: AtomicRefCell<anyhow::Result<Vec<String>>>, +} + +impl ApplyTask { + /// Returns the task name for debugging. + pub fn name() -> &'static str { + concat!(module_path!(), "apply") + } + + /// Runs the task on the background thread. + fn inner_run(&self) -> anyhow::Result<Vec<String>> { + let ApplyResults { + records: outgoing_records, + .. + } = self.engine.apply()?; + let outgoing_records_json = outgoing_records + .iter() + .map(|record| Ok(serde_json::to_string(record)?)) + .collect::<Result<_>>()?; + Ok(outgoing_records_json) + } + + /// Creates a task. The `callback` is bound to the current thread, and will + /// be called once, after the records are applied on the background thread. + pub fn new( + engine: Box<dyn BridgedEngine>, + callback: &mozIBridgedSyncEngineApplyCallback, + ) -> Result<ApplyTask> { + Ok(ApplyTask { + engine, + callback: ThreadPtrHolder::new( + cstr!("mozIBridgedSyncEngineApplyCallback"), + RefPtr::new(callback), + )?, + result: AtomicRefCell::new(Err(Error::DidNotRun(Self::name()).into())), + }) + } + + /// Dispatches the task to the given thread `target`. + pub fn dispatch(self, target: &nsIEventTarget) -> Result<()> { + let runnable = TaskRunnable::new(Self::name(), Box::new(self))?; + TaskRunnable::dispatch_with_options( + runnable, + target, + DispatchOptions::default().may_block(true), + )?; + Ok(()) + } +} + +impl Task for ApplyTask { + fn run(&self) { + *self.result.borrow_mut() = self.inner_run(); + } + + fn done(&self) -> result::Result<(), nsresult> { + let callback = self.callback.get().unwrap(); + match mem::replace( + &mut *self.result.borrow_mut(), + Err(Error::DidNotRun(Self::name()).into()), + ) { + Ok(envelopes) => { + let result = envelopes + .into_iter() + .map(nsCString::from) + .collect::<ThinVec<_>>(); + unsafe { callback.HandleSuccess(&result) } + } + Err(err) => { + let mut message = nsCString::new(); + write!(message, "{err}").unwrap(); + unsafe { callback.HandleError(NS_ERROR_FAILURE, &*message) } + } + } + .to_result() + } +} diff --git a/services/sync/modules-testing/fakeservices.sys.mjs b/services/sync/modules-testing/fakeservices.sys.mjs new file mode 100644 index 0000000000..4fd7534bf1 --- /dev/null +++ b/services/sync/modules-testing/fakeservices.sys.mjs @@ -0,0 +1,114 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Weave } from "resource://services-sync/main.sys.mjs"; +import { RawCryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +export function FakeFilesystemService(contents) { + this.fakeContents = contents; + let self = this; + + // Save away the unmocked versions of the functions we replace here for tests + // that really want the originals. As this may be called many times per test, + // we must be careful to not replace them with ones we previously replaced. + // (And why are we bothering with these mocks in the first place? Is the + // performance of the filesystem *really* such that it outweighs the downside + // of not running our real JSON functions in the tests? Eg, these mocks don't + // always throw exceptions when the real ones do. Anyway...) + for (let name of ["jsonSave", "jsonLoad", "jsonMove", "jsonRemove"]) { + let origName = "_real_" + name; + if (!Utils[origName]) { + Utils[origName] = Utils[name]; + } + } + + Utils.jsonSave = async function jsonSave(filePath, that, obj) { + let json = typeof obj == "function" ? obj.call(that) : obj; + self.fakeContents["weave/" + filePath + ".json"] = JSON.stringify(json); + }; + + Utils.jsonLoad = async function jsonLoad(filePath, that) { + let obj; + let json = self.fakeContents["weave/" + filePath + ".json"]; + if (json) { + obj = JSON.parse(json); + } + return obj; + }; + + Utils.jsonMove = function jsonMove(aFrom, aTo, that) { + const fromPath = "weave/" + aFrom + ".json"; + self.fakeContents["weave/" + aTo + ".json"] = self.fakeContents[fromPath]; + delete self.fakeContents[fromPath]; + return Promise.resolve(); + }; + + Utils.jsonRemove = function jsonRemove(filePath, that) { + delete self.fakeContents["weave/" + filePath + ".json"]; + return Promise.resolve(); + }; +} + +export function fakeSHA256HMAC(message) { + message = message.substr(0, 64); + while (message.length < 64) { + message += " "; + } + return message; +} + +export function FakeGUIDService() { + let latestGUID = 0; + + Utils.makeGUID = function makeGUID() { + // ensure that this always returns a unique 12 character string + let nextGUID = "fake-guid-" + String(latestGUID++).padStart(2, "0"); + return nextGUID.slice(nextGUID.length - 12, nextGUID.length); + }; +} + +/* + * Mock implementation of WeaveCrypto. It does not encrypt or + * decrypt, merely returning the input verbatim. + */ +export function FakeCryptoService() { + this.counter = 0; + + delete Weave.Crypto; // get rid of the getter first + Weave.Crypto = this; + + RawCryptoWrapper.prototype.ciphertextHMAC = function ciphertextHMAC( + keyBundle + ) { + return fakeSHA256HMAC(this.ciphertext); + }; +} + +FakeCryptoService.prototype = { + async encrypt(clearText, symmetricKey, iv) { + return clearText; + }, + + async decrypt(cipherText, symmetricKey, iv) { + return cipherText; + }, + + async generateRandomKey() { + return btoa("fake-symmetric-key-" + this.counter++); + }, + + generateRandomIV: function generateRandomIV() { + // A base64-encoded IV is 24 characters long + return btoa("fake-fake-fake-random-iv"); + }, + + expandData: function expandData(data, len) { + return data; + }, + + generateRandomBytes: function generateRandomBytes(byteCount) { + return "not-so-random-now-are-we-HA-HA-HA! >:)".slice(byteCount); + }, +}; diff --git a/services/sync/modules-testing/fxa_utils.sys.mjs b/services/sync/modules-testing/fxa_utils.sys.mjs new file mode 100644 index 0000000000..c953f0eaa3 --- /dev/null +++ b/services/sync/modules-testing/fxa_utils.sys.mjs @@ -0,0 +1,55 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Weave } from "resource://services-sync/main.sys.mjs"; +import { SyncAuthManager } from "resource://services-sync/sync_auth.sys.mjs"; + +import { TokenServerClient } from "resource://services-common/tokenserverclient.sys.mjs"; +import { configureFxAccountIdentity } from "resource://testing-common/services/sync/utils.sys.mjs"; + +// Create a new sync_auth object and initialize it with a +// mocked TokenServerClient which always receives the specified response. +export var initializeIdentityWithTokenServerResponse = function (response) { + // First create a mock "request" object that well' hack into the token server. + // A log for it + let requestLog = Log.repository.getLogger("testing.mock-rest"); + if (!requestLog.appenders.length) { + // might as well see what it says :) + requestLog.addAppender(new Log.DumpAppender()); + requestLog.level = Log.Level.Trace; + } + + // A mock request object. + function MockRESTRequest(url) {} + MockRESTRequest.prototype = { + _log: requestLog, + setHeader() {}, + async get() { + this.response = response; + return response; + }, + }; + // The mocked TokenServer client which will get the response. + function MockTSC() {} + MockTSC.prototype = new TokenServerClient(); + MockTSC.prototype.constructor = MockTSC; + MockTSC.prototype.newRESTRequest = function (url) { + return new MockRESTRequest(url); + }; + // Arrange for the same observerPrefix as sync_auth uses. + MockTSC.prototype.observerPrefix = "weave:service"; + + // tie it all together. + Weave.Status.__authManager = Weave.Service.identity = new SyncAuthManager(); + let syncAuthManager = Weave.Service.identity; + // a sanity check + if (!(syncAuthManager instanceof SyncAuthManager)) { + throw new Error("sync isn't configured to use sync_auth"); + } + let mockTSC = new MockTSC(); + configureFxAccountIdentity(syncAuthManager); + syncAuthManager._tokenServerClient = mockTSC; +}; diff --git a/services/sync/modules-testing/rotaryengine.sys.mjs b/services/sync/modules-testing/rotaryengine.sys.mjs new file mode 100644 index 0000000000..d7f2165e4d --- /dev/null +++ b/services/sync/modules-testing/rotaryengine.sys.mjs @@ -0,0 +1,120 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; + +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { SerializableSet, Utils } from "resource://services-sync/util.sys.mjs"; + +/* + * A fake engine implementation. + * This is used all over the place. + * + * Complete with record, store, and tracker implementations. + */ + +export function RotaryRecord(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +RotaryRecord.prototype = {}; +Object.setPrototypeOf(RotaryRecord.prototype, CryptoWrapper.prototype); +Utils.deferGetSet(RotaryRecord, "cleartext", ["denomination"]); + +export function RotaryStore(name, engine) { + Store.call(this, name, engine); + this.items = {}; +} + +RotaryStore.prototype = { + async create(record) { + this.items[record.id] = record.denomination; + }, + + async remove(record) { + delete this.items[record.id]; + }, + + async update(record) { + this.items[record.id] = record.denomination; + }, + + async itemExists(id) { + return id in this.items; + }, + + async createRecord(id, collection) { + let record = new RotaryRecord(collection, id); + + if (!(id in this.items)) { + record.deleted = true; + return record; + } + + record.denomination = this.items[id] || "Data for new record: " + id; + return record; + }, + + async changeItemID(oldID, newID) { + if (oldID in this.items) { + this.items[newID] = this.items[oldID]; + } + + delete this.items[oldID]; + }, + + async getAllIDs() { + let ids = {}; + for (let id in this.items) { + ids[id] = true; + } + return ids; + }, + + async wipe() { + this.items = {}; + }, +}; + +Object.setPrototypeOf(RotaryStore.prototype, Store.prototype); + +export function RotaryTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} + +RotaryTracker.prototype = {}; +Object.setPrototypeOf(RotaryTracker.prototype, LegacyTracker.prototype); + +export function RotaryEngine(service) { + SyncEngine.call(this, "Rotary", service); + // Ensure that the engine starts with a clean slate. + this.toFetch = new SerializableSet(); + this.previousFailed = new SerializableSet(); +} + +RotaryEngine.prototype = { + _storeObj: RotaryStore, + _trackerObj: RotaryTracker, + _recordObj: RotaryRecord, + + async _findDupe(item) { + // This is a Special Value® used for testing proper reconciling on dupe + // detection. + if (item.id == "DUPE_INCOMING") { + return "DUPE_LOCAL"; + } + + for (let [id, value] of Object.entries(this._store.items)) { + if (item.denomination == value) { + return id; + } + } + return null; + }, +}; +Object.setPrototypeOf(RotaryEngine.prototype, SyncEngine.prototype); diff --git a/services/sync/modules-testing/utils.sys.mjs b/services/sync/modules-testing/utils.sys.mjs new file mode 100644 index 0000000000..498bf9872a --- /dev/null +++ b/services/sync/modules-testing/utils.sys.mjs @@ -0,0 +1,319 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { Assert } from "resource://testing-common/Assert.sys.mjs"; + +import { initTestLogging } from "resource://testing-common/services/common/logging.sys.mjs"; +import { + FakeCryptoService, + FakeFilesystemService, + FakeGUIDService, + fakeSHA256HMAC, +} from "resource://testing-common/services/sync/fakeservices.sys.mjs"; + +import { + FxAccounts, + AccountState, +} from "resource://gre/modules/FxAccounts.sys.mjs"; +import { FxAccountsClient } from "resource://gre/modules/FxAccountsClient.sys.mjs"; + +import { SCOPE_OLD_SYNC } from "resource://gre/modules/FxAccountsCommon.sys.mjs"; + +// A mock "storage manager" for FxAccounts that doesn't actually write anywhere. +export function MockFxaStorageManager() {} + +MockFxaStorageManager.prototype = { + promiseInitialized: Promise.resolve(), + + initialize(accountData) { + this.accountData = accountData; + }, + + finalize() { + return Promise.resolve(); + }, + + getAccountData(fields = null) { + let result; + if (!this.accountData) { + result = null; + } else if (fields == null) { + // can't use cloneInto as the keys get upset... + result = {}; + for (let field of Object.keys(this.accountData)) { + result[field] = this.accountData[field]; + } + } else { + if (!Array.isArray(fields)) { + fields = [fields]; + } + result = {}; + for (let field of fields) { + result[field] = this.accountData[field]; + } + } + return Promise.resolve(result); + }, + + updateAccountData(updatedFields) { + for (let [name, value] of Object.entries(updatedFields)) { + if (value == null) { + delete this.accountData[name]; + } else { + this.accountData[name] = value; + } + } + return Promise.resolve(); + }, + + deleteAccountData() { + this.accountData = null; + return Promise.resolve(); + }, +}; + +/** + * First wait >100ms (nsITimers can take up to that much time to fire, so + * we can account for the timer in delayedAutoconnect) and then two event + * loop ticks (to account for the CommonUtils.nextTick() in autoConnect). + */ +export function waitForZeroTimer(callback) { + let ticks = 2; + function wait() { + if (ticks) { + ticks -= 1; + CommonUtils.nextTick(wait); + return; + } + callback(); + } + CommonUtils.namedTimer(wait, 150, {}, "timer"); +} + +export var promiseZeroTimer = function () { + return new Promise(resolve => { + waitForZeroTimer(resolve); + }); +}; + +export var promiseNamedTimer = function (wait, thisObj, name) { + return new Promise(resolve => { + CommonUtils.namedTimer(resolve, wait, thisObj, name); + }); +}; + +// Return an identity configuration suitable for testing with our identity +// providers. |overrides| can specify overrides for any default values. +// |server| is optional, but if specified, will be used to form the cluster +// URL for the FxA identity. +export var makeIdentityConfig = function (overrides) { + // first setup the defaults. + let result = { + // Username used in both fxaccount and sync identity configs. + username: "foo", + // fxaccount specific credentials. + fxaccount: { + user: { + email: "foo", + scopedKeys: { + [SCOPE_OLD_SYNC]: { + kid: "1234567890123-u7u7u7u7u7u7u7u7u7u7uw", + k: "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqg", + kty: "oct", + }, + }, + sessionToken: "sessionToken", + uid: "a".repeat(32), + verified: true, + }, + token: { + endpoint: null, + duration: 300, + id: "id", + key: "key", + hashed_fxa_uid: "f".repeat(32), // used during telemetry validation + // uid will be set to the username. + }, + }, + }; + + // Now handle any specified overrides. + if (overrides) { + if (overrides.username) { + result.username = overrides.username; + } + if (overrides.fxaccount) { + // TODO: allow just some attributes to be specified + result.fxaccount = overrides.fxaccount; + } + if (overrides.node_type) { + result.fxaccount.token.node_type = overrides.node_type; + } + } + return result; +}; + +export var makeFxAccountsInternalMock = function (config) { + return { + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(config.fxaccount.user); + let accountState = new AccountState(storageManager); + return accountState; + }, + getOAuthToken: () => Promise.resolve("some-access-token"), + destroyOAuthToken: () => Promise.resolve(), + keys: { + getScopedKeys: () => + Promise.resolve({ + "https://identity.mozilla.com/apps/oldsync": { + identifier: "https://identity.mozilla.com/apps/oldsync", + keyRotationSecret: + "0000000000000000000000000000000000000000000000000000000000000000", + keyRotationTimestamp: 1510726317123, + }, + }), + }, + profile: { + getProfile() { + return null; + }, + }, + }; +}; + +// Configure an instance of an FxAccount identity provider with the specified +// config (or the default config if not specified). +export var configureFxAccountIdentity = function ( + authService, + config = makeIdentityConfig(), + fxaInternal = makeFxAccountsInternalMock(config) +) { + // until we get better test infrastructure for bid_identity, we set the + // signedin user's "email" to the username, simply as many tests rely on this. + config.fxaccount.user.email = config.username; + + let fxa = new FxAccounts(fxaInternal); + + let MockFxAccountsClient = function () { + FxAccountsClient.apply(this); + }; + MockFxAccountsClient.prototype = { + accountStatus() { + return Promise.resolve(true); + }, + }; + Object.setPrototypeOf( + MockFxAccountsClient.prototype, + FxAccountsClient.prototype + ); + let mockFxAClient = new MockFxAccountsClient(); + fxa._internal._fxAccountsClient = mockFxAClient; + + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth(url, oauthToken) { + Assert.equal( + url, + Services.prefs.getStringPref("identity.sync.tokenserver.uri") + ); + Assert.ok(oauthToken, "oauth token present"); + config.fxaccount.token.uid = config.username; + return config.fxaccount.token; + }, + }; + authService._fxaService = fxa; + authService._tokenServerClient = mockTSC; + // Set the "account" of the sync auth manager to be the "email" of the + // logged in user of the mockFXA service. + authService._signedInUser = config.fxaccount.user; + authService._account = config.fxaccount.user.email; +}; + +export var configureIdentity = async function (identityOverrides, server) { + let config = makeIdentityConfig(identityOverrides, server); + // Must be imported after the identity configuration is set up. + let { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" + ); + + // If a server was specified, ensure FxA has a correct cluster URL available. + if (server && !config.fxaccount.token.endpoint) { + let ep = server.baseURI; + if (!ep.endsWith("/")) { + ep += "/"; + } + ep += "1.1/" + config.username + "/"; + config.fxaccount.token.endpoint = ep; + } + + configureFxAccountIdentity(Service.identity, config); + Services.prefs.setStringPref("services.sync.username", config.username); + // many of these tests assume all the auth stuff is setup and don't hit + // a path which causes that auth to magically happen - so do it now. + await Service.identity._ensureValidToken(); + + // and cheat to avoid requiring each test do an explicit login - give it + // a cluster URL. + if (config.fxaccount.token.endpoint) { + Service.clusterURL = config.fxaccount.token.endpoint; + } +}; + +export function syncTestLogging(level = "Trace") { + let logStats = initTestLogging(level); + Services.prefs.setStringPref("services.sync.log.logger", level); + Services.prefs.setStringPref("services.sync.log.logger.engine", ""); + return logStats; +} + +export var SyncTestingInfrastructure = async function (server, username) { + let config = makeIdentityConfig({ username }); + await configureIdentity(config, server); + return { + logStats: syncTestLogging(), + fakeFilesystem: new FakeFilesystemService({}), + fakeGUIDService: new FakeGUIDService(), + fakeCryptoService: new FakeCryptoService(), + }; +}; + +/** + * Turn WBO cleartext into fake "encrypted" payload as it goes over the wire. + */ +export function encryptPayload(cleartext) { + if (typeof cleartext == "object") { + cleartext = JSON.stringify(cleartext); + } + + return { + ciphertext: cleartext, // ciphertext == cleartext with fake crypto + IV: "irrelevant", + hmac: fakeSHA256HMAC(cleartext), + }; +} + +export var sumHistogram = function (name, options = {}) { + let histogram = options.key + ? Services.telemetry.getKeyedHistogramById(name) + : Services.telemetry.getHistogramById(name); + let snapshot = histogram.snapshot(); + let sum = -Infinity; + if (snapshot) { + if (options.key && snapshot[options.key]) { + sum = snapshot[options.key].sum; + } else { + sum = snapshot.sum; + } + } + histogram.clear(); + return sum; +}; diff --git a/services/sync/modules/SyncDisconnect.sys.mjs b/services/sync/modules/SyncDisconnect.sys.mjs new file mode 100644 index 0000000000..2206a462ac --- /dev/null +++ b/services/sync/modules/SyncDisconnect.sys.mjs @@ -0,0 +1,235 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// This module provides a facility for disconnecting Sync and FxA, optionally +// sanitizing profile data as part of the process. + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs", + Log: "resource://gre/modules/Log.sys.mjs", + PREF_LAST_FXA_USER: "resource://gre/modules/FxAccountsCommon.sys.mjs", + Sanitizer: "resource:///modules/Sanitizer.sys.mjs", + Utils: "resource://services-sync/util.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +export const SyncDisconnectInternal = { + lockRetryInterval: 1000, // wait 1 seconds before trying for the lock again. + lockRetryCount: 120, // Try 120 times (==2 mins) before giving up in disgust. + promiseDisconnectFinished: null, // If we are sanitizing, a promise for completion. + + // mocked by tests. + getWeave() { + return ChromeUtils.importESModule("resource://services-sync/main.sys.mjs") + .Weave; + }, + + // Returns a promise that resolves when we are not syncing, waiting until + // a current Sync completes if necessary. Resolves with true if we + // successfully waited, in which case the sync lock will have been taken to + // ensure future syncs don't state, or resolves with false if we gave up + // waiting for the sync to complete (in which case we didn't take a lock - + // but note that Sync probably remains locked in this case regardless.) + async promiseNotSyncing(abortController) { + let weave = this.getWeave(); + let log = lazy.Log.repository.getLogger("Sync.Service"); + // We might be syncing - poll for up to 2 minutes waiting for the lock. + // (2 minutes seems extreme, but should be very rare.) + return new Promise(resolve => { + abortController.signal.onabort = () => { + resolve(false); + }; + + let attempts = 0; + let checkLock = () => { + if (abortController.signal.aborted) { + // We've already resolved, so don't want a new timer to ever start. + return; + } + if (weave.Service.lock()) { + resolve(true); + return; + } + attempts += 1; + if (attempts >= this.lockRetryCount) { + log.error( + "Gave up waiting for the sync lock - going ahead with sanitize anyway" + ); + resolve(false); + return; + } + log.debug("Waiting a couple of seconds to get the sync lock"); + lazy.setTimeout(checkLock, this.lockRetryInterval); + }; + checkLock(); + }); + }, + + // Sanitize Sync-related data. + async doSanitizeSyncData() { + let weave = this.getWeave(); + // Get the sync logger - if stuff goes wrong it can be useful to have that + // recorded in the sync logs. + let log = lazy.Log.repository.getLogger("Sync.Service"); + log.info("Starting santitize of Sync data"); + try { + // We clobber data for all Sync engines that are enabled. + await weave.Service.promiseInitialized; + weave.Service.enabled = false; + + log.info("starting actual sanitization"); + for (let engine of weave.Service.engineManager.getAll()) { + if (engine.enabled) { + try { + log.info("Wiping engine", engine.name); + await engine.wipeClient(); + } catch (ex) { + log.error("Failed to wipe engine", ex); + } + } + } + // Reset the pref which is used to show a warning when a different user + // signs in - this is no longer a concern now that we've removed the + // data from the profile. + Services.prefs.clearUserPref(lazy.PREF_LAST_FXA_USER); + + log.info("Finished wiping sync data"); + } catch (ex) { + log.error("Failed to sanitize Sync data", ex); + console.error("Failed to sanitize Sync data", ex); + } + try { + // ensure any logs we wrote are flushed to disk. + await weave.Service.errorHandler.resetFileLog(); + } catch (ex) { + console.log("Failed to flush the Sync log", ex); + } + }, + + // Sanitize all Browser data. + async doSanitizeBrowserData() { + try { + // sanitize everything other than "open windows" (and we don't do that + // because it may confuse the user - they probably want to see + // about:prefs with the disconnection reflected. + let itemsToClear = Object.keys(lazy.Sanitizer.items).filter( + k => k != "openWindows" + ); + await lazy.Sanitizer.sanitize(itemsToClear); + } catch (ex) { + console.error("Failed to sanitize other data", ex); + } + }, + + async doSyncAndAccountDisconnect(shouldUnlock) { + // We do a startOver of Sync first - if we do the account first we end + // up with Sync configured but FxA not configured, which causes the browser + // UI to briefly enter a "needs reauth" state. + let Weave = this.getWeave(); + await Weave.Service.promiseInitialized; + await Weave.Service.startOver(); + await lazy.fxAccounts.signOut(); + // Sync may have been disabled if we santized, so re-enable it now or + // else the user will be unable to resync should they sign in before a + // restart. + Weave.Service.enabled = true; + + // and finally, if we managed to get the lock before, we should unlock it + // now. + if (shouldUnlock) { + Weave.Service.unlock(); + } + }, + + // Start the sanitization process. Returns a promise that resolves when + // the sanitize is complete, and an AbortController which can be used to + // abort the process of waiting for a sync to complete. + async _startDisconnect(abortController, sanitizeData = false) { + // This is a bit convoluted - we want to wait for a sync to finish before + // sanitizing, but want to abort that wait if the browser shuts down while + // we are waiting (in which case we'll charge ahead anyway). + // So we do this by using an AbortController and passing that to the + // function that waits for the sync lock - it will immediately resolve + // if the abort controller is aborted. + let log = lazy.Log.repository.getLogger("Sync.Service"); + + // If the master-password is locked then we will fail to fully sanitize, + // so prompt for that now. If canceled, we just abort now. + log.info("checking master-password state"); + if (!lazy.Utils.ensureMPUnlocked()) { + log.warn( + "The master-password needs to be unlocked to fully disconnect from sync" + ); + return; + } + + log.info("waiting for any existing syncs to complete"); + let locked = await this.promiseNotSyncing(abortController); + + if (sanitizeData) { + await this.doSanitizeSyncData(); + + // We disconnect before sanitizing the browser data - in a worst-case + // scenario where the sanitize takes so long that even the shutdown + // blocker doesn't allow it to finish, we should still at least be in + // a disconnected state on the next startup. + log.info("disconnecting account"); + await this.doSyncAndAccountDisconnect(locked); + + await this.doSanitizeBrowserData(); + } else { + log.info("disconnecting account"); + await this.doSyncAndAccountDisconnect(locked); + } + }, + + async disconnect(sanitizeData) { + if (this.promiseDisconnectFinished) { + throw new Error("A disconnect is already in progress"); + } + let abortController = new AbortController(); + let promiseDisconnectFinished = this._startDisconnect( + abortController, + sanitizeData + ); + this.promiseDisconnectFinished = promiseDisconnectFinished; + let shutdownBlocker = () => { + // oh dear - we are sanitizing (probably stuck waiting for a sync to + // complete) and the browser is shutting down. Let's avoid the wait + // for sync to complete and continue the process anyway. + abortController.abort(); + return promiseDisconnectFinished; + }; + lazy.AsyncShutdown.quitApplicationGranted.addBlocker( + "SyncDisconnect: removing requested data", + shutdownBlocker + ); + + // wait for it to finish - hopefully without the blocker being called. + await promiseDisconnectFinished; + this.promiseDisconnectFinished = null; + + // sanitize worked so remove our blocker - it's a noop if the blocker + // did call us. + lazy.AsyncShutdown.quitApplicationGranted.removeBlocker(shutdownBlocker); + }, +}; + +export const SyncDisconnect = { + get promiseDisconnectFinished() { + return SyncDisconnectInternal.promiseDisconnectFinished; + }, + + disconnect(sanitizeData) { + return SyncDisconnectInternal.disconnect(sanitizeData); + }, +}; diff --git a/services/sync/modules/SyncedTabs.sys.mjs b/services/sync/modules/SyncedTabs.sys.mjs new file mode 100644 index 0000000000..410244413e --- /dev/null +++ b/services/sync/modules/SyncedTabs.sys.mjs @@ -0,0 +1,348 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + CLIENT_NOT_CONFIGURED: "resource://services-sync/constants.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", +}); + +// The Sync XPCOM service +ChromeUtils.defineLazyGetter(lazy, "weaveXPCService", function () { + return Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; +}); + +// from MDN... +function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +// A topic we fire whenever we have new tabs available. This might be due +// to a request made by this module to refresh the tab list, or as the result +// of a regularly scheduled sync. The intent is that consumers just listen +// for this notification and update their UI in response. +const TOPIC_TABS_CHANGED = "services.sync.tabs.changed"; + +// The interval, in seconds, before which we consider the existing list +// of tabs "fresh enough" and don't force a new sync. +const TABS_FRESH_ENOUGH_INTERVAL_SECONDS = 30; + +ChromeUtils.defineLazyGetter(lazy, "log", () => { + const { Log } = ChromeUtils.importESModule( + "resource://gre/modules/Log.sys.mjs" + ); + let log = Log.repository.getLogger("Sync.RemoteTabs"); + log.manageLevelFromPref("services.sync.log.logger.tabs"); + return log; +}); + +// A private singleton that does the work. +let SyncedTabsInternal = { + /* Make a "tab" record. Returns a promise */ + async _makeTab(client, tab, url, showRemoteIcons) { + let icon; + if (showRemoteIcons) { + icon = tab.icon; + } + if (!icon) { + // By not specifying a size the favicon service will pick the default, + // that is usually set through setDefaultIconURIPreferredSize by the + // first browser window. Commonly it's 16px at current dpi. + icon = "page-icon:" + url; + } + return { + type: "tab", + title: tab.title || url, + url, + icon, + client: client.id, + lastUsed: tab.lastUsed, + inactive: tab.inactive, + }; + }, + + /* Make a "client" record. Returns a promise for consistency with _makeTab */ + async _makeClient(client) { + return { + id: client.id, + type: "client", + name: lazy.Weave.Service.clientsEngine.getClientName(client.id), + clientType: lazy.Weave.Service.clientsEngine.getClientType(client.id), + lastModified: client.lastModified * 1000, // sec to ms + tabs: [], + }; + }, + + _tabMatchesFilter(tab, filter) { + let reFilter = new RegExp(escapeRegExp(filter), "i"); + return reFilter.test(tab.url) || reFilter.test(tab.title); + }, + + _createRecentTabsList( + clients, + maxCount, + extraParams = { removeAllDupes: true, removeDeviceDupes: false } + ) { + let tabs = []; + + for (let client of clients) { + if (extraParams.removeDeviceDupes) { + client.tabs = this._filterRecentTabsDupes(client.tabs); + } + for (let tab of client.tabs) { + tab.device = client.name; + tab.deviceType = client.clientType; + } + tabs = [...tabs, ...client.tabs.reverse()]; + } + if (extraParams.removeAllDupes) { + tabs = this._filterRecentTabsDupes(tabs); + } + tabs = tabs.sort((a, b) => b.lastUsed - a.lastUsed).slice(0, maxCount); + return tabs; + }, + + _filterRecentTabsDupes(tabs) { + // Filter out any tabs with duplicate URLs preserving + // the duplicate with the most recent lastUsed value + return tabs.filter(tab => { + return !tabs.some(t => { + return t.url === tab.url && tab.lastUsed < t.lastUsed; + }); + }); + }, + + async getTabClients(filter) { + lazy.log.info("Generating tab list with filter", filter); + let result = []; + + // If Sync isn't ready, don't try and get anything. + if (!lazy.weaveXPCService.ready) { + lazy.log.debug("Sync isn't yet ready, so returning an empty tab list"); + return result; + } + + // A boolean that controls whether we should show the icon from the remote tab. + const showRemoteIcons = Services.prefs.getBoolPref( + "services.sync.syncedTabs.showRemoteIcons", + true + ); + + let engine = lazy.Weave.Service.engineManager.get("tabs"); + + let ntabs = 0; + let clientTabList = await engine.getAllClients(); + for (let client of clientTabList) { + if (!lazy.Weave.Service.clientsEngine.remoteClientExists(client.id)) { + continue; + } + let clientRepr = await this._makeClient(client); + lazy.log.debug("Processing client", clientRepr); + + for (let tab of client.tabs) { + let url = tab.urlHistory[0]; + lazy.log.trace("remote tab", url); + + if (!url) { + continue; + } + let tabRepr = await this._makeTab(client, tab, url, showRemoteIcons); + if (filter && !this._tabMatchesFilter(tabRepr, filter)) { + continue; + } + clientRepr.tabs.push(tabRepr); + } + // We return all clients, even those without tabs - the consumer should + // filter it if they care. + ntabs += clientRepr.tabs.length; + result.push(clientRepr); + } + lazy.log.info( + `Final tab list has ${result.length} clients with ${ntabs} tabs.` + ); + return result; + }, + + async syncTabs(force) { + if (!force) { + // Don't bother refetching tabs if we already did so recently + let lastFetch = Services.prefs.getIntPref( + "services.sync.lastTabFetch", + 0 + ); + let now = Math.floor(Date.now() / 1000); + if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL_SECONDS) { + lazy.log.info("_refetchTabs was done recently, do not doing it again"); + return false; + } + } + + // If Sync isn't configured don't try and sync, else we will get reports + // of a login failure. + if (lazy.Weave.Status.checkSetup() === lazy.CLIENT_NOT_CONFIGURED) { + lazy.log.info( + "Sync client is not configured, so not attempting a tab sync" + ); + return false; + } + // If the primary pass is locked, we should not try to sync + if (lazy.Weave.Utils.mpLocked()) { + lazy.log.info( + "Can't sync tabs due to the primary password being locked", + lazy.Weave.Status.login + ); + return false; + } + // Ask Sync to just do the tabs engine if it can. + try { + lazy.log.info("Doing a tab sync."); + await lazy.Weave.Service.sync({ why: "tabs", engines: ["tabs"] }); + return true; + } catch (ex) { + lazy.log.error("Sync failed", ex); + throw ex; + } + }, + + observe(subject, topic, data) { + lazy.log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`); + switch (topic) { + case "weave:engine:sync:finish": + if (data != "tabs") { + return; + } + // The tabs engine just finished syncing + // Set our lastTabFetch pref here so it tracks both explicit sync calls + // and normally scheduled ones. + Services.prefs.setIntPref( + "services.sync.lastTabFetch", + Math.floor(Date.now() / 1000) + ); + Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED); + break; + case "weave:service:start-over": + // start-over needs to notify so consumers find no tabs. + Services.prefs.clearUserPref("services.sync.lastTabFetch"); + Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED); + break; + case "nsPref:changed": + Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED); + break; + default: + break; + } + }, + + // Returns true if Sync is configured to Sync tabs, false otherwise + get isConfiguredToSyncTabs() { + if (!lazy.weaveXPCService.ready) { + lazy.log.debug("Sync isn't yet ready; assuming tab engine is enabled"); + return true; + } + + let engine = lazy.Weave.Service.engineManager.get("tabs"); + return engine && engine.enabled; + }, + + get hasSyncedThisSession() { + let engine = lazy.Weave.Service.engineManager.get("tabs"); + return engine && engine.hasSyncedThisSession; + }, +}; + +Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish"); +Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over"); +// Observe the pref the indicates the state of the tabs engine has changed. +// This will force consumers to re-evaluate the state of sync and update +// accordingly. +Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal); + +// The public interface. +export var SyncedTabs = { + // A mock-point for tests. + _internal: SyncedTabsInternal, + + // We make the topic for the observer notification public. + TOPIC_TABS_CHANGED, + + // Expose the interval used to determine if synced tabs data needs a new sync + TABS_FRESH_ENOUGH_INTERVAL_SECONDS, + + // Returns true if Sync is configured to Sync tabs, false otherwise + get isConfiguredToSyncTabs() { + return this._internal.isConfiguredToSyncTabs; + }, + + // Returns true if a tab sync has completed once this session. If this + // returns false, then getting back no clients/tabs possibly just means we + // are waiting for that first sync to complete. + get hasSyncedThisSession() { + return this._internal.hasSyncedThisSession; + }, + + // Return a promise that resolves with an array of client records, each with + // a .tabs array. Note that part of the contract for this module is that the + // returned objects are not shared between invocations, so callers are free + // to mutate the returned objects (eg, sort, truncate) however they see fit. + getTabClients(query) { + return this._internal.getTabClients(query); + }, + + // Starts a background request to start syncing tabs. Returns a promise that + // resolves when the sync is complete, but there's no resolved value - + // callers should be listening for TOPIC_TABS_CHANGED. + // If |force| is true we always sync. If false, we only sync if the most + // recent sync wasn't "recently". + syncTabs(force) { + return this._internal.syncTabs(force); + }, + + createRecentTabsList(clients, maxCount, extraParams) { + return this._internal._createRecentTabsList(clients, maxCount, extraParams); + }, + + sortTabClientsByLastUsed(clients) { + // First sort the list of tabs for each client. Note that + // this module promises that the objects it returns are never + // shared, so we are free to mutate those objects directly. + for (let client of clients) { + let tabs = client.tabs; + tabs.sort((a, b) => b.lastUsed - a.lastUsed); + } + // Now sort the clients - the clients are sorted in the order of the + // most recent tab for that client (ie, it is important the tabs for + // each client are already sorted.) + clients.sort((a, b) => { + if (!a.tabs.length) { + return 1; // b comes first. + } + if (!b.tabs.length) { + return -1; // a comes first. + } + return b.tabs[0].lastUsed - a.tabs[0].lastUsed; + }); + }, + + recordSyncedTabsTelemetry(object, tabEvent, extraOptions) { + Services.telemetry.setEventRecordingEnabled("synced_tabs", true); + Services.telemetry.recordEvent( + "synced_tabs", + tabEvent, + object, + null, + extraOptions + ); + }, + + // Get list of synced tabs across all devices/clients + // truncated by value of maxCount param, sorted by + // lastUsed value, and filtered for duplicate URLs + async getRecentTabs(maxCount, extraParams) { + let clients = await this.getTabClients(); + return this._internal._createRecentTabsList(clients, maxCount, extraParams); + }, +}; diff --git a/services/sync/modules/UIState.sys.mjs b/services/sync/modules/UIState.sys.mjs new file mode 100644 index 0000000000..8981d81f7d --- /dev/null +++ b/services/sync/modules/UIState.sys.mjs @@ -0,0 +1,285 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * @typedef {Object} UIState + * @property {string} status The Sync/FxA status, see STATUS_* constants. + * @property {string} [email] The FxA email configured to log-in with Sync. + * @property {string} [displayName] The user's FxA display name. + * @property {string} [avatarURL] The user's FxA avatar URL. + * @property {Date} [lastSync] The last sync time. + * @property {boolean} [syncing] Whether or not we are currently syncing. + */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + LOGIN_FAILED_LOGIN_REJECTED: "resource://services-sync/constants.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", +}); + +const TOPICS = [ + "weave:connected", + "weave:service:login:got-hashed-id", + "weave:service:login:error", + "weave:service:ready", + "weave:service:sync:start", + "weave:service:sync:finish", + "weave:service:sync:error", + "weave:service:start-over:finish", + "fxaccounts:onverified", + "fxaccounts:onlogin", // Defined in FxAccountsCommon, pulling it is expensive. + "fxaccounts:onlogout", + "fxaccounts:profilechange", + "fxaccounts:statechange", +]; + +const ON_UPDATE = "sync-ui-state:update"; + +const STATUS_NOT_CONFIGURED = "not_configured"; +const STATUS_LOGIN_FAILED = "login_failed"; +const STATUS_NOT_VERIFIED = "not_verified"; +const STATUS_SIGNED_IN = "signed_in"; + +const DEFAULT_STATE = { + status: STATUS_NOT_CONFIGURED, +}; + +const UIStateInternal = { + _initialized: false, + _state: null, + + // We keep _syncing out of the state object because we can only track it + // using sync events and we can't determine it at any point in time. + _syncing: false, + + get state() { + if (!this._state) { + return DEFAULT_STATE; + } + return Object.assign({}, this._state, { syncing: this._syncing }); + }, + + isReady() { + if (!this._initialized) { + this.init(); + return false; + } + return true; + }, + + init() { + this._initialized = true; + // Because the FxA toolbar is usually visible, this module gets loaded at + // browser startup, and we want to avoid pulling in all of FxA or Sync at + // that time, so we refresh the state after the browser has settled. + Services.tm.idleDispatchToMainThread(() => { + this.refreshState().catch(e => { + console.error(e); + }); + }, 2000); + }, + + // Used for testing. + reset() { + this._state = null; + this._syncing = false; + this._initialized = false; + }, + + observe(subject, topic, data) { + switch (topic) { + case "weave:service:sync:start": + this.toggleSyncActivity(true); + break; + case "weave:service:sync:finish": + case "weave:service:sync:error": + this.toggleSyncActivity(false); + break; + default: + this.refreshState().catch(e => { + console.error(e); + }); + break; + } + }, + + // Builds a new state from scratch. + async refreshState() { + const newState = {}; + await this._refreshFxAState(newState); + // Optimize the "not signed in" case to avoid refreshing twice just after + // startup - if there's currently no _state, and we still aren't configured, + // just early exit. + if (this._state == null && newState.status == DEFAULT_STATE.status) { + return this.state; + } + if (newState.syncEnabled) { + this._setLastSyncTime(newState); // We want this in case we change accounts. + } + this._state = newState; + + this.notifyStateUpdated(); + return this.state; + }, + + // Update the current state with the last sync time/currently syncing status. + toggleSyncActivity(syncing) { + this._syncing = syncing; + this._setLastSyncTime(this._state); + + this.notifyStateUpdated(); + }, + + notifyStateUpdated() { + Services.obs.notifyObservers(null, ON_UPDATE); + }, + + async _refreshFxAState(newState) { + let userData = await this._getUserData(); + await this._populateWithUserData(newState, userData); + }, + + async _populateWithUserData(state, userData) { + let status; + let syncUserName = Services.prefs.getStringPref( + "services.sync.username", + "" + ); + if (!userData) { + // If Sync thinks it is configured but there's no FxA user, then we + // want to enter the "login failed" state so the user can get + // reconfigured. + if (syncUserName) { + state.email = syncUserName; + status = STATUS_LOGIN_FAILED; + } else { + // everyone agrees nothing is configured. + status = STATUS_NOT_CONFIGURED; + } + } else { + let loginFailed = await this._loginFailed(); + if (loginFailed) { + status = STATUS_LOGIN_FAILED; + } else if (!userData.verified) { + status = STATUS_NOT_VERIFIED; + } else { + status = STATUS_SIGNED_IN; + } + state.uid = userData.uid; + state.email = userData.email; + state.displayName = userData.displayName; + // for better or worse, this module renames these attribues. + state.avatarURL = userData.avatar; + state.avatarIsDefault = userData.avatarDefault; + state.syncEnabled = !!syncUserName; + } + state.status = status; + }, + + async _getUserData() { + try { + return await this.fxAccounts.getSignedInUser(); + } catch (e) { + // This is most likely in tests, where we quickly log users in and out. + // The most likely scenario is a user logged out, so reflect that. + // Bug 995134 calls for better errors so we could retry if we were + // sure this was the failure reason. + console.error("Error updating FxA account info:", e); + return null; + } + }, + + _setLastSyncTime(state) { + if (state?.status == UIState.STATUS_SIGNED_IN) { + const lastSync = Services.prefs.getStringPref( + "services.sync.lastSync", + null + ); + state.lastSync = lastSync ? new Date(lastSync) : null; + } + }, + + async _loginFailed() { + // First ask FxA if it thinks the user needs re-authentication. In practice, + // this check is probably canonical (ie, we probably don't really need + // the check below at all as we drop local session info on the first sign + // of a problem) - but we keep it for now to keep the risk down. + let hasLocalSession = await this.fxAccounts.hasLocalSession(); + if (!hasLocalSession) { + return true; + } + + // Referencing Weave.Service will implicitly initialize sync, and we don't + // want to force that - so first check if it is ready. + let service = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + if (!service.ready) { + return false; + } + // LOGIN_FAILED_LOGIN_REJECTED explicitly means "you must log back in". + // All other login failures are assumed to be transient and should go + // away by themselves, so aren't reflected here. + return lazy.Weave.Status.login == lazy.LOGIN_FAILED_LOGIN_REJECTED; + }, + + set fxAccounts(mockFxAccounts) { + delete this.fxAccounts; + this.fxAccounts = mockFxAccounts; + }, +}; + +ChromeUtils.defineLazyGetter(UIStateInternal, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +for (let topic of TOPICS) { + Services.obs.addObserver(UIStateInternal, topic); +} + +export var UIState = { + _internal: UIStateInternal, + + ON_UPDATE, + + STATUS_NOT_CONFIGURED, + STATUS_LOGIN_FAILED, + STATUS_NOT_VERIFIED, + STATUS_SIGNED_IN, + + /** + * Returns true if the module has been initialized and the state set. + * If not, return false and trigger an init in the background. + */ + isReady() { + return this._internal.isReady(); + }, + + /** + * @returns {UIState} The current Sync/FxA UI State. + */ + get() { + return this._internal.state; + }, + + /** + * Refresh the state. Used for testing, don't call this directly since + * UIState already listens to Sync/FxA notifications to determine if the state + * needs to be refreshed. ON_UPDATE will be fired once the state is refreshed. + * + * @returns {Promise<UIState>} Resolved once the state is refreshed. + */ + refresh() { + return this._internal.refreshState(); + }, + + /** + * Reset the state of the whole module. Used for testing. + */ + reset() { + this._internal.reset(); + }, +}; diff --git a/services/sync/modules/addonsreconciler.sys.mjs b/services/sync/modules/addonsreconciler.sys.mjs new file mode 100644 index 0000000000..902f57348e --- /dev/null +++ b/services/sync/modules/addonsreconciler.sys.mjs @@ -0,0 +1,584 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains middleware to reconcile state of AddonManager for + * purposes of tracking events for Sync. The content in this file exists + * because AddonManager does not have a getChangesSinceX() API and adding + * that functionality properly was deemed too time-consuming at the time + * add-on sync was originally written. If/when AddonManager adds this API, + * this file can go away and the add-ons engine can be rewritten to use it. + * + * It was decided to have this tracking functionality exist in a separate + * standalone file so it could be more easily understood, tested, and + * hopefully ported. + */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { AddonManager } from "resource://gre/modules/AddonManager.sys.mjs"; + +const DEFAULT_STATE_FILE = "addonsreconciler"; + +export var CHANGE_INSTALLED = 1; +export var CHANGE_UNINSTALLED = 2; +export var CHANGE_ENABLED = 3; +export var CHANGE_DISABLED = 4; + +/** + * Maintains state of add-ons. + * + * State is maintained in 2 data structures, an object mapping add-on IDs + * to metadata and an array of changes over time. The object mapping can be + * thought of as a minimal copy of data from AddonManager which is needed for + * Sync. The array is effectively a log of changes over time. + * + * The data structures are persisted to disk by serializing to a JSON file in + * the current profile. The data structures are updated by 2 mechanisms. First, + * they can be refreshed from the global state of the AddonManager. This is a + * sure-fire way of ensuring the reconciler is up to date. Second, the + * reconciler adds itself as an AddonManager listener. When it receives change + * notifications, it updates its internal state incrementally. + * + * The internal state is persisted to a JSON file in the profile directory. + * + * An instance of this is bound to an AddonsEngine instance. In reality, it + * likely exists as a singleton. To AddonsEngine, it functions as a store and + * an entity which emits events for tracking. + * + * The usage pattern for instances of this class is: + * + * let reconciler = new AddonsReconciler(...); + * await reconciler.ensureStateLoaded(); + * + * // At this point, your instance should be ready to use. + * + * When you are finished with the instance, please call: + * + * reconciler.stopListening(); + * await reconciler.saveState(...); + * + * This class uses the AddonManager AddonListener interface. + * When an add-on is installed, listeners are called in the following order: + * AL.onInstalling, AL.onInstalled + * + * For uninstalls, we see AL.onUninstalling then AL.onUninstalled. + * + * Enabling and disabling work by sending: + * + * AL.onEnabling, AL.onEnabled + * AL.onDisabling, AL.onDisabled + * + * Actions can be undone. All undoable actions notify the same + * AL.onOperationCancelled event. We treat this event like any other. + * + * When an add-on is uninstalled from about:addons, the user is offered an + * "Undo" option, which leads to the following sequence of events as + * observed by an AddonListener: + * Add-ons are first disabled then they are actually uninstalled. So, we will + * see AL.onDisabling and AL.onDisabled. The onUninstalling and onUninstalled + * events only come after the Addon Manager is closed or another view is + * switched to. In the case of Sync performing the uninstall, the uninstall + * events will occur immediately. However, we still see disabling events and + * heed them like they were normal. In the end, the state is proper. + */ +export function AddonsReconciler(queueCaller) { + this._log = Log.repository.getLogger("Sync.AddonsReconciler"); + this._log.manageLevelFromPref("services.sync.log.logger.addonsreconciler"); + this.queueCaller = queueCaller; + + Svc.Obs.add("xpcom-shutdown", this.stopListening, this); +} + +AddonsReconciler.prototype = { + /** Flag indicating whether we are listening to AddonManager events. */ + _listening: false, + + /** + * Define this as false if the reconciler should not persist state + * to disk when handling events. + * + * This allows test code to avoid spinning to write during observer + * notifications and xpcom shutdown, which appears to cause hangs on WinXP + * (Bug 873861). + */ + _shouldPersist: true, + + /** Log logger instance */ + _log: null, + + /** + * Container for add-on metadata. + * + * Keys are add-on IDs. Values are objects which describe the state of the + * add-on. This is a minimal mirror of data that can be queried from + * AddonManager. In some cases, we retain data longer than AddonManager. + */ + _addons: {}, + + /** + * List of add-on changes over time. + * + * Each element is an array of [time, change, id]. + */ + _changes: [], + + /** + * Objects subscribed to changes made to this instance. + */ + _listeners: [], + + /** + * Accessor for add-ons in this object. + * + * Returns an object mapping add-on IDs to objects containing metadata. + */ + get addons() { + return this._addons; + }, + + async ensureStateLoaded() { + if (!this._promiseStateLoaded) { + this._promiseStateLoaded = this.loadState(); + } + return this._promiseStateLoaded; + }, + + /** + * Load reconciler state from a file. + * + * The path is relative to the weave directory in the profile. If no + * path is given, the default one is used. + * + * If the file does not exist or there was an error parsing the file, the + * state will be transparently defined as empty. + * + * @param file + * Path to load. ".json" is appended automatically. If not defined, + * a default path will be consulted. + */ + async loadState(file = DEFAULT_STATE_FILE) { + let json = await Utils.jsonLoad(file, this); + this._addons = {}; + this._changes = []; + + if (!json) { + this._log.debug("No data seen in loaded file: " + file); + return false; + } + + let version = json.version; + if (!version || version != 1) { + this._log.error( + "Could not load JSON file because version not " + + "supported: " + + version + ); + return false; + } + + this._addons = json.addons; + for (let id in this._addons) { + let record = this._addons[id]; + record.modified = new Date(record.modified); + } + + for (let [time, change, id] of json.changes) { + this._changes.push([new Date(time), change, id]); + } + + return true; + }, + + /** + * Saves the current state to a file in the local profile. + * + * @param file + * String path in profile to save to. If not defined, the default + * will be used. + */ + async saveState(file = DEFAULT_STATE_FILE) { + let state = { version: 1, addons: {}, changes: [] }; + + for (let [id, record] of Object.entries(this._addons)) { + state.addons[id] = {}; + for (let [k, v] of Object.entries(record)) { + if (k == "modified") { + state.addons[id][k] = v.getTime(); + } else { + state.addons[id][k] = v; + } + } + } + + for (let [time, change, id] of this._changes) { + state.changes.push([time.getTime(), change, id]); + } + + this._log.info("Saving reconciler state to file: " + file); + await Utils.jsonSave(file, this, state); + }, + + /** + * Registers a change listener with this instance. + * + * Change listeners are called every time a change is recorded. The listener + * is an object with the function "changeListener" that takes 3 arguments, + * the Date at which the change happened, the type of change (a CHANGE_* + * constant), and the add-on state object reflecting the current state of + * the add-on at the time of the change. + * + * @param listener + * Object containing changeListener function. + */ + addChangeListener: function addChangeListener(listener) { + if (!this._listeners.includes(listener)) { + this._log.debug("Adding change listener."); + this._listeners.push(listener); + } + }, + + /** + * Removes a previously-installed change listener from the instance. + * + * @param listener + * Listener instance to remove. + */ + removeChangeListener: function removeChangeListener(listener) { + this._listeners = this._listeners.filter(element => { + if (element == listener) { + this._log.debug("Removing change listener."); + return false; + } + return true; + }); + }, + + /** + * Tells the instance to start listening for AddonManager changes. + * + * This is typically called automatically when Sync is loaded. + */ + startListening: function startListening() { + if (this._listening) { + return; + } + + this._log.info("Registering as Add-on Manager listener."); + AddonManager.addAddonListener(this); + this._listening = true; + }, + + /** + * Tells the instance to stop listening for AddonManager changes. + * + * The reconciler should always be listening. This should only be called when + * the instance is being destroyed. + * + * This function will get called automatically on XPCOM shutdown. However, it + * is a best practice to call it yourself. + */ + stopListening: function stopListening() { + if (!this._listening) { + return; + } + + this._log.debug("Stopping listening and removing AddonManager listener."); + AddonManager.removeAddonListener(this); + this._listening = false; + }, + + /** + * Refreshes the global state of add-ons by querying the AddonManager. + */ + async refreshGlobalState() { + this._log.info("Refreshing global state from AddonManager."); + + let installs; + let addons = await AddonManager.getAllAddons(); + + let ids = {}; + + for (let addon of addons) { + ids[addon.id] = true; + await this.rectifyStateFromAddon(addon); + } + + // Look for locally-defined add-ons that no longer exist and update their + // record. + for (let [id, addon] of Object.entries(this._addons)) { + if (id in ids) { + continue; + } + + // If the id isn't in ids, it means that the add-on has been deleted or + // the add-on is in the process of being installed. We detect the + // latter by seeing if an AddonInstall is found for this add-on. + + if (!installs) { + installs = await AddonManager.getAllInstalls(); + } + + let installFound = false; + for (let install of installs) { + if ( + install.addon && + install.addon.id == id && + install.state == AddonManager.STATE_INSTALLED + ) { + installFound = true; + break; + } + } + + if (installFound) { + continue; + } + + if (addon.installed) { + addon.installed = false; + this._log.debug( + "Adding change because add-on not present in " + + "Add-on Manager: " + + id + ); + await this._addChange(new Date(), CHANGE_UNINSTALLED, addon); + } + } + + // See note for _shouldPersist. + if (this._shouldPersist) { + await this.saveState(); + } + }, + + /** + * Rectifies the state of an add-on from an Addon instance. + * + * This basically says "given an Addon instance, assume it is truth and + * apply changes to the local state to reflect it." + * + * This function could result in change listeners being called if the local + * state differs from the passed add-on's state. + * + * @param addon + * Addon instance being updated. + */ + async rectifyStateFromAddon(addon) { + this._log.debug( + `Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})` + ); + + let id = addon.id; + let enabled = !addon.userDisabled; + let guid = addon.syncGUID; + let now = new Date(); + + if (!(id in this._addons)) { + let record = { + id, + guid, + enabled, + installed: true, + modified: now, + type: addon.type, + scope: addon.scope, + foreignInstall: addon.foreignInstall, + isSyncable: addon.isSyncable, + }; + this._addons[id] = record; + this._log.debug( + "Adding change because add-on not present locally: " + id + ); + await this._addChange(now, CHANGE_INSTALLED, record); + return; + } + + let record = this._addons[id]; + record.isSyncable = addon.isSyncable; + + if (!record.installed) { + // It is possible the record is marked as uninstalled because an + // uninstall is pending. + if (!(addon.pendingOperations & AddonManager.PENDING_UNINSTALL)) { + record.installed = true; + record.modified = now; + } + } + + if (record.enabled != enabled) { + record.enabled = enabled; + record.modified = now; + let change = enabled ? CHANGE_ENABLED : CHANGE_DISABLED; + this._log.debug("Adding change because enabled state changed: " + id); + await this._addChange(new Date(), change, record); + } + + if (record.guid != guid) { + record.guid = guid; + // We don't record a change because the Sync engine rectifies this on its + // own. This is tightly coupled with Sync. If this code is ever lifted + // outside of Sync, this exception should likely be removed. + } + }, + + /** + * Record a change in add-on state. + * + * @param date + * Date at which the change occurred. + * @param change + * The type of the change. A CHANGE_* constant. + * @param state + * The new state of the add-on. From this.addons. + */ + async _addChange(date, change, state) { + this._log.info("Change recorded for " + state.id); + this._changes.push([date, change, state.id]); + + for (let listener of this._listeners) { + try { + await listener.changeListener(date, change, state); + } catch (ex) { + this._log.error("Exception calling change listener", ex); + } + } + }, + + /** + * Obtain the set of changes to add-ons since the date passed. + * + * This will return an array of arrays. Each entry in the array has the + * elements [date, change_type, id], where + * + * date - Date instance representing when the change occurred. + * change_type - One of CHANGE_* constants. + * id - ID of add-on that changed. + */ + getChangesSinceDate(date) { + let length = this._changes.length; + for (let i = 0; i < length; i++) { + if (this._changes[i][0] >= date) { + return this._changes.slice(i); + } + } + + return []; + }, + + /** + * Prunes all recorded changes from before the specified Date. + * + * @param date + * Entries older than this Date will be removed. + */ + pruneChangesBeforeDate(date) { + this._changes = this._changes.filter(function test_age(change) { + return change[0] >= date; + }); + }, + + /** + * Obtains the set of all known Sync GUIDs for add-ons. + */ + getAllSyncGUIDs() { + let result = {}; + for (let id in this.addons) { + result[id] = true; + } + + return result; + }, + + /** + * Obtain the add-on state record for an add-on by Sync GUID. + * + * If the add-on could not be found, returns null. + * + * @param guid + * Sync GUID of add-on to retrieve. + */ + getAddonStateFromSyncGUID(guid) { + for (let id in this.addons) { + let addon = this.addons[id]; + if (addon.guid == guid) { + return addon; + } + } + + return null; + }, + + /** + * Handler that is invoked as part of the AddonManager listeners. + */ + async _handleListener(action, addon) { + // Since this is called as an observer, we explicitly trap errors and + // log them to ourselves so we don't see errors reported elsewhere. + try { + let id = addon.id; + this._log.debug("Add-on change: " + action + " to " + id); + + switch (action) { + case "onEnabled": + case "onDisabled": + case "onInstalled": + case "onInstallEnded": + case "onOperationCancelled": + await this.rectifyStateFromAddon(addon); + break; + + case "onUninstalled": + let id = addon.id; + let addons = this.addons; + if (id in addons) { + let now = new Date(); + let record = addons[id]; + record.installed = false; + record.modified = now; + this._log.debug( + "Adding change because of uninstall listener: " + id + ); + await this._addChange(now, CHANGE_UNINSTALLED, record); + } + } + + // See note for _shouldPersist. + if (this._shouldPersist) { + await this.saveState(); + } + } catch (ex) { + this._log.warn("Exception", ex); + } + }, + + // AddonListeners + onEnabled: function onEnabled(addon) { + this.queueCaller.enqueueCall(() => + this._handleListener("onEnabled", addon) + ); + }, + onDisabled: function onDisabled(addon) { + this.queueCaller.enqueueCall(() => + this._handleListener("onDisabled", addon) + ); + }, + onInstalled: function onInstalled(addon) { + this.queueCaller.enqueueCall(() => + this._handleListener("onInstalled", addon) + ); + }, + onUninstalled: function onUninstalled(addon) { + this.queueCaller.enqueueCall(() => + this._handleListener("onUninstalled", addon) + ); + }, + onOperationCancelled: function onOperationCancelled(addon) { + this.queueCaller.enqueueCall(() => + this._handleListener("onOperationCancelled", addon) + ); + }, +}; diff --git a/services/sync/modules/addonutils.sys.mjs b/services/sync/modules/addonutils.sys.mjs new file mode 100644 index 0000000000..08a8c5b5f0 --- /dev/null +++ b/services/sync/modules/addonutils.sys.mjs @@ -0,0 +1,391 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Svc } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AddonRepository: "resource://gre/modules/addons/AddonRepository.sys.mjs", +}); + +function AddonUtilsInternal() { + this._log = Log.repository.getLogger("Sync.AddonUtils"); + this._log.Level = + Log.Level[Svc.PrefBranch.getStringPref("log.logger.addonutils", null)]; +} +AddonUtilsInternal.prototype = { + /** + * Obtain an AddonInstall object from an AddonSearchResult instance. + * + * The returned promise will be an AddonInstall on success or null (failure or + * addon not found) + * + * @param addon + * AddonSearchResult to obtain install from. + */ + getInstallFromSearchResult(addon) { + this._log.debug("Obtaining install for " + addon.id); + + // We should theoretically be able to obtain (and use) addon.install if + // it is available. However, the addon.sourceURI rewriting won't be + // reflected in the AddonInstall, so we can't use it. If we ever get rid + // of sourceURI rewriting, we can avoid having to reconstruct the + // AddonInstall. + return lazy.AddonManager.getInstallForURL(addon.sourceURI.spec, { + name: addon.name, + icons: addon.iconURL, + version: addon.version, + telemetryInfo: { source: "sync" }, + }); + }, + + /** + * Installs an add-on from an AddonSearchResult instance. + * + * The options argument defines extra options to control the install. + * Recognized keys in this map are: + * + * syncGUID - Sync GUID to use for the new add-on. + * enabled - Boolean indicating whether the add-on should be enabled upon + * install. + * + * The result object has the following keys: + * + * id ID of add-on that was installed. + * install AddonInstall that was installed. + * addon Addon that was installed. + * + * @param addon + * AddonSearchResult to install add-on from. + * @param options + * Object with additional metadata describing how to install add-on. + */ + async installAddonFromSearchResult(addon, options) { + this._log.info("Trying to install add-on from search result: " + addon.id); + + const install = await this.getInstallFromSearchResult(addon); + if (!install) { + throw new Error("AddonInstall not available: " + addon.id); + } + + try { + this._log.info("Installing " + addon.id); + let log = this._log; + + return new Promise((res, rej) => { + let listener = { + onInstallStarted: function onInstallStarted(install) { + if (!options) { + return; + } + + if (options.syncGUID) { + log.info( + "Setting syncGUID of " + install.name + ": " + options.syncGUID + ); + install.addon.syncGUID = options.syncGUID; + } + + // We only need to change userDisabled if it is disabled because + // enabled is the default. + if ("enabled" in options && !options.enabled) { + log.info( + "Marking add-on as disabled for install: " + install.name + ); + install.addon.disable(); + } + }, + onInstallEnded(install, addon) { + install.removeListener(listener); + + res({ id: addon.id, install, addon }); + }, + onInstallFailed(install) { + install.removeListener(listener); + + rej(new Error("Install failed: " + install.error)); + }, + onDownloadFailed(install) { + install.removeListener(listener); + + rej(new Error("Download failed: " + install.error)); + }, + }; + install.addListener(listener); + install.install(); + }); + } catch (ex) { + this._log.error("Error installing add-on", ex); + throw ex; + } + }, + + /** + * Uninstalls the addon instance. + * + * @param addon + * Addon instance to uninstall. + */ + async uninstallAddon(addon) { + return new Promise(res => { + let listener = { + onUninstalling(uninstalling, needsRestart) { + if (addon.id != uninstalling.id) { + return; + } + + // We assume restartless add-ons will send the onUninstalled event + // soon. + if (!needsRestart) { + return; + } + + // For non-restartless add-ons, we issue the callback on uninstalling + // because we will likely never see the uninstalled event. + lazy.AddonManager.removeAddonListener(listener); + res(addon); + }, + onUninstalled(uninstalled) { + if (addon.id != uninstalled.id) { + return; + } + + lazy.AddonManager.removeAddonListener(listener); + res(addon); + }, + }; + lazy.AddonManager.addAddonListener(listener); + addon.uninstall(); + }); + }, + + /** + * Installs multiple add-ons specified by metadata. + * + * The first argument is an array of objects. Each object must have the + * following keys: + * + * id - public ID of the add-on to install. + * syncGUID - syncGUID for new add-on. + * enabled - boolean indicating whether the add-on should be enabled. + * requireSecureURI - Boolean indicating whether to require a secure + * URI when installing from a remote location. This defaults to + * true. + * + * The callback will be called when activity on all add-ons is complete. The + * callback receives 2 arguments, error and result. + * + * If error is truthy, it contains a string describing the overall error. + * + * The 2nd argument to the callback is always an object with details on the + * overall execution state. It contains the following keys: + * + * installedIDs Array of add-on IDs that were installed. + * installs Array of AddonInstall instances that were installed. + * addons Array of Addon instances that were installed. + * errors Array of errors encountered. Only has elements if error is + * truthy. + * + * @param installs + * Array of objects describing add-ons to install. + */ + async installAddons(installs) { + let ids = []; + for (let addon of installs) { + ids.push(addon.id); + } + + let addons = await lazy.AddonRepository.getAddonsByIDs(ids); + this._log.info( + `Found ${addons.length} / ${ids.length}` + + " add-ons during repository search." + ); + + let ourResult = { + installedIDs: [], + installs: [], + addons: [], + skipped: [], + errors: [], + }; + + let toInstall = []; + + // Rewrite the "src" query string parameter of the source URI to note + // that the add-on was installed by Sync and not something else so + // server-side metrics aren't skewed (bug 708134). The server should + // ideally send proper URLs, but this solution was deemed too + // complicated at the time the functionality was implemented. + for (let addon of addons) { + // Find the specified options for this addon. + let options; + for (let install of installs) { + if (install.id == addon.id) { + options = install; + break; + } + } + if (!this.canInstallAddon(addon, options)) { + ourResult.skipped.push(addon.id); + continue; + } + + // We can go ahead and attempt to install it. + toInstall.push(addon); + + // We should always be able to QI the nsIURI to nsIURL. If not, we + // still try to install the add-on, but we don't rewrite the URL, + // potentially skewing metrics. + try { + addon.sourceURI.QueryInterface(Ci.nsIURL); + } catch (ex) { + this._log.warn( + "Unable to QI sourceURI to nsIURL: " + addon.sourceURI.spec + ); + continue; + } + + let params = addon.sourceURI.query + .split("&") + .map(function rewrite(param) { + if (param.indexOf("src=") == 0) { + return "src=sync"; + } + return param; + }); + + addon.sourceURI = addon.sourceURI + .mutate() + .setQuery(params.join("&")) + .finalize(); + } + + if (!toInstall.length) { + return ourResult; + } + + const installPromises = []; + // Start all the installs asynchronously. They will report back to us + // as they finish, eventually triggering the global callback. + for (let addon of toInstall) { + let options = {}; + for (let install of installs) { + if (install.id == addon.id) { + options = install; + break; + } + } + + installPromises.push( + (async () => { + try { + const result = await this.installAddonFromSearchResult( + addon, + options + ); + ourResult.installedIDs.push(result.id); + ourResult.installs.push(result.install); + ourResult.addons.push(result.addon); + } catch (error) { + ourResult.errors.push(error); + } + })() + ); + } + + await Promise.all(installPromises); + + if (ourResult.errors.length) { + throw new Error("1 or more add-ons failed to install"); + } + return ourResult; + }, + + /** + * Returns true if we are able to install the specified addon, false + * otherwise. It is expected that this will log the reason if it returns + * false. + * + * @param addon + * (Addon) Add-on instance to check. + * @param options + * (object) The options specified for this addon. See installAddons() + * for the valid elements. + */ + canInstallAddon(addon, options) { + // sourceURI presence isn't enforced by AddonRepository. So, we skip + // add-ons without a sourceURI. + if (!addon.sourceURI) { + this._log.info( + "Skipping install of add-on because missing sourceURI: " + addon.id + ); + return false; + } + // Verify that the source URI uses TLS. We don't allow installs from + // insecure sources for security reasons. The Addon Manager ensures + // that cert validation etc is performed. + // (We should also consider just dropping this entirely and calling + // XPIProvider.isInstallAllowed, but that has additional semantics we might + // need to think through...) + let requireSecureURI = true; + if (options && options.requireSecureURI !== undefined) { + requireSecureURI = options.requireSecureURI; + } + + if (requireSecureURI) { + let scheme = addon.sourceURI.scheme; + if (scheme != "https") { + this._log.info( + `Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted` + ); + return false; + } + } + + // Policy prevents either installing this addon or any addon + if ( + Services.policies && + (!Services.policies.mayInstallAddon(addon) || + !Services.policies.isAllowed("xpinstall")) + ) { + this._log.info( + `Skipping install of "${addon.id}" due to enterprise policy` + ); + return false; + } + + this._log.info(`Add-on "${addon.id}" is able to be installed`); + return true; + }, + + /** + * Update the user disabled flag for an add-on. + * + * If the new flag matches the existing or if the add-on + * isn't currently active, the function will return immediately. + * + * @param addon + * (Addon) Add-on instance to operate on. + * @param value + * (bool) New value for add-on's userDisabled property. + */ + updateUserDisabled(addon, value) { + if (addon.userDisabled == value) { + return; + } + + this._log.info("Updating userDisabled flag: " + addon.id + " -> " + value); + if (value) { + addon.disable(); + } else { + addon.enable(); + } + }, +}; + +export const AddonUtils = new AddonUtilsInternal(); diff --git a/services/sync/modules/bridged_engine.sys.mjs b/services/sync/modules/bridged_engine.sys.mjs new file mode 100644 index 0000000000..45e5f685cd --- /dev/null +++ b/services/sync/modules/bridged_engine.sys.mjs @@ -0,0 +1,499 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file has all the machinery for hooking up bridged engines implemented + * in Rust. It's the JavaScript side of the Golden Gate bridge that connects + * Desktop Sync to a Rust `BridgedEngine`, via the `mozIBridgedSyncEngine` + * XPCOM interface. + * + * Creating a bridged engine only takes a few lines of code, since most of the + * hard work is done on the Rust side. On the JS side, you'll need to subclass + * `BridgedEngine` (instead of `SyncEngine`), supply a `mozIBridgedSyncEngine` + * for your subclass to wrap, and optionally implement and override the tracker. + */ + +import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs"; +import { RawCryptoWrapper } from "resource://services-sync/record.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Log: "resource://gre/modules/Log.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +/** + * A stub store that converts between raw decrypted incoming records and + * envelopes. Since the interface we need is so minimal, this class doesn't + * inherit from the base `Store` implementation...it would take more code to + * override all those behaviors! + * + * This class isn't meant to be subclassed, because bridged engines shouldn't + * override their store classes in `_storeObj`. + */ +class BridgedStore { + constructor(name, engine) { + if (!engine) { + throw new Error("Store must be associated with an Engine instance."); + } + this.engine = engine; + this._log = lazy.Log.repository.getLogger(`Sync.Engine.${name}.Store`); + this._batchChunkSize = 500; + } + + async applyIncomingBatch(records, countTelemetry) { + for (let chunk of lazy.PlacesUtils.chunkArray( + records, + this._batchChunkSize + )) { + let incomingEnvelopesAsJSON = chunk.map(record => + JSON.stringify(record.toIncomingBso()) + ); + this._log.trace("incoming envelopes", incomingEnvelopesAsJSON); + await this.engine._bridge.storeIncoming(incomingEnvelopesAsJSON); + } + // Array of failed records. + return []; + } + + async wipe() { + await this.engine._bridge.wipe(); + } +} + +/** + * A wrapper class to convert between BSOs on the JS side, and envelopes on the + * Rust side. This class intentionally subclasses `RawCryptoWrapper`, because we + * don't want the stringification and parsing machinery in `CryptoWrapper`. + * + * This class isn't meant to be subclassed, because bridged engines shouldn't + * override their record classes in `_recordObj`. + */ +class BridgedRecord extends RawCryptoWrapper { + /** + * Creates an outgoing record from a BSO returned by a bridged engine. + * + * @param {String} collection The collection name. + * @param {Object} bso The outgoing bso (ie, a sync15::bso::OutgoingBso) returned from + * `mozIBridgedSyncEngine::apply`. + * @return {BridgedRecord} A Sync record ready to encrypt and upload. + */ + static fromOutgoingBso(collection, bso) { + // The BSO has already been JSON serialized coming out of Rust, so the + // envelope has been flattened. + if (typeof bso.id != "string") { + throw new TypeError("Outgoing BSO missing ID"); + } + if (typeof bso.payload != "string") { + throw new TypeError("Outgoing BSO missing payload"); + } + let record = new BridgedRecord(collection, bso.id); + record.cleartext = bso.payload; + return record; + } + + transformBeforeEncrypt(cleartext) { + if (typeof cleartext != "string") { + throw new TypeError("Outgoing bridged engine records must be strings"); + } + return cleartext; + } + + transformAfterDecrypt(cleartext) { + if (typeof cleartext != "string") { + throw new TypeError("Incoming bridged engine records must be strings"); + } + return cleartext; + } + + /* + * Converts this incoming record into an envelope to pass to a bridged engine. + * This object must be kept in sync with `sync15::IncomingBso`. + * + * @return {Object} The incoming envelope, to pass to + * `mozIBridgedSyncEngine::storeIncoming`. + */ + toIncomingBso() { + return { + id: this.data.id, + modified: this.data.modified, + payload: this.cleartext, + }; + } +} + +class BridgeError extends Error { + constructor(code, message) { + super(message); + this.name = "BridgeError"; + // TODO: We may want to use a different name for this, since errors with + // a `result` property are treated specially by telemetry, discarding the + // message...but, unlike other `nserror`s, the message is actually useful, + // and we still want to capture it. + this.result = code; + } +} + +class InterruptedError extends Error { + constructor(message) { + super(message); + this.name = "InterruptedError"; + } +} + +/** + * Adapts a `Log.sys.mjs` logger to a `mozIServicesLogSink`. This class is copied + * from `SyncedBookmarksMirror.jsm`. + */ +export class LogAdapter { + constructor(log) { + this.log = log; + } + + get maxLevel() { + let level = this.log.level; + if (level <= lazy.Log.Level.All) { + return Ci.mozIServicesLogSink.LEVEL_TRACE; + } + if (level <= lazy.Log.Level.Info) { + return Ci.mozIServicesLogSink.LEVEL_DEBUG; + } + if (level <= lazy.Log.Level.Warn) { + return Ci.mozIServicesLogSink.LEVEL_WARN; + } + if (level <= lazy.Log.Level.Error) { + return Ci.mozIServicesLogSink.LEVEL_ERROR; + } + return Ci.mozIServicesLogSink.LEVEL_OFF; + } + + trace(message) { + this.log.trace(message); + } + + debug(message) { + this.log.debug(message); + } + + warn(message) { + this.log.warn(message); + } + + error(message) { + this.log.error(message); + } +} + +// This converts the XPCOM-defined, callback-based mozIBridgedSyncEngine to +// a promise-based implementation. +export class BridgeWrapperXPCOM { + constructor(component) { + this.comp = component; + } + + // A few sync, non-callback based attributes. + get storageVersion() { + return this.comp.storageVersion; + } + + get allowSkippedRecord() { + return this.comp.allowSkippedRecord; + } + + get logger() { + return this.comp.logger; + } + + // And the async functions we promisify. + // Note this is `lastSync` via uniffi but `getLastSync` via xpcom + lastSync() { + return BridgeWrapperXPCOM.#promisify(this.comp.getLastSync); + } + + setLastSync(lastSyncMillis) { + return BridgeWrapperXPCOM.#promisify(this.comp.setLastSync, lastSyncMillis); + } + + getSyncId() { + return BridgeWrapperXPCOM.#promisify(this.comp.getSyncId); + } + + resetSyncId() { + return BridgeWrapperXPCOM.#promisify(this.comp.resetSyncId); + } + + ensureCurrentSyncId(newSyncId) { + return BridgeWrapperXPCOM.#promisify( + this.comp.ensureCurrentSyncId, + newSyncId + ); + } + + syncStarted() { + return BridgeWrapperXPCOM.#promisify(this.comp.syncStarted); + } + + storeIncoming(incomingEnvelopesAsJSON) { + return BridgeWrapperXPCOM.#promisify( + this.comp.storeIncoming, + incomingEnvelopesAsJSON + ); + } + + apply() { + return BridgeWrapperXPCOM.#promisify(this.comp.apply); + } + + setUploaded(newTimestampMillis, uploadedIds) { + return BridgeWrapperXPCOM.#promisify( + this.comp.setUploaded, + newTimestampMillis, + uploadedIds + ); + } + + syncFinished() { + return BridgeWrapperXPCOM.#promisify(this.comp.syncFinished); + } + + reset() { + return BridgeWrapperXPCOM.#promisify(this.comp.reset); + } + + wipe() { + return BridgeWrapperXPCOM.#promisify(this.comp.wipe); + } + + // Converts a XPCOM bridged function that takes a callback into one that returns a + // promise. + static #promisify(func, ...params) { + return new Promise((resolve, reject) => { + func(...params, { + // This object implicitly implements all three callback interfaces + // (`mozIBridgedSyncEngine{Apply, Result}Callback`), because they have + // the same methods. The only difference is the type of the argument + // passed to `handleSuccess`, which doesn't matter in JS. + handleSuccess: resolve, + handleError(code, message) { + reject(transformError(code, message)); + }, + }); + }); + } +} + +/** + * A base class used to plug a Rust engine into Sync, and have it work like any + * other engine. The constructor takes a bridge as its first argument, which is + * a "bridged sync engine", as defined by UniFFI in the application-services + * crate. + * For backwards compatibility, this can also be an instance of an XPCOM + * component class that implements `mozIBridgedSyncEngine`, wrapped in + * a `BridgeWrapperXPCOM` wrapper. + * (Note that at time of writing, the above is slightly aspirational; the + * actual definition of the UniFFI shared bridged engine is still in flux.) + * + * This class inherits from `SyncEngine`, which has a lot of machinery that we + * don't need, but that's fairly easy to override. It would be harder to + * reimplement the machinery that we _do_ need here. However, because of that, + * this class has lots of methods that do nothing, or return empty data. The + * docs above each method explain what it's overriding, and why. + * + * This class is designed to be subclassed, but the only part that your engine + * may want to override is `_trackerObj`. Even then, using the default (no-op) + * tracker is fine, because the shape of the `Tracker` interface may not make + * sense for all engines. + */ +export function BridgedEngine(name, service) { + SyncEngine.call(this, name, service); +} + +BridgedEngine.prototype = { + /** + * The Rust implemented bridge. Must be set by the engine which subclasses us. + */ + _bridge: null, + /** + * The tracker class for this engine. Subclasses may want to override this + * with their own tracker, though using the default `Tracker` is fine. + */ + _trackerObj: Tracker, + + /** Returns the record class for all bridged engines. */ + get _recordObj() { + return BridgedRecord; + }, + + set _recordObj(obj) { + throw new TypeError("Don't override the record class for bridged engines"); + }, + + /** Returns the store class for all bridged engines. */ + get _storeObj() { + return BridgedStore; + }, + + set _storeObj(obj) { + throw new TypeError("Don't override the store class for bridged engines"); + }, + + /** Returns the storage version for this engine. */ + get version() { + return this._bridge.storageVersion; + }, + + // Legacy engines allow sync to proceed if some records are too large to + // upload (eg, a payload that's bigger than the server's published limits). + // If this returns true, we will just skip the record without even attempting + // to upload. If this is false, we'll abort the entire batch. + // If the engine allows this, it will need to detect this scenario by noticing + // the ID is not in the 'success' records reported to `setUploaded`. + // (Note that this is not to be confused with the fact server's can currently + // reject records as part of a POST - but we hope to remove this ability from + // the server API. Note also that this is not bullet-proof - if the count of + // records is high, it's possible that we will have committed a previous + // batch before we hit the relevant limits, so things might have been written. + // We hope to fix this by ensuring batch limits are such that this is + // impossible) + get allowSkippedRecord() { + return this._bridge.allowSkippedRecord; + }, + + /** + * Returns the sync ID for this engine. This is exposed for tests, but + * Sync code always calls `resetSyncID()` and `ensureCurrentSyncID()`, + * not this. + * + * @returns {String?} The sync ID, or `null` if one isn't set. + */ + async getSyncID() { + // Note that all methods on an XPCOM class instance are automatically bound, + // so we don't need to write `this._bridge.getSyncId.bind(this._bridge)`. + let syncID = await this._bridge.getSyncId(); + return syncID; + }, + + async resetSyncID() { + await this._deleteServerCollection(); + let newSyncID = await this.resetLocalSyncID(); + return newSyncID; + }, + + async resetLocalSyncID() { + let newSyncID = await this._bridge.resetSyncId(); + return newSyncID; + }, + + async ensureCurrentSyncID(newSyncID) { + let assignedSyncID = await this._bridge.ensureCurrentSyncId(newSyncID); + return assignedSyncID; + }, + + async getLastSync() { + // The bridge defines lastSync as integer ms, but sync itself wants to work + // in a float seconds with 2 decimal places. + let lastSyncMS = await this._bridge.lastSync(); + return Math.round(lastSyncMS / 10) / 100; + }, + + async setLastSync(lastSyncSeconds) { + await this._bridge.setLastSync(Math.round(lastSyncSeconds * 1000)); + }, + + /** + * Returns the initial changeset for the sync. Bridged engines handle + * reconciliation internally, so we don't know what changed until after we've + * stored and applied all incoming records. So we return an empty changeset + * here, and replace it with the real one in `_processIncoming`. + */ + async pullChanges() { + return {}; + }, + + async trackRemainingChanges() { + await this._bridge.syncFinished(); + }, + + /** + * Marks a record for a hard-`DELETE` at the end of the sync. The base method + * also removes it from the tracker, but we don't use the tracker for that, + * so we override the method to just mark. + */ + _deleteId(id) { + this._noteDeletedId(id); + }, + + /** + * Always stage incoming records, bypassing the base engine's reconciliation + * machinery. + */ + async _reconcile() { + return true; + }, + + async _syncStartup() { + await super._syncStartup(); + await this._bridge.syncStarted(); + }, + + async _processIncoming(newitems) { + await super._processIncoming(newitems); + + let outgoingBsosAsJSON = await this._bridge.apply(); + let changeset = {}; + for (let bsoAsJSON of outgoingBsosAsJSON) { + this._log.trace("outgoing bso", bsoAsJSON); + let record = BridgedRecord.fromOutgoingBso( + this.name, + JSON.parse(bsoAsJSON) + ); + changeset[record.id] = { + synced: false, + record, + }; + } + this._modified.replace(changeset); + }, + + /** + * Notify the bridged engine that we've successfully uploaded a batch, so + * that it can update its local state. For example, if the engine uses a + * mirror and a temp table for outgoing records, it can write the uploaded + * records from the outgoing table back to the mirror. + */ + async _onRecordsWritten(succeeded, failed, serverModifiedTime) { + // JS uses seconds but Rust uses milliseconds so we'll need to convert + let serverModifiedMS = Math.round(serverModifiedTime * 1000); + await this._bridge.setUploaded(Math.floor(serverModifiedMS), succeeded); + }, + + async _createTombstone() { + throw new Error("Bridged engines don't support weak uploads"); + }, + + async _createRecord(id) { + let change = this._modified.changes[id]; + if (!change) { + throw new TypeError("Can't create record for unchanged item"); + } + return change.record; + }, + + async _resetClient() { + await super._resetClient(); + await this._bridge.reset(); + }, +}; +Object.setPrototypeOf(BridgedEngine.prototype, SyncEngine.prototype); + +function transformError(code, message) { + switch (code) { + case Cr.NS_ERROR_ABORT: + return new InterruptedError(message); + + default: + return new BridgeError(code, message); + } +} diff --git a/services/sync/modules/collection_validator.sys.mjs b/services/sync/modules/collection_validator.sys.mjs new file mode 100644 index 0000000000..a64ede10e9 --- /dev/null +++ b/services/sync/modules/collection_validator.sys.mjs @@ -0,0 +1,267 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", +}); + +export class CollectionProblemData { + constructor() { + this.missingIDs = 0; + this.clientDuplicates = []; + this.duplicates = []; + this.clientMissing = []; + this.serverMissing = []; + this.serverDeleted = []; + this.serverUnexpected = []; + this.differences = []; + } + + /** + * Produce a list summarizing problems found. Each entry contains {name, count}, + * where name is the field name for the problem, and count is the number of times + * the problem was encountered. + * + * Validation has failed if all counts are not 0. + */ + getSummary() { + return [ + { name: "clientMissing", count: this.clientMissing.length }, + { name: "serverMissing", count: this.serverMissing.length }, + { name: "serverDeleted", count: this.serverDeleted.length }, + { name: "serverUnexpected", count: this.serverUnexpected.length }, + { name: "differences", count: this.differences.length }, + { name: "missingIDs", count: this.missingIDs }, + { name: "clientDuplicates", count: this.clientDuplicates.length }, + { name: "duplicates", count: this.duplicates.length }, + ]; + } +} + +export class CollectionValidator { + // Construct a generic collection validator. This is intended to be called by + // subclasses. + // - name: Name of the engine + // - idProp: Property that identifies a record. That is, if a client and server + // record have the same value for the idProp property, they should be + // compared against eachother. + // - props: Array of properties that should be compared + constructor(name, idProp, props) { + this.name = name; + this.props = props; + this.idProp = idProp; + + // This property deals with the fact that form history records are never + // deleted from the server. The FormValidator subclass needs to ignore the + // client missing records, and it uses this property to achieve it - + // (Bug 1354016). + this.ignoresMissingClients = false; + } + + // Should a custom ProblemData type be needed, return it here. + emptyProblemData() { + return new CollectionProblemData(); + } + + async getServerItems(engine) { + let collection = engine.itemSource(); + let collectionKey = engine.service.collectionKeys.keyForCollection( + engine.name + ); + collection.full = true; + let result = await collection.getBatched(); + if (!result.response.success) { + throw result.response; + } + let cleartexts = []; + + await lazy.Async.yieldingForEach(result.records, async record => { + await record.decrypt(collectionKey); + cleartexts.push(record.cleartext); + }); + + return cleartexts; + } + + // Should return a promise that resolves to an array of client items. + getClientItems() { + return Promise.reject("Must implement"); + } + + /** + * Can we guarantee validation will fail with a reason that isn't actually a + * problem? For example, if we know there are pending changes left over from + * the last sync, this should resolve to false. By default resolves to true. + */ + async canValidate() { + return true; + } + + // Turn the client item into something that can be compared with the server item, + // and is also safe to mutate. + normalizeClientItem(item) { + return Cu.cloneInto(item, {}); + } + + // Turn the server item into something that can be easily compared with the client + // items. + async normalizeServerItem(item) { + return item; + } + + // Return whether or not a server item should be present on the client. Expected + // to be overridden. + clientUnderstands(item) { + return true; + } + + // Return whether or not a client item should be present on the server. Expected + // to be overridden + async syncedByClient(item) { + return true; + } + + // Compare the server item and the client item, and return a list of property + // names that are different. Can be overridden if needed. + getDifferences(client, server) { + let differences = []; + for (let prop of this.props) { + let clientProp = client[prop]; + let serverProp = server[prop]; + if ((clientProp || "") !== (serverProp || "")) { + differences.push(prop); + } + } + return differences; + } + + // Returns an object containing + // problemData: an instance of the class returned by emptyProblemData(), + // clientRecords: Normalized client records + // records: Normalized server records, + // deletedRecords: Array of ids that were marked as deleted by the server. + async compareClientWithServer(clientItems, serverItems) { + const yieldState = lazy.Async.yieldState(); + + const clientRecords = []; + + await lazy.Async.yieldingForEach( + clientItems, + item => { + clientRecords.push(this.normalizeClientItem(item)); + }, + yieldState + ); + + const serverRecords = []; + await lazy.Async.yieldingForEach( + serverItems, + async item => { + serverRecords.push(await this.normalizeServerItem(item)); + }, + yieldState + ); + + let problems = this.emptyProblemData(); + let seenServer = new Map(); + let serverDeleted = new Set(); + let allRecords = new Map(); + + for (let record of serverRecords) { + let id = record[this.idProp]; + if (!id) { + ++problems.missingIDs; + continue; + } + if (record.deleted) { + serverDeleted.add(record); + } else { + let serverHasPossibleDupe = seenServer.has(id); + if (serverHasPossibleDupe) { + problems.duplicates.push(id); + } else { + seenServer.set(id, record); + allRecords.set(id, { server: record, client: null }); + } + record.understood = this.clientUnderstands(record); + } + } + + let seenClient = new Map(); + for (let record of clientRecords) { + let id = record[this.idProp]; + record.shouldSync = await this.syncedByClient(record); + let clientHasPossibleDupe = seenClient.has(id); + if (clientHasPossibleDupe && record.shouldSync) { + // Only report duplicate client IDs for syncable records. + problems.clientDuplicates.push(id); + continue; + } + seenClient.set(id, record); + let combined = allRecords.get(id); + if (combined) { + combined.client = record; + } else { + allRecords.set(id, { client: record, server: null }); + } + } + + for (let [id, { server, client }] of allRecords) { + if (!client && !server) { + throw new Error("Impossible: no client or server record for " + id); + } else if (server && !client) { + if (!this.ignoresMissingClients && server.understood) { + problems.clientMissing.push(id); + } + } else if (client && !server) { + if (client.shouldSync) { + problems.serverMissing.push(id); + } + } else { + if (!client.shouldSync) { + if (!problems.serverUnexpected.includes(id)) { + problems.serverUnexpected.push(id); + } + continue; + } + let differences = this.getDifferences(client, server); + if (differences && differences.length) { + problems.differences.push({ id, differences }); + } + } + } + return { + problemData: problems, + clientRecords, + records: serverRecords, + deletedRecords: [...serverDeleted], + }; + } + + async validate(engine) { + let start = Cu.now(); + let clientItems = await this.getClientItems(); + let serverItems = await this.getServerItems(engine); + let serverRecordCount = serverItems.length; + let result = await this.compareClientWithServer(clientItems, serverItems); + let end = Cu.now(); + let duration = end - start; + engine._log.debug(`Validated ${this.name} in ${duration}ms`); + engine._log.debug(`Problem summary`); + for (let { name, count } of result.problemData.getSummary()) { + engine._log.debug(` ${name}: ${count}`); + } + return { + duration, + version: this.version, + problems: result.problemData, + recordCount: serverRecordCount, + }; + } +} + +// Default to 0, some engines may override. +CollectionValidator.prototype.version = 0; diff --git a/services/sync/modules/constants.sys.mjs b/services/sync/modules/constants.sys.mjs new file mode 100644 index 0000000000..35c0ac2f0b --- /dev/null +++ b/services/sync/modules/constants.sys.mjs @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Don't manually modify this line, as it is automatically replaced on merge day +// by the gecko_migration.py script. +export const WEAVE_VERSION = "1.126.0"; + +// Sync Server API version that the client supports. +export const SYNC_API_VERSION = "1.5"; + +// Version of the data format this client supports. The data format describes +// how records are packaged; this is separate from the Server API version and +// the per-engine cleartext formats. +export const STORAGE_VERSION = 5; +export const PREFS_BRANCH = "services.sync."; + +// Put in [] because those aren't allowed in a collection name. +export const DEFAULT_KEYBUNDLE_NAME = "[default]"; + +// Key dimensions. +export const SYNC_KEY_ENCODED_LENGTH = 26; +export const SYNC_KEY_DECODED_LENGTH = 16; + +export const NO_SYNC_NODE_INTERVAL = 10 * 60 * 1000; // 10 minutes + +export const MAX_ERROR_COUNT_BEFORE_BACKOFF = 3; + +// Backoff intervals +export const MINIMUM_BACKOFF_INTERVAL = 15 * 60 * 1000; // 15 minutes +export const MAXIMUM_BACKOFF_INTERVAL = 8 * 60 * 60 * 1000; // 8 hours + +// HMAC event handling timeout. +// 10 minutes = a compromise between the multi-desktop sync interval +// and the mobile sync interval. +export const HMAC_EVENT_INTERVAL = 600000; + +// How long to wait between sync attempts if the Master Password is locked. +export const MASTER_PASSWORD_LOCKED_RETRY_INTERVAL = 15 * 60 * 1000; // 15 minutes + +// 50 is hardcoded here because of URL length restrictions. +// (GUIDs can be up to 64 chars long.) +// Individual engines can set different values for their limit if their +// identifiers are shorter. +export const DEFAULT_GUID_FETCH_BATCH_SIZE = 50; + +// Default batch size for download batching +// (how many records are fetched at a time from the server when batching is used). +export const DEFAULT_DOWNLOAD_BATCH_SIZE = 1000; + +// score thresholds for early syncs +export const SINGLE_USER_THRESHOLD = 1000; +export const MULTI_DEVICE_THRESHOLD = 300; + +// Other score increment constants +export const SCORE_INCREMENT_SMALL = 1; +export const SCORE_INCREMENT_MEDIUM = 10; + +// Instant sync score increment +export const SCORE_INCREMENT_XLARGE = 300 + 1; //MULTI_DEVICE_THRESHOLD + 1 + +// Delay before incrementing global score +export const SCORE_UPDATE_DELAY = 100; + +// Delay for the back observer debouncer. This is chosen to be longer than any +// observed spurious idle/back events and short enough to pre-empt user activity. +export const IDLE_OBSERVER_BACK_DELAY = 100; + +// Duplicate URI_LENGTH_MAX from Places (from nsNavHistory.h), used to discard +// tabs with huge uris during tab sync. +export const URI_LENGTH_MAX = 65536; + +export const MAX_HISTORY_UPLOAD = 5000; +export const MAX_HISTORY_DOWNLOAD = 5000; + +// Top-level statuses +export const STATUS_OK = "success.status_ok"; +export const SYNC_FAILED = "error.sync.failed"; +export const LOGIN_FAILED = "error.login.failed"; +export const SYNC_FAILED_PARTIAL = "error.sync.failed_partial"; +export const CLIENT_NOT_CONFIGURED = "service.client_not_configured"; +export const STATUS_DISABLED = "service.disabled"; +export const MASTER_PASSWORD_LOCKED = "service.master_password_locked"; + +// success states +export const LOGIN_SUCCEEDED = "success.login"; +export const SYNC_SUCCEEDED = "success.sync"; +export const ENGINE_SUCCEEDED = "success.engine"; + +// login failure status codes +export const LOGIN_FAILED_NO_USERNAME = "error.login.reason.no_username"; +export const LOGIN_FAILED_NO_PASSPHRASE = "error.login.reason.no_recoverykey"; +export const LOGIN_FAILED_NETWORK_ERROR = "error.login.reason.network"; +export const LOGIN_FAILED_SERVER_ERROR = "error.login.reason.server"; +export const LOGIN_FAILED_INVALID_PASSPHRASE = "error.login.reason.recoverykey"; +export const LOGIN_FAILED_LOGIN_REJECTED = "error.login.reason.account"; + +// sync failure status codes +export const METARECORD_DOWNLOAD_FAIL = + "error.sync.reason.metarecord_download_fail"; +export const VERSION_OUT_OF_DATE = "error.sync.reason.version_out_of_date"; +export const CREDENTIALS_CHANGED = "error.sync.reason.credentials_changed"; +export const ABORT_SYNC_COMMAND = "aborting sync, process commands said so"; +export const NO_SYNC_NODE_FOUND = "error.sync.reason.no_node_found"; +export const OVER_QUOTA = "error.sync.reason.over_quota"; +export const SERVER_MAINTENANCE = "error.sync.reason.serverMaintenance"; + +export const RESPONSE_OVER_QUOTA = "14"; + +// engine failure status codes +export const ENGINE_UPLOAD_FAIL = "error.engine.reason.record_upload_fail"; +export const ENGINE_DOWNLOAD_FAIL = "error.engine.reason.record_download_fail"; +export const ENGINE_UNKNOWN_FAIL = "error.engine.reason.unknown_fail"; +export const ENGINE_APPLY_FAIL = "error.engine.reason.apply_fail"; +// an upload failure where the batch was interrupted with a 412 +export const ENGINE_BATCH_INTERRUPTED = "error.engine.reason.batch_interrupted"; + +// Ways that a sync can be disabled (messages only to be printed in debug log) +export const kSyncMasterPasswordLocked = + "User elected to leave Primary Password locked"; +export const kSyncWeaveDisabled = "Weave is disabled"; +export const kSyncNetworkOffline = "Network is offline"; +export const kSyncBackoffNotMet = + "Trying to sync before the server said it's okay"; +export const kFirstSyncChoiceNotMade = + "User has not selected an action for first sync"; +export const kSyncNotConfigured = "Sync is not configured"; +export const kFirefoxShuttingDown = "Firefox is about to shut down"; + +export const DEVICE_TYPE_DESKTOP = "desktop"; +export const DEVICE_TYPE_MOBILE = "mobile"; + +export const SQLITE_MAX_VARIABLE_NUMBER = 999; diff --git a/services/sync/modules/doctor.sys.mjs b/services/sync/modules/doctor.sys.mjs new file mode 100644 index 0000000000..ebcf38e2a2 --- /dev/null +++ b/services/sync/modules/doctor.sys.mjs @@ -0,0 +1,201 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// A doctor for our collections. She can be asked to make a consultation, and +// may just diagnose an issue without attempting to cure it, may diagnose and +// attempt to cure, or may decide she is overworked and underpaid. +// Or something - naming is hard :) + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { Observers } from "resource://services-common/observers.sys.mjs"; +import { Service } from "resource://services-sync/service.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { Svc } from "resource://services-sync/util.sys.mjs"; + +const log = Log.repository.getLogger("Sync.Doctor"); + +export var Doctor = { + async consult(recentlySyncedEngines) { + if (!Services.telemetry.canRecordBase) { + log.info("Skipping consultation: telemetry reporting is disabled"); + return; + } + + let engineInfos = this._getEnginesToValidate(recentlySyncedEngines); + + await this._runValidators(engineInfos); + }, + + _getEnginesToValidate(recentlySyncedEngines) { + let result = {}; + for (let e of recentlySyncedEngines) { + let prefPrefix = `engine.${e.name}.`; + if ( + !Svc.PrefBranch.getBoolPref(prefPrefix + "validation.enabled", false) + ) { + log.info(`Skipping check of ${e.name} - disabled via preferences`); + continue; + } + // Check the last validation time for the engine. + let lastValidation = Svc.PrefBranch.getIntPref( + prefPrefix + "validation.lastTime", + 0 + ); + let validationInterval = Svc.PrefBranch.getIntPref( + prefPrefix + "validation.interval" + ); + let nowSeconds = this._now(); + + if (nowSeconds - lastValidation < validationInterval) { + log.info( + `Skipping validation of ${e.name}: too recent since last validation attempt` + ); + continue; + } + // Update the time now, even if we decline to actually perform a + // validation. We don't want to check the rest of these more frequently + // than once a day. + Svc.PrefBranch.setIntPref( + prefPrefix + "validation.lastTime", + Math.floor(nowSeconds) + ); + + // Validation only occurs a certain percentage of the time. + let validationProbability = + Svc.PrefBranch.getIntPref( + prefPrefix + "validation.percentageChance", + 0 + ) / 100.0; + if (validationProbability < Math.random()) { + log.info( + `Skipping validation of ${e.name}: Probability threshold not met` + ); + continue; + } + + let maxRecords = Svc.PrefBranch.getIntPref( + prefPrefix + "validation.maxRecords" + ); + if (!maxRecords) { + log.info(`Skipping validation of ${e.name}: No maxRecords specified`); + continue; + } + // OK, so this is a candidate - the final decision will be based on the + // number of records actually found. + result[e.name] = { engine: e, maxRecords }; + } + return result; + }, + + async _runValidators(engineInfos) { + if (!Object.keys(engineInfos).length) { + log.info("Skipping validation: no engines qualify"); + return; + } + + if (Object.values(engineInfos).filter(i => i.maxRecords != -1).length) { + // at least some of the engines have maxRecord restrictions which require + // us to ask the server for the counts. + let countInfo = await this._fetchCollectionCounts(); + for (let [engineName, recordCount] of Object.entries(countInfo)) { + if (engineName in engineInfos) { + engineInfos[engineName].recordCount = recordCount; + } + } + } + + for (let [ + engineName, + { engine, maxRecords, recordCount }, + ] of Object.entries(engineInfos)) { + // maxRecords of -1 means "any number", so we can skip asking the server. + // Used for tests. + if (maxRecords >= 0 && recordCount > maxRecords) { + log.debug( + `Skipping validation for ${engineName} because ` + + `the number of records (${recordCount}) is greater ` + + `than the maximum allowed (${maxRecords}).` + ); + continue; + } + let validator = engine.getValidator(); + if (!validator) { + // This is probably only possible in profile downgrade cases. + log.warn( + `engine.getValidator returned null for ${engineName} but the pref that controls validation is enabled.` + ); + continue; + } + + if (!(await validator.canValidate())) { + log.debug( + `Skipping validation for ${engineName} because validator.canValidate() is false` + ); + continue; + } + + // Let's do it! + Services.console.logStringMessage( + `Sync is about to run a consistency check of ${engine.name}. This may be slow, and ` + + `can be controlled using the pref "services.sync.${engine.name}.validation.enabled".\n` + + `If you encounter any problems because of this, please file a bug.` + ); + + try { + log.info(`Running validator for ${engine.name}`); + let result = await validator.validate(engine); + let { problems, version, duration, recordCount } = result; + Observers.notify( + "weave:engine:validate:finish", + { + version, + checked: recordCount, + took: duration, + problems: problems ? problems.getSummary(true) : null, + }, + engine.name + ); + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + log.error(`Failed to run validation on ${engine.name}!`, ex); + Observers.notify("weave:engine:validate:error", ex, engine.name); + // Keep validating -- there's no reason to think that a failure for one + // validator would mean the others will fail. + } + } + }, + + // mainly for mocking. + async _fetchCollectionCounts() { + let collectionCountsURL = Service.userBaseURL + "info/collection_counts"; + try { + let infoResp = await Service._fetchInfo(collectionCountsURL); + if (!infoResp.success) { + log.error( + "Can't fetch collection counts: request to info/collection_counts responded with " + + infoResp.status + ); + return {}; + } + return infoResp.obj; // might throw because obj is a getter which parses json. + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + // Not running validation is totally fine, so we just write an error log and return. + log.error("Caught error when fetching counts", ex); + return {}; + } + }, + + // functions used so tests can mock them + _now() { + // We use the server time, which is SECONDS + return Resource.serverTime; + }, +}; diff --git a/services/sync/modules/engines.sys.mjs b/services/sync/modules/engines.sys.mjs new file mode 100644 index 0000000000..0d490ac4b3 --- /dev/null +++ b/services/sync/modules/engines.sys.mjs @@ -0,0 +1,2274 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { JSONFile } from "resource://gre/modules/JSONFile.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { Observers } from "resource://services-common/observers.sys.mjs"; + +import { + DEFAULT_DOWNLOAD_BATCH_SIZE, + DEFAULT_GUID_FETCH_BATCH_SIZE, + ENGINE_BATCH_INTERRUPTED, + ENGINE_DOWNLOAD_FAIL, + ENGINE_UPLOAD_FAIL, + VERSION_OUT_OF_DATE, + PREFS_BRANCH, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Collection, + CryptoWrapper, +} from "resource://services-sync/record.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { + SerializableSet, + Svc, + Utils, +} from "resource://services-sync/util.sys.mjs"; +import { SyncedRecordsTelemetry } from "resource://services-sync/telemetry.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +function ensureDirectory(path) { + return IOUtils.makeDirectory(PathUtils.parent(path), { + createAncestors: true, + }); +} + +/** + * Trackers are associated with a single engine and deal with + * listening for changes to their particular data type. + * + * The base `Tracker` only supports listening for changes, and bumping the score + * to indicate how urgently the engine wants to sync. It does not persist any + * data. Engines that track changes directly in the storage layer (like + * bookmarks, bridged engines, addresses, and credit cards) or only upload a + * single record (tabs and preferences) should subclass `Tracker`. + */ +export function Tracker(name, engine) { + if (!engine) { + throw new Error("Tracker must be associated with an Engine instance."); + } + + name = name || "Unnamed"; + this.name = name.toLowerCase(); + this.engine = engine; + + this._log = Log.repository.getLogger(`Sync.Engine.${name}.Tracker`); + + this._score = 0; + + this.asyncObserver = Async.asyncObserver(this, this._log); +} + +Tracker.prototype = { + // New-style trackers use change sources to filter out changes made by Sync in + // observer notifications, so we don't want to let the engine ignore all + // changes during a sync. + get ignoreAll() { + return false; + }, + + // Define an empty setter so that the engine doesn't throw a `TypeError` + // setting a read-only property. + set ignoreAll(value) {}, + + /* + * Score can be called as often as desired to decide which engines to sync + * + * Valid values for score: + * -1: Do not sync unless the user specifically requests it (almost disabled) + * 0: Nothing has changed + * 100: Please sync me ASAP! + * + * Setting it to other values should (but doesn't currently) throw an exception + */ + get score() { + return this._score; + }, + + set score(value) { + this._score = value; + Observers.notify("weave:engine:score:updated", this.name); + }, + + // Should be called by service everytime a sync has been done for an engine + resetScore() { + this._score = 0; + }, + + // Unsupported, and throws a more descriptive error to ensure callers aren't + // accidentally using persistence. + async getChangedIDs() { + throw new TypeError("This tracker doesn't store changed IDs"); + }, + + // Also unsupported. + async addChangedID(id, when) { + throw new TypeError("Can't add changed ID to this tracker"); + }, + + // Ditto. + async removeChangedID(...ids) { + throw new TypeError("Can't remove changed IDs from this tracker"); + }, + + // This method is called at various times, so we override with a no-op + // instead of throwing. + clearChangedIDs() {}, + + _now() { + return Date.now() / 1000; + }, + + _isTracking: false, + + start() { + if (!this.engineIsEnabled()) { + return; + } + this._log.trace("start()."); + if (!this._isTracking) { + this.onStart(); + this._isTracking = true; + } + }, + + async stop() { + this._log.trace("stop()."); + if (this._isTracking) { + await this.asyncObserver.promiseObserversComplete(); + this.onStop(); + this._isTracking = false; + } + }, + + // Override these in your subclasses. + onStart() {}, + onStop() {}, + async observe(subject, topic, data) {}, + + engineIsEnabled() { + if (!this.engine) { + // Can't tell -- we must be running in a test! + return true; + } + return this.engine.enabled; + }, + + /** + * Starts or stops listening for changes depending on the associated engine's + * enabled state. + * + * @param {Boolean} engineEnabled Whether the engine was enabled. + */ + async onEngineEnabledChanged(engineEnabled) { + if (engineEnabled == this._isTracking) { + return; + } + + if (engineEnabled) { + this.start(); + } else { + await this.stop(); + this.clearChangedIDs(); + } + }, + + async finalize() { + await this.stop(); + }, +}; + +/* + * A tracker that persists a list of IDs for all changed items that need to be + * synced. This is 🚨 _extremely deprecated_ 🚨 and only kept around for current + * engines. ⚠️ Please **don't use it** for new engines! ⚠️ + * + * Why is this kind of external change tracking deprecated? Because it causes + * consistency issues due to missed notifications, interrupted syncs, and the + * tracker's view of what changed diverging from the data store's. + */ +export function LegacyTracker(name, engine) { + Tracker.call(this, name, engine); + + this._ignored = []; + this.file = this.name; + this._storage = new JSONFile({ + path: Utils.jsonFilePath("changes", this.file), + dataPostProcessor: json => this._dataPostProcessor(json), + beforeSave: () => this._beforeSave(), + }); + this._ignoreAll = false; +} + +LegacyTracker.prototype = { + get ignoreAll() { + return this._ignoreAll; + }, + + set ignoreAll(value) { + this._ignoreAll = value; + }, + + // Default to an empty object if the file doesn't exist. + _dataPostProcessor(json) { + return (typeof json == "object" && json) || {}; + }, + + // Ensure the Weave storage directory exists before writing the file. + _beforeSave() { + return ensureDirectory(this._storage.path); + }, + + async getChangedIDs() { + await this._storage.load(); + return this._storage.data; + }, + + _saveChangedIDs() { + this._storage.saveSoon(); + }, + + // ignore/unignore specific IDs. Useful for ignoring items that are + // being processed, or that shouldn't be synced. + // But note: not persisted to disk + + ignoreID(id) { + this.unignoreID(id); + this._ignored.push(id); + }, + + unignoreID(id) { + let index = this._ignored.indexOf(id); + if (index != -1) { + this._ignored.splice(index, 1); + } + }, + + async _saveChangedID(id, when) { + this._log.trace(`Adding changed ID: ${id}, ${JSON.stringify(when)}`); + const changedIDs = await this.getChangedIDs(); + changedIDs[id] = when; + this._saveChangedIDs(); + }, + + async addChangedID(id, when) { + if (!id) { + this._log.warn("Attempted to add undefined ID to tracker"); + return false; + } + + if (this.ignoreAll || this._ignored.includes(id)) { + return false; + } + + // Default to the current time in seconds if no time is provided. + if (when == null) { + when = this._now(); + } + + const changedIDs = await this.getChangedIDs(); + // Add/update the entry if we have a newer time. + if ((changedIDs[id] || -Infinity) < when) { + await this._saveChangedID(id, when); + } + + return true; + }, + + async removeChangedID(...ids) { + if (!ids.length || this.ignoreAll) { + return false; + } + for (let id of ids) { + if (!id) { + this._log.warn("Attempted to remove undefined ID from tracker"); + continue; + } + if (this._ignored.includes(id)) { + this._log.debug(`Not removing ignored ID ${id} from tracker`); + continue; + } + const changedIDs = await this.getChangedIDs(); + if (changedIDs[id] != null) { + this._log.trace("Removing changed ID " + id); + delete changedIDs[id]; + } + } + this._saveChangedIDs(); + return true; + }, + + clearChangedIDs() { + this._log.trace("Clearing changed ID list"); + this._storage.data = {}; + this._saveChangedIDs(); + }, + + async finalize() { + // Persist all pending tracked changes to disk, and wait for the final write + // to finish. + await super.finalize(); + this._saveChangedIDs(); + await this._storage.finalize(); + }, +}; +Object.setPrototypeOf(LegacyTracker.prototype, Tracker.prototype); + +/** + * The Store serves as the interface between Sync and stored data. + * + * The name "store" is slightly a misnomer because it doesn't actually "store" + * anything. Instead, it serves as a gateway to something that actually does + * the "storing." + * + * The store is responsible for record management inside an engine. It tells + * Sync what items are available for Sync, converts items to and from Sync's + * record format, and applies records from Sync into changes on the underlying + * store. + * + * Store implementations require a number of functions to be implemented. These + * are all documented below. + * + * For stores that deal with many records or which have expensive store access + * routines, it is highly recommended to implement a custom applyIncomingBatch + * and/or applyIncoming function on top of the basic APIs. + */ + +export function Store(name, engine) { + if (!engine) { + throw new Error("Store must be associated with an Engine instance."); + } + + name = name || "Unnamed"; + this.name = name.toLowerCase(); + this.engine = engine; + + this._log = Log.repository.getLogger(`Sync.Engine.${name}.Store`); + + ChromeUtils.defineLazyGetter(this, "_timer", function () { + return Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + }); +} + +Store.prototype = { + /** + * Apply multiple incoming records against the store. + * + * This is called with a set of incoming records to process. The function + * should look at each record, reconcile with the current local state, and + * make the local changes required to bring its state in alignment with the + * record. + * + * The default implementation simply iterates over all records and calls + * applyIncoming(). Store implementations may overwrite this function + * if desired. + * + * @param records Array of records to apply + * @param a SyncedRecordsTelemetry obj that will keep track of failed reasons + * @return Array of record IDs which did not apply cleanly + */ + async applyIncomingBatch(records, countTelemetry) { + let failed = []; + + await Async.yieldingForEach(records, async record => { + try { + await this.applyIncoming(record); + } catch (ex) { + if (ex.code == SyncEngine.prototype.eEngineAbortApplyIncoming) { + // This kind of exception should have a 'cause' attribute, which is an + // originating exception. + // ex.cause will carry its stack with it when rethrown. + throw ex.cause; + } + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.warn("Failed to apply incoming record " + record.id, ex); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + }); + + return failed; + }, + + /** + * Apply a single record against the store. + * + * This takes a single record and makes the local changes required so the + * local state matches what's in the record. + * + * The default implementation calls one of remove(), create(), or update() + * depending on the state obtained from the store itself. Store + * implementations may overwrite this function if desired. + * + * @param record + * Record to apply + */ + async applyIncoming(record) { + if (record.deleted) { + await this.remove(record); + } else if (!(await this.itemExists(record.id))) { + await this.create(record); + } else { + await this.update(record); + } + }, + + // override these in derived objects + + /** + * Create an item in the store from a record. + * + * This is called by the default implementation of applyIncoming(). If using + * applyIncomingBatch(), this won't be called unless your store calls it. + * + * @param record + * The store record to create an item from + */ + async create(record) { + throw new Error("override create in a subclass"); + }, + + /** + * Remove an item in the store from a record. + * + * This is called by the default implementation of applyIncoming(). If using + * applyIncomingBatch(), this won't be called unless your store calls it. + * + * @param record + * The store record to delete an item from + */ + async remove(record) { + throw new Error("override remove in a subclass"); + }, + + /** + * Update an item from a record. + * + * This is called by the default implementation of applyIncoming(). If using + * applyIncomingBatch(), this won't be called unless your store calls it. + * + * @param record + * The record to use to update an item from + */ + async update(record) { + throw new Error("override update in a subclass"); + }, + + /** + * Determine whether a record with the specified ID exists. + * + * Takes a string record ID and returns a booleans saying whether the record + * exists. + * + * @param id + * string record ID + * @return boolean indicating whether record exists locally + */ + async itemExists(id) { + throw new Error("override itemExists in a subclass"); + }, + + /** + * Create a record from the specified ID. + * + * If the ID is known, the record should be populated with metadata from + * the store. If the ID is not known, the record should be created with the + * delete field set to true. + * + * @param id + * string record ID + * @param collection + * Collection to add record to. This is typically passed into the + * constructor for the newly-created record. + * @return record type for this engine + */ + async createRecord(id, collection) { + throw new Error("override createRecord in a subclass"); + }, + + /** + * Change the ID of a record. + * + * @param oldID + * string old/current record ID + * @param newID + * string new record ID + */ + async changeItemID(oldID, newID) { + throw new Error("override changeItemID in a subclass"); + }, + + /** + * Obtain the set of all known record IDs. + * + * @return Object with ID strings as keys and values of true. The values + * are ignored. + */ + async getAllIDs() { + throw new Error("override getAllIDs in a subclass"); + }, + + /** + * Wipe all data in the store. + * + * This function is called during remote wipes or when replacing local data + * with remote data. + * + * This function should delete all local data that the store is managing. It + * can be thought of as clearing out all state and restoring the "new + * browser" state. + */ + async wipe() { + throw new Error("override wipe in a subclass"); + }, +}; + +export function EngineManager(service) { + this.service = service; + + this._engines = {}; + + this._altEngineInfo = {}; + + // This will be populated by Service on startup. + this._declined = new Set(); + this._log = Log.repository.getLogger("Sync.EngineManager"); + this._log.manageLevelFromPref("services.sync.log.logger.service.engines"); + // define the default level for all engine logs here (although each engine + // allows its level to be controlled via a specific, non-default pref) + Log.repository + .getLogger(`Sync.Engine`) + .manageLevelFromPref("services.sync.log.logger.engine"); +} + +EngineManager.prototype = { + get(name) { + // Return an array of engines if we have an array of names + if (Array.isArray(name)) { + let engines = []; + name.forEach(function (name) { + let engine = this.get(name); + if (engine) { + engines.push(engine); + } + }, this); + return engines; + } + + return this._engines[name]; // Silently returns undefined for unknown names. + }, + + getAll() { + let engines = []; + for (let [, engine] of Object.entries(this._engines)) { + engines.push(engine); + } + return engines; + }, + + /** + * If a user has changed a pref that controls which variant of a sync engine + * for a given collection we use, unregister the old engine and register the + * new one. + * + * This is called by EngineSynchronizer before every sync. + */ + async switchAlternatives() { + for (let [name, info] of Object.entries(this._altEngineInfo)) { + let prefValue = info.prefValue; + if (prefValue === info.lastValue) { + this._log.trace( + `No change for engine ${name} (${info.pref} is still ${prefValue})` + ); + continue; + } + // Unregister the old engine, register the new one. + this._log.info( + `Switching ${name} engine ("${info.pref}" went from ${info.lastValue} => ${prefValue})` + ); + try { + await this._removeAndFinalize(name); + } catch (e) { + this._log.warn(`Failed to remove previous ${name} engine...`, e); + } + let engineType = prefValue ? info.whenTrue : info.whenFalse; + try { + // If register throws, we'll try again next sync, but until then there + // won't be an engine registered for this collection. + await this.register(engineType); + info.lastValue = prefValue; + // Note: engineType.name is using Function.prototype.name. + this._log.info(`Switched the ${name} engine to use ${engineType.name}`); + } catch (e) { + this._log.warn( + `Switching the ${name} engine to use ${engineType.name} failed (couldn't register)`, + e + ); + } + } + }, + + async registerAlternatives(name, pref, whenTrue, whenFalse) { + let info = { name, pref, whenTrue, whenFalse }; + + XPCOMUtils.defineLazyPreferenceGetter(info, "prefValue", pref, false); + + let chosen = info.prefValue ? info.whenTrue : info.whenFalse; + info.lastValue = info.prefValue; + this._altEngineInfo[name] = info; + + await this.register(chosen); + }, + + /** + * N.B., does not pay attention to the declined list. + */ + getEnabled() { + return this.getAll() + .filter(engine => engine.enabled) + .sort((a, b) => a.syncPriority - b.syncPriority); + }, + + get enabledEngineNames() { + return this.getEnabled().map(e => e.name); + }, + + persistDeclined() { + Svc.PrefBranch.setStringPref( + "declinedEngines", + [...this._declined].join(",") + ); + }, + + /** + * Returns an array. + */ + getDeclined() { + return [...this._declined]; + }, + + setDeclined(engines) { + this._declined = new Set(engines); + this.persistDeclined(); + }, + + isDeclined(engineName) { + return this._declined.has(engineName); + }, + + /** + * Accepts a Set or an array. + */ + decline(engines) { + for (let e of engines) { + this._declined.add(e); + } + this.persistDeclined(); + }, + + undecline(engines) { + for (let e of engines) { + this._declined.delete(e); + } + this.persistDeclined(); + }, + + /** + * Register an Engine to the service. Alternatively, give an array of engine + * objects to register. + * + * @param engineObject + * Engine object used to get an instance of the engine + * @return The engine object if anything failed + */ + async register(engineObject) { + if (Array.isArray(engineObject)) { + for (const e of engineObject) { + await this.register(e); + } + return; + } + + try { + let engine = new engineObject(this.service); + let name = engine.name; + if (name in this._engines) { + this._log.error("Engine '" + name + "' is already registered!"); + } else { + if (engine.initialize) { + await engine.initialize(); + } + this._engines[name] = engine; + } + } catch (ex) { + let name = engineObject || ""; + name = name.prototype || ""; + name = name.name || ""; + + this._log.error(`Could not initialize engine ${name}`, ex); + } + }, + + async unregister(val) { + let name = val; + if (val instanceof SyncEngine) { + name = val.name; + } + await this._removeAndFinalize(name); + delete this._altEngineInfo[name]; + }, + + // Common code for disabling an engine by name, that doesn't complain if the + // engine doesn't exist. Doesn't touch the engine's alternative info (if any + // exists). + async _removeAndFinalize(name) { + if (name in this._engines) { + let engine = this._engines[name]; + delete this._engines[name]; + await engine.finalize(); + } + }, + + async clear() { + for (let name in this._engines) { + let engine = this._engines[name]; + delete this._engines[name]; + await engine.finalize(); + } + this._altEngineInfo = {}; + }, +}; + +export function SyncEngine(name, service) { + if (!service) { + throw new Error("SyncEngine must be associated with a Service instance."); + } + + this.Name = name || "Unnamed"; + this.name = name.toLowerCase(); + this.service = service; + + this._notify = Utils.notify("weave:engine:"); + this._log = Log.repository.getLogger("Sync.Engine." + this.Name); + this._log.manageLevelFromPref(`services.sync.log.logger.engine.${this.name}`); + + this._modified = this.emptyChangeset(); + this._tracker; // initialize tracker to load previously changed IDs + this._log.debug("Engine constructed"); + + this._toFetchStorage = new JSONFile({ + path: Utils.jsonFilePath("toFetch", this.name), + dataPostProcessor: json => this._metadataPostProcessor(json), + beforeSave: () => this._beforeSaveMetadata(), + }); + + this._previousFailedStorage = new JSONFile({ + path: Utils.jsonFilePath("failed", this.name), + dataPostProcessor: json => this._metadataPostProcessor(json), + beforeSave: () => this._beforeSaveMetadata(), + }); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_enabled", + `services.sync.engine.${this.prefName}`, + false + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_syncID", + `services.sync.${this.name}.syncID`, + "" + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_lastSync", + `services.sync.${this.name}.lastSync`, + "0", + null, + v => parseFloat(v) + ); + // Async initializations can be made in the initialize() method. + + this.asyncObserver = Async.asyncObserver(this, this._log); +} + +// Enumeration to define approaches to handling bad records. +// Attached to the constructor to allow use as a kind of static enumeration. +SyncEngine.kRecoveryStrategy = { + ignore: "ignore", + retry: "retry", + error: "error", +}; + +SyncEngine.prototype = { + _recordObj: CryptoWrapper, + // _storeObj, and _trackerObj should to be overridden in subclasses + _storeObj: Store, + _trackerObj: Tracker, + version: 1, + + // Local 'constant'. + // Signal to the engine that processing further records is pointless. + eEngineAbortApplyIncoming: "error.engine.abort.applyincoming", + + // Should we keep syncing if we find a record that cannot be uploaded (ever)? + // If this is false, we'll throw, otherwise, we'll ignore the record and + // continue. This currently can only happen due to the record being larger + // than the record upload limit. + allowSkippedRecord: true, + + // Which sortindex to use when retrieving records for this engine. + _defaultSort: undefined, + + _hasSyncedThisSession: false, + + _metadataPostProcessor(json) { + if (Array.isArray(json)) { + // Pre-`JSONFile` storage stored an array, but `JSONFile` defaults to + // an object, so we wrap the array for consistency. + json = { ids: json }; + } + if (!json.ids) { + json.ids = []; + } + // The set serializes the same way as an array, but offers more efficient + // methods of manipulation. + json.ids = new SerializableSet(json.ids); + return json; + }, + + async _beforeSaveMetadata() { + await ensureDirectory(this._toFetchStorage.path); + await ensureDirectory(this._previousFailedStorage.path); + }, + + // A relative priority to use when computing an order + // for engines to be synced. Higher-priority engines + // (lower numbers) are synced first. + // It is recommended that a unique value be used for each engine, + // in order to guarantee a stable sequence. + syncPriority: 0, + + // How many records to pull in a single sync. This is primarily to avoid very + // long first syncs against profiles with many history records. + downloadLimit: null, + + // How many records to pull at one time when specifying IDs. This is to avoid + // URI length limitations. + guidFetchBatchSize: DEFAULT_GUID_FETCH_BATCH_SIZE, + + downloadBatchSize: DEFAULT_DOWNLOAD_BATCH_SIZE, + + async initialize() { + await this._toFetchStorage.load(); + await this._previousFailedStorage.load(); + Services.prefs.addObserver( + `${PREFS_BRANCH}engine.${this.prefName}`, + this.asyncObserver, + true + ); + this._log.debug("SyncEngine initialized", this.name); + }, + + get prefName() { + return this.name; + }, + + get enabled() { + return this._enabled; + }, + + set enabled(val) { + if (!!val != this._enabled) { + Svc.PrefBranch.setBoolPref("engine." + this.prefName, !!val); + } + }, + + get score() { + return this._tracker.score; + }, + + get _store() { + let store = new this._storeObj(this.Name, this); + this.__defineGetter__("_store", () => store); + return store; + }, + + get _tracker() { + let tracker = new this._trackerObj(this.Name, this); + this.__defineGetter__("_tracker", () => tracker); + return tracker; + }, + + get storageURL() { + return this.service.storageURL; + }, + + get engineURL() { + return this.storageURL + this.name; + }, + + get cryptoKeysURL() { + return this.storageURL + "crypto/keys"; + }, + + get metaURL() { + return this.storageURL + "meta/global"; + }, + + startTracking() { + this._tracker.start(); + }, + + // Returns a promise + stopTracking() { + return this._tracker.stop(); + }, + + // Listens for engine enabled state changes, and updates the tracker's state. + // This is an async observer because the tracker waits on all its async + // observers to finish when it's stopped. + async observe(subject, topic, data) { + if ( + topic == "nsPref:changed" && + data == `services.sync.engine.${this.prefName}` + ) { + await this._tracker.onEngineEnabledChanged(this._enabled); + } + }, + + async sync() { + if (!this.enabled) { + return false; + } + + if (!this._sync) { + throw new Error("engine does not implement _sync method"); + } + + return this._notify("sync", this.name, this._sync)(); + }, + + // Override this method to return a new changeset type. + emptyChangeset() { + return new Changeset(); + }, + + /** + * Returns the local sync ID for this engine, or `""` if the engine hasn't + * synced for the first time. This is exposed for tests. + * + * @return the current sync ID. + */ + async getSyncID() { + return this._syncID; + }, + + /** + * Ensures that the local sync ID for the engine matches the sync ID for the + * collection on the server. A mismatch indicates that another client wiped + * the collection; we're syncing after a node reassignment, and another + * client synced before us; or the store was replaced since the last sync. + * In case of a mismatch, we need to reset all local Sync state and start + * over as a first sync. + * + * In most cases, this method should return the new sync ID as-is. However, an + * engine may ignore the given ID and assign a different one, if it determines + * that the sync ID on the server is out of date. The bookmarks engine uses + * this to wipe the server and other clients on the first sync after the user + * restores from a backup. + * + * @param newSyncID + * The new sync ID for the collection from `meta/global`. + * @return The assigned sync ID. If this doesn't match `newSyncID`, we'll + * replace the sync ID in `meta/global` with the assigned ID. + */ + async ensureCurrentSyncID(newSyncID) { + let existingSyncID = this._syncID; + if (existingSyncID == newSyncID) { + return existingSyncID; + } + this._log.debug( + `Engine syncIDs differ (old="${existingSyncID}", new="${newSyncID}") - resetting the engine` + ); + await this.resetClient(); + Svc.PrefBranch.setStringPref(this.name + ".syncID", newSyncID); + Svc.PrefBranch.setStringPref(this.name + ".lastSync", "0"); + return newSyncID; + }, + + /** + * Resets the local sync ID for the engine, wipes the server, and resets all + * local Sync state to start over as a first sync. + * + * @return the new sync ID. + */ + async resetSyncID() { + let newSyncID = await this.resetLocalSyncID(); + await this.wipeServer(); + return newSyncID; + }, + + /** + * Resets the local sync ID for the engine, signaling that we're starting over + * as a first sync. + * + * @return the new sync ID. + */ + async resetLocalSyncID() { + return this.ensureCurrentSyncID(Utils.makeGUID()); + }, + + /** + * Allows overriding scheduler logic -- added to help reduce kinto server + * getting hammered because our scheduler never got tuned for it. + * + * Note: Overriding engines must take resyncs into account -- score will not + * be cleared. + */ + shouldSkipSync(syncReason) { + return false; + }, + + /* + * lastSync is a timestamp in server time. + */ + async getLastSync() { + return this._lastSync; + }, + async setLastSync(lastSync) { + // Store the value as a string to keep floating point precision + Svc.PrefBranch.setStringPref(this.name + ".lastSync", lastSync.toString()); + }, + async resetLastSync() { + this._log.debug("Resetting " + this.name + " last sync time"); + await this.setLastSync(0); + }, + + get hasSyncedThisSession() { + return this._hasSyncedThisSession; + }, + + set hasSyncedThisSession(hasSynced) { + this._hasSyncedThisSession = hasSynced; + }, + + get toFetch() { + this._toFetchStorage.ensureDataReady(); + return this._toFetchStorage.data.ids; + }, + + set toFetch(ids) { + if (ids.constructor.name != "SerializableSet") { + throw new Error( + "Bug: Attempted to set toFetch to something that isn't a SerializableSet" + ); + } + this._toFetchStorage.data = { ids }; + this._toFetchStorage.saveSoon(); + }, + + get previousFailed() { + this._previousFailedStorage.ensureDataReady(); + return this._previousFailedStorage.data.ids; + }, + + set previousFailed(ids) { + if (ids.constructor.name != "SerializableSet") { + throw new Error( + "Bug: Attempted to set previousFailed to something that isn't a SerializableSet" + ); + } + this._previousFailedStorage.data = { ids }; + this._previousFailedStorage.saveSoon(); + }, + + /* + * Returns a changeset for this sync. Engine implementations can override this + * method to bypass the tracker for certain or all changed items. + */ + async getChangedIDs() { + return this._tracker.getChangedIDs(); + }, + + // Create a new record using the store and add in metadata. + async _createRecord(id) { + let record = await this._store.createRecord(id, this.name); + record.id = id; + record.collection = this.name; + return record; + }, + + // Creates a tombstone Sync record with additional metadata. + _createTombstone(id) { + let tombstone = new this._recordObj(this.name, id); + tombstone.id = id; + tombstone.collection = this.name; + tombstone.deleted = true; + return tombstone; + }, + + // Any setup that needs to happen at the beginning of each sync. + async _syncStartup() { + // Determine if we need to wipe on outdated versions + let metaGlobal = await this.service.recordManager.get(this.metaURL); + let engines = metaGlobal.payload.engines || {}; + let engineData = engines[this.name] || {}; + + // Assume missing versions are 0 and wipe the server + if ((engineData.version || 0) < this.version) { + this._log.debug("Old engine data: " + [engineData.version, this.version]); + + // Clear the server and reupload everything on bad version or missing + // meta. Note that we don't regenerate per-collection keys here. + let newSyncID = await this.resetSyncID(); + + // Set the newer version and newly generated syncID + engineData.version = this.version; + engineData.syncID = newSyncID; + + // Put the new data back into meta/global and mark for upload + engines[this.name] = engineData; + metaGlobal.payload.engines = engines; + metaGlobal.changed = true; + } else if (engineData.version > this.version) { + // Don't sync this engine if the server has newer data + + let error = new Error("New data: " + [engineData.version, this.version]); + error.failureCode = VERSION_OUT_OF_DATE; + throw error; + } else { + // Changes to syncID mean we'll need to upload everything + let assignedSyncID = await this.ensureCurrentSyncID(engineData.syncID); + if (assignedSyncID != engineData.syncID) { + engineData.syncID = assignedSyncID; + metaGlobal.changed = true; + } + } + + // Save objects that need to be uploaded in this._modified. As we + // successfully upload objects we remove them from this._modified. If an + // error occurs or any objects fail to upload, they will remain in + // this._modified. At the end of a sync, or after an error, we add all + // objects remaining in this._modified to the tracker. + let initialChanges = await this.pullChanges(); + this._modified.replace(initialChanges); + // Clear the tracker now. If the sync fails we'll add the ones we failed + // to upload back. + this._tracker.clearChangedIDs(); + this._tracker.resetScore(); + + // Keep track of what to delete at the end of sync + this._delete = {}; + }, + + async pullChanges() { + let lastSync = await this.getLastSync(); + if (lastSync) { + return this.pullNewChanges(); + } + this._log.debug("First sync, uploading all items"); + return this.pullAllChanges(); + }, + + /** + * A tiny abstraction to make it easier to test incoming record + * application. + */ + itemSource() { + return new Collection(this.engineURL, this._recordObj, this.service); + }, + + /** + * Download and apply remote records changed since the last sync. This + * happens in three stages. + * + * In the first stage, we fetch full records for all changed items, newest + * first, up to the download limit. The limit lets us make progress for large + * collections, where the sync is likely to be interrupted before we + * can fetch everything. + * + * In the second stage, we fetch the IDs of any remaining records changed + * since the last sync, add them to our backlog, and fast-forward our last + * sync time. + * + * In the third stage, we fetch and apply records for all backlogged IDs, + * as well as any records that failed to apply during the last sync. We + * request records for the IDs in chunks, to avoid exceeding URL length + * limits, then remove successfully applied records from the backlog, and + * record IDs of any records that failed to apply to retry on the next sync. + */ + async _processIncoming() { + this._log.trace("Downloading & applying server changes"); + + let newitems = this.itemSource(); + let lastSync = await this.getLastSync(); + + newitems.newer = lastSync; + newitems.full = true; + + let downloadLimit = Infinity; + if (this.downloadLimit) { + // Fetch new records up to the download limit. Currently, only the history + // engine sets a limit, since the history collection has the highest volume + // of changed records between syncs. The other engines fetch all records + // changed since the last sync. + if (this._defaultSort) { + // A download limit with a sort order doesn't make sense: we won't know + // which records to backfill. + throw new Error("Can't specify download limit with default sort order"); + } + newitems.sort = "newest"; + downloadLimit = newitems.limit = this.downloadLimit; + } else if (this._defaultSort) { + // The bookmarks engine fetches records by sort index; other engines leave + // the order unspecified. We can remove `_defaultSort` entirely after bug + // 1305563: the sort index won't matter because we'll buffer all bookmarks + // before applying. + newitems.sort = this._defaultSort; + } + + // applied => number of items that should be applied. + // failed => number of items that failed in this sync. + // newFailed => number of items that failed for the first time in this sync. + // reconciled => number of items that were reconciled. + // failedReasons => {name, count} of reasons a record failed + let countTelemetry = new SyncedRecordsTelemetry(); + let count = countTelemetry.incomingCounts; + let recordsToApply = []; + let failedInCurrentSync = new SerializableSet(); + + let oldestModified = this.lastModified; + let downloadedIDs = new Set(); + + // Stage 1: Fetch new records from the server, up to the download limit. + if (this.lastModified == null || this.lastModified > lastSync) { + let { response, records } = await newitems.getBatched( + this.downloadBatchSize + ); + if (!response.success) { + response.failureCode = ENGINE_DOWNLOAD_FAIL; + throw response; + } + + await Async.yieldingForEach(records, async record => { + downloadedIDs.add(record.id); + + if (record.modified < oldestModified) { + oldestModified = record.modified; + } + + let { shouldApply, error } = await this._maybeReconcile(record); + if (error) { + failedInCurrentSync.add(record.id); + count.failed++; + countTelemetry.addIncomingFailedReason(error.message); + return; + } + if (!shouldApply) { + count.reconciled++; + return; + } + recordsToApply.push(record); + }); + + let failedToApply = await this._applyRecords( + recordsToApply, + countTelemetry + ); + Utils.setAddAll(failedInCurrentSync, failedToApply); + + // `applied` is a bit of a misnomer: it counts records that *should* be + // applied, so it also includes records that we tried to apply and failed. + // `recordsToApply.length - failedToApply.length` is the number of records + // that we *successfully* applied. + count.failed += failedToApply.length; + count.applied += recordsToApply.length; + } + + // Stage 2: If we reached our download limit, we might still have records + // on the server that changed since the last sync. Fetch the IDs for the + // remaining records, and add them to the backlog. Note that this stage + // only runs for engines that set a download limit. + if (downloadedIDs.size == downloadLimit) { + let guidColl = this.itemSource(); + + guidColl.newer = lastSync; + guidColl.older = oldestModified; + guidColl.sort = "oldest"; + + let guids = await guidColl.get(); + if (!guids.success) { + throw guids; + } + + // Filtering out already downloaded IDs here isn't necessary. We only do + // that in case the Sync server doesn't support `older` (bug 1316110). + let remainingIDs = guids.obj.filter(id => !downloadedIDs.has(id)); + if (remainingIDs.length) { + this.toFetch = Utils.setAddAll(this.toFetch, remainingIDs); + } + } + + // Fast-foward the lastSync timestamp since we have backlogged the + // remaining items. + if (lastSync < this.lastModified) { + lastSync = this.lastModified; + await this.setLastSync(lastSync); + } + + // Stage 3: Backfill records from the backlog, and those that failed to + // decrypt or apply during the last sync. We only backfill up to the + // download limit, to prevent a large backlog for one engine from blocking + // the others. We'll keep processing the backlog on subsequent engine syncs. + let failedInPreviousSync = this.previousFailed; + let idsToBackfill = Array.from( + Utils.setAddAll( + Utils.subsetOfSize(this.toFetch, downloadLimit), + failedInPreviousSync + ) + ); + + // Note that we intentionally overwrite the previously failed list here. + // Records that fail to decrypt or apply in two consecutive syncs are likely + // corrupt; we remove them from the list because retrying and failing on + // every subsequent sync just adds noise. + this.previousFailed = failedInCurrentSync; + + let backfilledItems = this.itemSource(); + + backfilledItems.sort = "newest"; + backfilledItems.full = true; + + // `getBatched` includes the list of IDs as a query parameter, so we need to fetch + // records in chunks to avoid exceeding URI length limits. + if (this.guidFetchBatchSize) { + for (let ids of lazy.PlacesUtils.chunkArray( + idsToBackfill, + this.guidFetchBatchSize + )) { + backfilledItems.ids = ids; + + let { response, records } = await backfilledItems.getBatched( + this.downloadBatchSize + ); + if (!response.success) { + response.failureCode = ENGINE_DOWNLOAD_FAIL; + throw response; + } + + let backfilledRecordsToApply = []; + let failedInBackfill = []; + + await Async.yieldingForEach(records, async record => { + let { shouldApply, error } = await this._maybeReconcile(record); + if (error) { + failedInBackfill.push(record.id); + count.failed++; + countTelemetry.addIncomingFailedReason(error.message); + return; + } + if (!shouldApply) { + count.reconciled++; + return; + } + backfilledRecordsToApply.push(record); + }); + + let failedToApply = await this._applyRecords( + backfilledRecordsToApply, + countTelemetry + ); + failedInBackfill.push(...failedToApply); + + count.failed += failedToApply.length; + count.applied += backfilledRecordsToApply.length; + + this.toFetch = Utils.setDeleteAll(this.toFetch, ids); + this.previousFailed = Utils.setAddAll( + this.previousFailed, + failedInBackfill + ); + + if (lastSync < this.lastModified) { + lastSync = this.lastModified; + await this.setLastSync(lastSync); + } + } + } + + count.newFailed = 0; + for (let item of this.previousFailed) { + // Anything that failed in the current sync that also failed in + // the previous sync means there is likely something wrong with + // the record, we remove it from trying again to prevent + // infinitely syncing corrupted records + if (failedInPreviousSync.has(item)) { + this.previousFailed.delete(item); + } else { + // otherwise it's a new failed and we count it as so + ++count.newFailed; + } + } + + count.succeeded = Math.max(0, count.applied - count.failed); + this._log.info( + [ + "Records:", + count.applied, + "applied,", + count.succeeded, + "successfully,", + count.failed, + "failed to apply,", + count.newFailed, + "newly failed to apply,", + count.reconciled, + "reconciled.", + ].join(" ") + ); + Observers.notify("weave:engine:sync:applied", count, this.name); + }, + + async _maybeReconcile(item) { + let key = this.service.collectionKeys.keyForCollection(this.name); + + // Grab a later last modified if possible + if (this.lastModified == null || item.modified > this.lastModified) { + this.lastModified = item.modified; + } + + try { + try { + await item.decrypt(key); + } catch (ex) { + if (!Utils.isHMACMismatch(ex)) { + throw ex; + } + let strategy = await this.handleHMACMismatch(item, true); + if (strategy == SyncEngine.kRecoveryStrategy.retry) { + // You only get one retry. + try { + // Try decrypting again, typically because we've got new keys. + this._log.info("Trying decrypt again..."); + key = this.service.collectionKeys.keyForCollection(this.name); + await item.decrypt(key); + strategy = null; + } catch (ex) { + if (!Utils.isHMACMismatch(ex)) { + throw ex; + } + strategy = await this.handleHMACMismatch(item, false); + } + } + + switch (strategy) { + case null: + // Retry succeeded! No further handling. + break; + case SyncEngine.kRecoveryStrategy.retry: + this._log.debug("Ignoring second retry suggestion."); + // Fall through to error case. + case SyncEngine.kRecoveryStrategy.error: + this._log.warn("Error decrypting record", ex); + return { shouldApply: false, error: ex }; + case SyncEngine.kRecoveryStrategy.ignore: + this._log.debug( + "Ignoring record " + item.id + " with bad HMAC: already handled." + ); + return { shouldApply: false, error: null }; + } + } + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.warn("Error decrypting record", ex); + return { shouldApply: false, error: ex }; + } + + if (this._shouldDeleteRemotely(item)) { + this._log.trace("Deleting item from server without applying", item); + await this._deleteId(item.id); + return { shouldApply: false, error: null }; + } + + let shouldApply; + try { + shouldApply = await this._reconcile(item); + } catch (ex) { + if (ex.code == SyncEngine.prototype.eEngineAbortApplyIncoming) { + this._log.warn("Reconciliation failed: aborting incoming processing."); + throw ex.cause; + } else if (!Async.isShutdownException(ex)) { + this._log.warn("Failed to reconcile incoming record " + item.id, ex); + return { shouldApply: false, error: ex }; + } else { + throw ex; + } + } + + if (!shouldApply) { + this._log.trace("Skipping reconciled incoming item " + item.id); + } + + return { shouldApply, error: null }; + }, + + async _applyRecords(records, countTelemetry) { + this._tracker.ignoreAll = true; + try { + let failedIDs = await this._store.applyIncomingBatch( + records, + countTelemetry + ); + return failedIDs; + } catch (ex) { + // Catch any error that escapes from applyIncomingBatch. At present + // those will all be abort events. + this._log.warn("Got exception, aborting processIncoming", ex); + throw ex; + } finally { + this._tracker.ignoreAll = false; + } + }, + + // Indicates whether an incoming item should be deleted from the server at + // the end of the sync. Engines can override this method to clean up records + // that shouldn't be on the server. + _shouldDeleteRemotely(remoteItem) { + return false; + }, + + /** + * Find a GUID of an item that is a duplicate of the incoming item but happens + * to have a different GUID + * + * @return GUID of the similar item; falsy otherwise + */ + async _findDupe(item) { + // By default, assume there's no dupe items for the engine + }, + + /** + * Called before a remote record is discarded due to failed reconciliation. + * Used by bookmark sync to merge folder child orders. + */ + beforeRecordDiscard(localRecord, remoteRecord, remoteIsNewer) {}, + + // Called when the server has a record marked as deleted, but locally we've + // changed it more recently than the deletion. If we return false, the + // record will be deleted locally. If we return true, we'll reupload the + // record to the server -- any extra work that's needed as part of this + // process should be done at this point (such as mark the record's parent + // for reuploading in the case of bookmarks). + async _shouldReviveRemotelyDeletedRecord(remoteItem) { + return true; + }, + + async _deleteId(id) { + await this._tracker.removeChangedID(id); + this._noteDeletedId(id); + }, + + // Marks an ID for deletion at the end of the sync. + _noteDeletedId(id) { + if (this._delete.ids == null) { + this._delete.ids = [id]; + } else { + this._delete.ids.push(id); + } + }, + + async _switchItemToDupe(localDupeGUID, incomingItem) { + // The local, duplicate ID is always deleted on the server. + await this._deleteId(localDupeGUID); + + // We unconditionally change the item's ID in case the engine knows of + // an item but doesn't expose it through itemExists. If the API + // contract were stronger, this could be changed. + this._log.debug( + "Switching local ID to incoming: " + + localDupeGUID + + " -> " + + incomingItem.id + ); + return this._store.changeItemID(localDupeGUID, incomingItem.id); + }, + + /** + * Reconcile incoming record with local state. + * + * This function essentially determines whether to apply an incoming record. + * + * @param item + * Record from server to be tested for application. + * @return boolean + * Truthy if incoming record should be applied. False if not. + */ + async _reconcile(item) { + if (this._log.level <= Log.Level.Trace) { + this._log.trace("Incoming: " + item); + } + + // We start reconciling by collecting a bunch of state. We do this here + // because some state may change during the course of this function and we + // need to operate on the original values. + let existsLocally = await this._store.itemExists(item.id); + let locallyModified = this._modified.has(item.id); + + // TODO Handle clock drift better. Tracked in bug 721181. + let remoteAge = Resource.serverTime - item.modified; + let localAge = locallyModified + ? Date.now() / 1000 - this._modified.getModifiedTimestamp(item.id) + : null; + let remoteIsNewer = remoteAge < localAge; + + this._log.trace( + "Reconciling " + + item.id + + ". exists=" + + existsLocally + + "; modified=" + + locallyModified + + "; local age=" + + localAge + + "; incoming age=" + + remoteAge + ); + + // We handle deletions first so subsequent logic doesn't have to check + // deleted flags. + if (item.deleted) { + // If the item doesn't exist locally, there is nothing for us to do. We + // can't check for duplicates because the incoming record has no data + // which can be used for duplicate detection. + if (!existsLocally) { + this._log.trace( + "Ignoring incoming item because it was deleted and " + + "the item does not exist locally." + ); + return false; + } + + // We decide whether to process the deletion by comparing the record + // ages. If the item is not modified locally, the remote side wins and + // the deletion is processed. If it is modified locally, we take the + // newer record. + if (!locallyModified) { + this._log.trace( + "Applying incoming delete because the local item " + + "exists and isn't modified." + ); + return true; + } + this._log.trace("Incoming record is deleted but we had local changes."); + + if (remoteIsNewer) { + this._log.trace("Remote record is newer -- deleting local record."); + return true; + } + // If the local record is newer, we defer to individual engines for + // how to handle this. By default, we revive the record. + let willRevive = await this._shouldReviveRemotelyDeletedRecord(item); + this._log.trace("Local record is newer -- reviving? " + willRevive); + + return !willRevive; + } + + // At this point the incoming record is not for a deletion and must have + // data. If the incoming record does not exist locally, we check for a local + // duplicate existing under a different ID. The default implementation of + // _findDupe() is empty, so engines have to opt in to this functionality. + // + // If we find a duplicate, we change the local ID to the incoming ID and we + // refresh the metadata collected above. See bug 710448 for the history + // of this logic. + if (!existsLocally) { + let localDupeGUID = await this._findDupe(item); + if (localDupeGUID) { + this._log.trace( + "Local item " + + localDupeGUID + + " is a duplicate for " + + "incoming item " + + item.id + ); + + // The current API contract does not mandate that the ID returned by + // _findDupe() actually exists. Therefore, we have to perform this + // check. + existsLocally = await this._store.itemExists(localDupeGUID); + + // If the local item was modified, we carry its metadata forward so + // appropriate reconciling can be performed. + if (this._modified.has(localDupeGUID)) { + locallyModified = true; + localAge = + this._tracker._now() - + this._modified.getModifiedTimestamp(localDupeGUID); + remoteIsNewer = remoteAge < localAge; + + this._modified.changeID(localDupeGUID, item.id); + } else { + locallyModified = false; + localAge = null; + } + + // Tell the engine to do whatever it needs to switch the items. + await this._switchItemToDupe(localDupeGUID, item); + + this._log.debug( + "Local item after duplication: age=" + + localAge + + "; modified=" + + locallyModified + + "; exists=" + + existsLocally + ); + } else { + this._log.trace("No duplicate found for incoming item: " + item.id); + } + } + + // At this point we've performed duplicate detection. But, nothing here + // should depend on duplicate detection as the above should have updated + // state seamlessly. + + if (!existsLocally) { + // If the item doesn't exist locally and we have no local modifications + // to the item (implying that it was not deleted), always apply the remote + // item. + if (!locallyModified) { + this._log.trace( + "Applying incoming because local item does not exist " + + "and was not deleted." + ); + return true; + } + + // If the item was modified locally but isn't present, it must have + // been deleted. If the incoming record is younger, we restore from + // that record. + if (remoteIsNewer) { + this._log.trace( + "Applying incoming because local item was deleted " + + "before the incoming item was changed." + ); + this._modified.delete(item.id); + return true; + } + + this._log.trace( + "Ignoring incoming item because the local item's " + + "deletion is newer." + ); + return false; + } + + // If the remote and local records are the same, there is nothing to be + // done, so we don't do anything. In the ideal world, this logic wouldn't + // be here and the engine would take a record and apply it. The reason we + // want to defer this logic is because it would avoid a redundant and + // possibly expensive dip into the storage layer to query item state. + // This should get addressed in the async rewrite, so we ignore it for now. + let localRecord = await this._createRecord(item.id); + let recordsEqual = Utils.deepEquals(item.cleartext, localRecord.cleartext); + + // If the records are the same, we don't need to do anything. This does + // potentially throw away a local modification time. But, if the records + // are the same, does it matter? + if (recordsEqual) { + this._log.trace( + "Ignoring incoming item because the local item is identical." + ); + + this._modified.delete(item.id); + return false; + } + + // At this point the records are different. + + // If we have no local modifications, always take the server record. + if (!locallyModified) { + this._log.trace("Applying incoming record because no local conflicts."); + return true; + } + + // At this point, records are different and the local record is modified. + // We resolve conflicts by record age, where the newest one wins. This does + // result in data loss and should be handled by giving the engine an + // opportunity to merge the records. Bug 720592 tracks this feature. + this._log.warn( + "DATA LOSS: Both local and remote changes to record: " + item.id + ); + if (!remoteIsNewer) { + this.beforeRecordDiscard(localRecord, item, remoteIsNewer); + } + return remoteIsNewer; + }, + + // Upload outgoing records. + async _uploadOutgoing() { + this._log.trace("Uploading local changes to server."); + + // collection we'll upload + let up = new Collection(this.engineURL, null, this.service); + let modifiedIDs = new Set(this._modified.ids()); + let countTelemetry = new SyncedRecordsTelemetry(); + let counts = countTelemetry.outgoingCounts; + this._log.info(`Uploading ${modifiedIDs.size} outgoing records`); + if (modifiedIDs.size) { + counts.sent = modifiedIDs.size; + + let failed = []; + let successful = []; + let lastSync = await this.getLastSync(); + let handleResponse = async (postQueue, resp, batchOngoing) => { + // Note: We don't want to update this.lastSync, or this._modified until + // the batch is complete, however we want to remember success/failure + // indicators for when that happens. + if (!resp.success) { + this._log.debug(`Uploading records failed: ${resp.status}`); + resp.failureCode = + resp.status == 412 ? ENGINE_BATCH_INTERRUPTED : ENGINE_UPLOAD_FAIL; + throw resp; + } + + // Update server timestamp from the upload. + failed = failed.concat(Object.keys(resp.obj.failed)); + successful = successful.concat(resp.obj.success); + + if (batchOngoing) { + // Nothing to do yet + return; + } + + if (failed.length && this._log.level <= Log.Level.Debug) { + this._log.debug( + "Records that will be uploaded again because " + + "the server couldn't store them: " + + failed.join(", ") + ); + } + + counts.failed += failed.length; + Object.values(failed).forEach(message => { + countTelemetry.addOutgoingFailedReason(message); + }); + + for (let id of successful) { + this._modified.delete(id); + } + + await this._onRecordsWritten( + successful, + failed, + postQueue.lastModified + ); + + // Advance lastSync since we've finished the batch. + if (postQueue.lastModified > lastSync) { + lastSync = postQueue.lastModified; + await this.setLastSync(lastSync); + } + + // clear for next batch + failed.length = 0; + successful.length = 0; + }; + + let postQueue = up.newPostQueue(this._log, lastSync, handleResponse); + + for (let id of modifiedIDs) { + let out; + let ok = false; + try { + out = await this._createRecord(id); + if (this._log.level <= Log.Level.Trace) { + this._log.trace("Outgoing: " + out); + } + await out.encrypt( + this.service.collectionKeys.keyForCollection(this.name) + ); + ok = true; + } catch (ex) { + this._log.warn("Error creating record", ex); + ++counts.failed; + countTelemetry.addOutgoingFailedReason(ex.message); + if (Async.isShutdownException(ex) || !this.allowSkippedRecord) { + if (!this.allowSkippedRecord) { + // Don't bother for shutdown errors + Observers.notify("weave:engine:sync:uploaded", counts, this.name); + } + throw ex; + } + } + if (ok) { + let { enqueued, error } = await postQueue.enqueue(out); + if (!enqueued) { + ++counts.failed; + countTelemetry.addOutgoingFailedReason(error.message); + if (!this.allowSkippedRecord) { + Observers.notify("weave:engine:sync:uploaded", counts, this.name); + this._log.warn( + `Failed to enqueue record "${id}" (aborting)`, + error + ); + throw error; + } + this._modified.delete(id); + this._log.warn( + `Failed to enqueue record "${id}" (skipping)`, + error + ); + } + } + await Async.promiseYield(); + } + await postQueue.flush(true); + } + + if (counts.sent || counts.failed) { + Observers.notify("weave:engine:sync:uploaded", counts, this.name); + } + }, + + async _onRecordsWritten(succeeded, failed, serverModifiedTime) { + // Implement this method to take specific actions against successfully + // uploaded records and failed records. + }, + + // Any cleanup necessary. + // Save the current snapshot so as to calculate changes at next sync + async _syncFinish() { + this._log.trace("Finishing up sync"); + + let doDelete = async (key, val) => { + let coll = new Collection(this.engineURL, this._recordObj, this.service); + coll[key] = val; + await coll.delete(); + }; + + for (let [key, val] of Object.entries(this._delete)) { + // Remove the key for future uses + delete this._delete[key]; + + this._log.trace("doing post-sync deletions", { key, val }); + // Send a simple delete for the property + if (key != "ids" || val.length <= 100) { + await doDelete(key, val); + } else { + // For many ids, split into chunks of at most 100 + while (val.length) { + await doDelete(key, val.slice(0, 100)); + val = val.slice(100); + } + } + } + this.hasSyncedThisSession = true; + await this._tracker.asyncObserver.promiseObserversComplete(); + }, + + async _syncCleanup() { + try { + // Mark failed WBOs as changed again so they are reuploaded next time. + await this.trackRemainingChanges(); + } finally { + this._modified.clear(); + } + }, + + async _sync() { + try { + Async.checkAppReady(); + await this._syncStartup(); + Async.checkAppReady(); + Observers.notify("weave:engine:sync:status", "process-incoming"); + await this._processIncoming(); + Async.checkAppReady(); + Observers.notify("weave:engine:sync:status", "upload-outgoing"); + try { + await this._uploadOutgoing(); + Async.checkAppReady(); + await this._syncFinish(); + } catch (ex) { + if (!ex.status || ex.status != 412) { + throw ex; + } + // a 412 posting just means another client raced - but we don't want + // to treat that as a sync error - the next sync is almost certain + // to work. + this._log.warn("412 error during sync - will retry."); + } + } finally { + await this._syncCleanup(); + } + }, + + async canDecrypt() { + // Report failure even if there's nothing to decrypt + let canDecrypt = false; + + // Fetch the most recently uploaded record and try to decrypt it + let test = new Collection(this.engineURL, this._recordObj, this.service); + test.limit = 1; + test.sort = "newest"; + test.full = true; + + let key = this.service.collectionKeys.keyForCollection(this.name); + + // Any failure fetching/decrypting will just result in false + try { + this._log.trace("Trying to decrypt a record from the server.."); + let json = (await test.get()).obj[0]; + let record = new this._recordObj(); + record.deserialize(json); + await record.decrypt(key); + canDecrypt = true; + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.debug("Failed test decrypt", ex); + } + + return canDecrypt; + }, + + /** + * Deletes the collection for this engine on the server, and removes all local + * Sync metadata for this engine. This does *not* remove any existing data on + * other clients. This is called when we reset the sync ID. + */ + async wipeServer() { + await this._deleteServerCollection(); + await this._resetClient(); + }, + + /** + * Deletes the collection for this engine on the server, without removing + * any local Sync metadata or user data. Deleting the collection will not + * remove any user data on other clients, but will force other clients to + * start over as a first sync. + */ + async _deleteServerCollection() { + let response = await this.service.resource(this.engineURL).delete(); + if (response.status != 200 && response.status != 404) { + throw response; + } + }, + + async removeClientData() { + // Implement this method in engines that store client specific data + // on the server. + }, + + /* + * Decide on (and partially effect) an error-handling strategy. + * + * Asks the Service to respond to an HMAC error, which might result in keys + * being downloaded. That call returns true if an action which might allow a + * retry to occur. + * + * If `mayRetry` is truthy, and the Service suggests a retry, + * handleHMACMismatch returns kRecoveryStrategy.retry. Otherwise, it returns + * kRecoveryStrategy.error. + * + * Subclasses of SyncEngine can override this method to allow for different + * behavior -- e.g., to delete and ignore erroneous entries. + * + * All return values will be part of the kRecoveryStrategy enumeration. + */ + async handleHMACMismatch(item, mayRetry) { + // By default we either try again, or bail out noisily. + return (await this.service.handleHMACEvent()) && mayRetry + ? SyncEngine.kRecoveryStrategy.retry + : SyncEngine.kRecoveryStrategy.error; + }, + + /** + * Returns a changeset containing all items in the store. The default + * implementation returns a changeset with timestamps from long ago, to + * ensure we always use the remote version if one exists. + * + * This function is only called for the first sync. Subsequent syncs call + * `pullNewChanges`. + * + * @return A `Changeset` object. + */ + async pullAllChanges() { + let changes = {}; + let ids = await this._store.getAllIDs(); + for (let id in ids) { + changes[id] = 0; + } + return changes; + }, + + /* + * Returns a changeset containing entries for all currently tracked items. + * The default implementation returns a changeset with timestamps indicating + * when the item was added to the tracker. + * + * @return A `Changeset` object. + */ + async pullNewChanges() { + await this._tracker.asyncObserver.promiseObserversComplete(); + return this.getChangedIDs(); + }, + + /** + * Adds all remaining changeset entries back to the tracker, typically for + * items that failed to upload. This method is called at the end of each sync. + * + */ + async trackRemainingChanges() { + for (let [id, change] of this._modified.entries()) { + await this._tracker.addChangedID(id, change); + } + }, + + /** + * Removes all local Sync metadata for this engine, but keeps all existing + * local user data. + */ + async resetClient() { + return this._notify("reset-client", this.name, this._resetClient)(); + }, + + async _resetClient() { + await this.resetLastSync(); + this.hasSyncedThisSession = false; + this.previousFailed = new SerializableSet(); + this.toFetch = new SerializableSet(); + }, + + /** + * Removes all local Sync metadata and user data for this engine. + */ + async wipeClient() { + return this._notify("wipe-client", this.name, this._wipeClient)(); + }, + + async _wipeClient() { + await this.resetClient(); + this._log.debug("Deleting all local data"); + this._tracker.ignoreAll = true; + await this._store.wipe(); + this._tracker.ignoreAll = false; + this._tracker.clearChangedIDs(); + }, + + /** + * If one exists, initialize and return a validator for this engine (which + * must have a `validate(engine)` method that returns a promise to an object + * with a getSummary method). Otherwise return null. + */ + getValidator() { + return null; + }, + + async finalize() { + Services.prefs.removeObserver( + `${PREFS_BRANCH}engine.${this.prefName}`, + this.asyncObserver + ); + await this.asyncObserver.promiseObserversComplete(); + await this._tracker.finalize(); + await this._toFetchStorage.finalize(); + await this._previousFailedStorage.finalize(); + }, + + // Returns a new watchdog. Exposed for tests. + _newWatchdog() { + return Async.watchdog(); + }, +}; + +/** + * A changeset is created for each sync in `Engine::get{Changed, All}IDs`, + * and stores opaque change data for tracked IDs. The default implementation + * only records timestamps, though engines can extend this to store additional + * data for each entry. + */ +export class Changeset { + // Creates an empty changeset. + constructor() { + this.changes = {}; + } + + // Returns the last modified time, in seconds, for an entry in the changeset. + // `id` is guaranteed to be in the set. + getModifiedTimestamp(id) { + return this.changes[id]; + } + + // Adds a change for a tracked ID to the changeset. + set(id, change) { + this.changes[id] = change; + } + + // Adds multiple entries to the changeset, preserving existing entries. + insert(changes) { + Object.assign(this.changes, changes); + } + + // Overwrites the existing set of tracked changes with new entries. + replace(changes) { + this.changes = changes; + } + + // Indicates whether an entry is in the changeset. + has(id) { + return id in this.changes; + } + + // Deletes an entry from the changeset. Used to clean up entries for + // reconciled and successfully uploaded records. + delete(id) { + delete this.changes[id]; + } + + // Changes the ID of an entry in the changeset. Used when reconciling + // duplicates that have local changes. + changeID(oldID, newID) { + this.changes[newID] = this.changes[oldID]; + delete this.changes[oldID]; + } + + // Returns an array of all tracked IDs in this changeset. + ids() { + return Object.keys(this.changes); + } + + // Returns an array of `[id, change]` tuples. Used to repopulate the tracker + // with entries for failed uploads at the end of a sync. + entries() { + return Object.entries(this.changes); + } + + // Returns the number of entries in this changeset. + count() { + return this.ids().length; + } + + // Clears the changeset. + clear() { + this.changes = {}; + } +} diff --git a/services/sync/modules/engines/addons.sys.mjs b/services/sync/modules/engines/addons.sys.mjs new file mode 100644 index 0000000000..782d23239e --- /dev/null +++ b/services/sync/modules/engines/addons.sys.mjs @@ -0,0 +1,818 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This file defines the add-on sync functionality. + * + * There are currently a number of known limitations: + * - We only sync XPI extensions and themes available from addons.mozilla.org. + * We hope to expand support for other add-ons eventually. + * - We only attempt syncing of add-ons between applications of the same type. + * This means add-ons will not synchronize between Firefox desktop and + * Firefox mobile, for example. This is because of significant add-on + * incompatibility between application types. + * + * Add-on records exist for each known {add-on, app-id} pair in the Sync client + * set. Each record has a randomly chosen GUID. The records then contain + * basic metadata about the add-on. + * + * We currently synchronize: + * + * - Installations + * - Uninstallations + * - User enabling and disabling + * + * Synchronization is influenced by the following preferences: + * + * - services.sync.addons.ignoreUserEnabledChanges + * - services.sync.addons.trustedSourceHostnames + * + * and also influenced by whether addons have repository caching enabled and + * whether they allow installation of addons from insecure options (both of + * which are themselves influenced by the "extensions." pref branch) + * + * See the documentation in all.js for the behavior of these prefs. + */ + +import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs"; +import { AddonsReconciler } from "resource://services-sync/addonsreconciler.sys.mjs"; +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AddonRepository: "resource://gre/modules/addons/AddonRepository.sys.mjs", +}); + +// 7 days in milliseconds. +const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000; + +/** + * AddonRecord represents the state of an add-on in an application. + * + * Each add-on has its own record for each application ID it is installed + * on. + * + * The ID of add-on records is a randomly-generated GUID. It is random instead + * of deterministic so the URIs of the records cannot be guessed and so + * compromised server credentials won't result in disclosure of the specific + * add-ons present in a Sync account. + * + * The record contains the following fields: + * + * addonID + * ID of the add-on. This correlates to the "id" property on an Addon type. + * + * applicationID + * The application ID this record is associated with. + * + * enabled + * Boolean stating whether add-on is enabled or disabled by the user. + * + * source + * String indicating where an add-on is from. Currently, we only support + * the value "amo" which indicates that the add-on came from the official + * add-ons repository, addons.mozilla.org. In the future, we may support + * installing add-ons from other sources. This provides a future-compatible + * mechanism for clients to only apply records they know how to handle. + */ +function AddonRecord(collection, id) { + CryptoWrapper.call(this, collection, id); +} +AddonRecord.prototype = { + _logName: "Record.Addon", +}; +Object.setPrototypeOf(AddonRecord.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(AddonRecord, "cleartext", [ + "addonID", + "applicationID", + "enabled", + "source", +]); + +/** + * The AddonsEngine handles synchronization of add-ons between clients. + * + * The engine maintains an instance of an AddonsReconciler, which is the entity + * maintaining state for add-ons. It provides the history and tracking APIs + * that AddonManager doesn't. + * + * The engine instance overrides a handful of functions on the base class. The + * rationale for each is documented by that function. + */ +export function AddonsEngine(service) { + SyncEngine.call(this, "Addons", service); + + this._reconciler = new AddonsReconciler(this._tracker.asyncObserver); +} + +AddonsEngine.prototype = { + _storeObj: AddonsStore, + _trackerObj: AddonsTracker, + _recordObj: AddonRecord, + version: 1, + + syncPriority: 5, + + _reconciler: null, + + async initialize() { + await SyncEngine.prototype.initialize.call(this); + await this._reconciler.ensureStateLoaded(); + }, + + /** + * Override parent method to find add-ons by their public ID, not Sync GUID. + */ + async _findDupe(item) { + let id = item.addonID; + + // The reconciler should have been updated at the top of the sync, so we + // can assume it is up to date when this function is called. + let addons = this._reconciler.addons; + if (!(id in addons)) { + return null; + } + + let addon = addons[id]; + if (addon.guid != item.id) { + return addon.guid; + } + + return null; + }, + + /** + * Override getChangedIDs to pull in tracker changes plus changes from the + * reconciler log. + */ + async getChangedIDs() { + let changes = {}; + const changedIDs = await this._tracker.getChangedIDs(); + for (let [id, modified] of Object.entries(changedIDs)) { + changes[id] = modified; + } + + let lastSync = await this.getLastSync(); + let lastSyncDate = new Date(lastSync * 1000); + + // The reconciler should have been refreshed at the beginning of a sync and + // we assume this function is only called from within a sync. + let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate); + let addons = this._reconciler.addons; + for (let change of reconcilerChanges) { + let changeTime = change[0]; + let id = change[2]; + + if (!(id in addons)) { + continue; + } + + // Keep newest modified time. + if (id in changes && changeTime < changes[id]) { + continue; + } + + if (!(await this.isAddonSyncable(addons[id]))) { + continue; + } + + this._log.debug("Adding changed add-on from changes log: " + id); + let addon = addons[id]; + changes[addon.guid] = changeTime.getTime() / 1000; + } + + return changes; + }, + + /** + * Override start of sync function to refresh reconciler. + * + * Many functions in this class assume the reconciler is refreshed at the + * top of a sync. If this ever changes, those functions should be revisited. + * + * Technically speaking, we don't need to refresh the reconciler on every + * sync since it is installed as an AddonManager listener. However, add-ons + * are complicated and we force a full refresh, just in case the listeners + * missed something. + */ + async _syncStartup() { + // We refresh state before calling parent because syncStartup in the parent + // looks for changed IDs, which is dependent on add-on state being up to + // date. + await this._refreshReconcilerState(); + return SyncEngine.prototype._syncStartup.call(this); + }, + + /** + * Override end of sync to perform a little housekeeping on the reconciler. + * + * We prune changes to prevent the reconciler state from growing without + * bound. Even if it grows unbounded, there would have to be many add-on + * changes (thousands) for it to slow things down significantly. This is + * highly unlikely to occur. Still, we exercise defense just in case. + */ + async _syncCleanup() { + let lastSync = await this.getLastSync(); + let ms = 1000 * lastSync - PRUNE_ADDON_CHANGES_THRESHOLD; + this._reconciler.pruneChangesBeforeDate(new Date(ms)); + return SyncEngine.prototype._syncCleanup.call(this); + }, + + /** + * Helper function to ensure reconciler is up to date. + * + * This will load the reconciler's state from the file + * system (if needed) and refresh the state of the reconciler. + */ + async _refreshReconcilerState() { + this._log.debug("Refreshing reconciler state"); + return this._reconciler.refreshGlobalState(); + }, + + // Returns a promise + isAddonSyncable(addon, ignoreRepoCheck) { + return this._store.isAddonSyncable(addon, ignoreRepoCheck); + }, +}; +Object.setPrototypeOf(AddonsEngine.prototype, SyncEngine.prototype); + +/** + * This is the primary interface between Sync and the Addons Manager. + * + * In addition to the core store APIs, we provide convenience functions to wrap + * Add-on Manager APIs with Sync-specific semantics. + */ +function AddonsStore(name, engine) { + Store.call(this, name, engine); +} +AddonsStore.prototype = { + // Define the add-on types (.type) that we support. + _syncableTypes: ["extension", "theme"], + + _extensionsPrefs: Services.prefs.getBranch("extensions."), + + get reconciler() { + return this.engine._reconciler; + }, + + /** + * Override applyIncoming to filter out records we can't handle. + */ + async applyIncoming(record) { + // The fields we look at aren't present when the record is deleted. + if (!record.deleted) { + // Ignore records not belonging to our application ID because that is the + // current policy. + if (record.applicationID != Services.appinfo.ID) { + this._log.info( + "Ignoring incoming record from other App ID: " + record.id + ); + return; + } + + // Ignore records that aren't from the official add-on repository, as that + // is our current policy. + if (record.source != "amo") { + this._log.info( + "Ignoring unknown add-on source (" + + record.source + + ")" + + " for " + + record.id + ); + return; + } + } + + // Ignore incoming records for which an existing non-syncable addon + // exists. Note that we do not insist that the addon manager already have + // metadata for this addon - it's possible our reconciler previously saw the + // addon but the addon-manager cache no longer has it - which is fine for a + // new incoming addon. + // (Note that most other cases where the addon-manager cache is invalid + // doesn't get this treatment because that cache self-repairs after some + // time - but it only re-populates addons which are currently installed.) + let existingMeta = this.reconciler.addons[record.addonID]; + if ( + existingMeta && + !(await this.isAddonSyncable(existingMeta, /* ignoreRepoCheck */ true)) + ) { + this._log.info( + "Ignoring incoming record for an existing but non-syncable addon", + record.addonID + ); + return; + } + + await Store.prototype.applyIncoming.call(this, record); + }, + + /** + * Provides core Store API to create/install an add-on from a record. + */ + async create(record) { + // This will throw if there was an error. This will get caught by the sync + // engine and the record will try to be applied later. + const results = await AddonUtils.installAddons([ + { + id: record.addonID, + syncGUID: record.id, + enabled: record.enabled, + requireSecureURI: this._extensionsPrefs.getBoolPref( + "install.requireSecureOrigin", + true + ), + }, + ]); + + if (results.skipped.includes(record.addonID)) { + this._log.info("Add-on skipped: " + record.addonID); + // Just early-return for skipped addons - we don't want to arrange to + // try again next time because the condition that caused up to skip + // will remain true for this addon forever. + return; + } + + let addon; + for (let a of results.addons) { + if (a.id == record.addonID) { + addon = a; + break; + } + } + + // This should never happen, but is present as a fail-safe. + if (!addon) { + throw new Error("Add-on not found after install: " + record.addonID); + } + + this._log.info("Add-on installed: " + record.addonID); + }, + + /** + * Provides core Store API to remove/uninstall an add-on from a record. + */ + async remove(record) { + // If this is called, the payload is empty, so we have to find by GUID. + let addon = await this.getAddonByGUID(record.id); + if (!addon) { + // We don't throw because if the add-on could not be found then we assume + // it has already been uninstalled and there is nothing for this function + // to do. + return; + } + + this._log.info("Uninstalling add-on: " + addon.id); + await AddonUtils.uninstallAddon(addon); + }, + + /** + * Provides core Store API to update an add-on from a record. + */ + async update(record) { + let addon = await this.getAddonByID(record.addonID); + + // update() is called if !this.itemExists. And, since itemExists consults + // the reconciler only, we need to take care of some corner cases. + // + // First, the reconciler could know about an add-on that was uninstalled + // and no longer present in the add-ons manager. + if (!addon) { + await this.create(record); + return; + } + + // It's also possible that the add-on is non-restartless and has pending + // install/uninstall activity. + // + // We wouldn't get here if the incoming record was for a deletion. So, + // check for pending uninstall and cancel if necessary. + if (addon.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL) { + addon.cancelUninstall(); + + // We continue with processing because there could be state or ID change. + } + + await this.updateUserDisabled(addon, !record.enabled); + }, + + /** + * Provide core Store API to determine if a record exists. + */ + async itemExists(guid) { + let addon = this.reconciler.getAddonStateFromSyncGUID(guid); + + return !!addon; + }, + + /** + * Create an add-on record from its GUID. + * + * @param guid + * Add-on GUID (from extensions DB) + * @param collection + * Collection to add record to. + * + * @return AddonRecord instance + */ + async createRecord(guid, collection) { + let record = new AddonRecord(collection, guid); + record.applicationID = Services.appinfo.ID; + + let addon = this.reconciler.getAddonStateFromSyncGUID(guid); + + // If we don't know about this GUID or if it has been uninstalled, we mark + // the record as deleted. + if (!addon || !addon.installed) { + record.deleted = true; + return record; + } + + record.modified = addon.modified.getTime() / 1000; + + record.addonID = addon.id; + record.enabled = addon.enabled; + + // This needs to be dynamic when add-ons don't come from AddonRepository. + record.source = "amo"; + + return record; + }, + + /** + * Changes the id of an add-on. + * + * This implements a core API of the store. + */ + async changeItemID(oldID, newID) { + // We always update the GUID in the reconciler because it will be + // referenced later in the sync process. + let state = this.reconciler.getAddonStateFromSyncGUID(oldID); + if (state) { + state.guid = newID; + await this.reconciler.saveState(); + } + + let addon = await this.getAddonByGUID(oldID); + if (!addon) { + this._log.debug( + "Cannot change item ID (" + + oldID + + ") in Add-on " + + "Manager because old add-on not present: " + + oldID + ); + return; + } + + addon.syncGUID = newID; + }, + + /** + * Obtain the set of all syncable add-on Sync GUIDs. + * + * This implements a core Store API. + */ + async getAllIDs() { + let ids = {}; + + let addons = this.reconciler.addons; + for (let id in addons) { + let addon = addons[id]; + if (await this.isAddonSyncable(addon)) { + ids[addon.guid] = true; + } + } + + return ids; + }, + + /** + * Wipe engine data. + * + * This uninstalls all syncable addons from the application. In case of + * error, it logs the error and keeps trying with other add-ons. + */ + async wipe() { + this._log.info("Processing wipe."); + + await this.engine._refreshReconcilerState(); + + // We only wipe syncable add-ons. Wipe is a Sync feature not a security + // feature. + let ids = await this.getAllIDs(); + for (let guid in ids) { + let addon = await this.getAddonByGUID(guid); + if (!addon) { + this._log.debug( + "Ignoring add-on because it couldn't be obtained: " + guid + ); + continue; + } + + this._log.info("Uninstalling add-on as part of wipe: " + addon.id); + await Utils.catch.call(this, () => addon.uninstall())(); + } + }, + + /** ************************************************************************* + * Functions below are unique to this store and not part of the Store API * + ***************************************************************************/ + + /** + * Obtain an add-on from its public ID. + * + * @param id + * Add-on ID + * @return Addon or undefined if not found + */ + async getAddonByID(id) { + return lazy.AddonManager.getAddonByID(id); + }, + + /** + * Obtain an add-on from its Sync GUID. + * + * @param guid + * Add-on Sync GUID + * @return DBAddonInternal or null + */ + async getAddonByGUID(guid) { + return lazy.AddonManager.getAddonBySyncGUID(guid); + }, + + /** + * Determines whether an add-on is suitable for Sync. + * + * @param addon + * Addon instance + * @param ignoreRepoCheck + * Should we skip checking the Addons repository (primarially useful + * for testing and validation). + * @return Boolean indicating whether it is appropriate for Sync + */ + async isAddonSyncable(addon, ignoreRepoCheck = false) { + // Currently, we limit syncable add-ons to those that are: + // 1) In a well-defined set of types + // 2) Installed in the current profile + // 3) Not installed by a foreign entity (i.e. installed by the app) + // since they act like global extensions. + // 4) Is not a hotfix. + // 5) The addons XPIProvider doesn't veto it (i.e not being installed in + // the profile directory, or any other reasons it says the addon can't + // be synced) + // 6) Are installed from AMO + + // We could represent the test as a complex boolean expression. We go the + // verbose route so the failure reason is logged. + if (!addon) { + this._log.debug("Null object passed to isAddonSyncable."); + return false; + } + + if (!this._syncableTypes.includes(addon.type)) { + this._log.debug( + addon.id + " not syncable: type not in allowed list: " + addon.type + ); + return false; + } + + if (!(addon.scope & lazy.AddonManager.SCOPE_PROFILE)) { + this._log.debug(addon.id + " not syncable: not installed in profile."); + return false; + } + + // If the addon manager says it's not syncable, we skip it. + if (!addon.isSyncable) { + this._log.debug(addon.id + " not syncable: vetoed by the addon manager."); + return false; + } + + // This may be too aggressive. If an add-on is downloaded from AMO and + // manually placed in the profile directory, foreignInstall will be set. + // Arguably, that add-on should be syncable. + // TODO Address the edge case and come up with more robust heuristics. + if (addon.foreignInstall) { + this._log.debug(addon.id + " not syncable: is foreign install."); + return false; + } + + // If the AddonRepository's cache isn't enabled (which it typically isn't + // in tests), getCachedAddonByID always returns null - so skip the check + // in that case. We also provide a way to specifically opt-out of the check + // even if the cache is enabled, which is used by the validators. + if (ignoreRepoCheck || !lazy.AddonRepository.cacheEnabled) { + return true; + } + + let result = await new Promise(res => { + lazy.AddonRepository.getCachedAddonByID(addon.id, res); + }); + + if (!result) { + this._log.debug( + addon.id + " not syncable: add-on not found in add-on repository." + ); + return false; + } + + return this.isSourceURITrusted(result.sourceURI); + }, + + /** + * Determine whether an add-on's sourceURI field is trusted and the add-on + * can be installed. + * + * This function should only ever be called from isAddonSyncable(). It is + * exposed as a separate function to make testing easier. + * + * @param uri + * nsIURI instance to validate + * @return bool + */ + isSourceURITrusted: function isSourceURITrusted(uri) { + // For security reasons, we currently limit synced add-ons to those + // installed from trusted hostname(s). We additionally require TLS with + // the add-ons site to help prevent forgeries. + let trustedHostnames = Svc.PrefBranch.getStringPref( + "addons.trustedSourceHostnames", + "" + ).split(","); + + if (!uri) { + this._log.debug("Undefined argument to isSourceURITrusted()."); + return false; + } + + // Scheme is validated before the hostname because uri.host may not be + // populated for certain schemes. It appears to always be populated for + // https, so we avoid the potential NS_ERROR_FAILURE on field access. + if (uri.scheme != "https") { + this._log.debug("Source URI not HTTPS: " + uri.spec); + return false; + } + + if (!trustedHostnames.includes(uri.host)) { + this._log.debug("Source hostname not trusted: " + uri.host); + return false; + } + + return true; + }, + + /** + * Update the userDisabled flag on an add-on. + * + * This will enable or disable an add-on. It has no return value and does + * not catch or handle exceptions thrown by the addon manager. If no action + * is needed it will return immediately. + * + * @param addon + * Addon instance to manipulate. + * @param value + * Boolean to which to set userDisabled on the passed Addon. + */ + async updateUserDisabled(addon, value) { + if (addon.userDisabled == value) { + return; + } + + // A pref allows changes to the enabled flag to be ignored. + if (Svc.PrefBranch.getBoolPref("addons.ignoreUserEnabledChanges", false)) { + this._log.info( + "Ignoring enabled state change due to preference: " + addon.id + ); + return; + } + + AddonUtils.updateUserDisabled(addon, value); + // updating this flag doesn't send a notification for appDisabled addons, + // meaning the reconciler will not update its state and may resync the + // addon - so explicitly rectify the state (bug 1366994) + if (addon.appDisabled) { + await this.reconciler.rectifyStateFromAddon(addon); + } + }, +}; + +Object.setPrototypeOf(AddonsStore.prototype, Store.prototype); + +/** + * The add-ons tracker keeps track of real-time changes to add-ons. + * + * It hooks up to the reconciler and receives notifications directly from it. + */ +function AddonsTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +AddonsTracker.prototype = { + get reconciler() { + return this.engine._reconciler; + }, + + get store() { + return this.engine._store; + }, + + /** + * This callback is executed whenever the AddonsReconciler sends out a change + * notification. See AddonsReconciler.addChangeListener(). + */ + async changeListener(date, change, addon) { + this._log.debug("changeListener invoked: " + change + " " + addon.id); + // Ignore changes that occur during sync. + if (this.ignoreAll) { + return; + } + + if (!(await this.store.isAddonSyncable(addon))) { + this._log.debug( + "Ignoring change because add-on isn't syncable: " + addon.id + ); + return; + } + + const added = await this.addChangedID(addon.guid, date.getTime() / 1000); + if (added) { + this.score += SCORE_INCREMENT_XLARGE; + } + }, + + onStart() { + this.reconciler.startListening(); + this.reconciler.addChangeListener(this); + }, + + onStop() { + this.reconciler.removeChangeListener(this); + this.reconciler.stopListening(); + }, +}; + +Object.setPrototypeOf(AddonsTracker.prototype, LegacyTracker.prototype); + +export class AddonValidator extends CollectionValidator { + constructor(engine = null) { + super("addons", "id", ["addonID", "enabled", "applicationID", "source"]); + this.engine = engine; + } + + async getClientItems() { + return lazy.AddonManager.getAllAddons(); + } + + normalizeClientItem(item) { + let enabled = !item.userDisabled; + if (item.pendingOperations & lazy.AddonManager.PENDING_ENABLE) { + enabled = true; + } else if (item.pendingOperations & lazy.AddonManager.PENDING_DISABLE) { + enabled = false; + } + return { + enabled, + id: item.syncGUID, + addonID: item.id, + applicationID: Services.appinfo.ID, + source: "amo", // check item.foreignInstall? + original: item, + }; + } + + async normalizeServerItem(item) { + let guid = await this.engine._findDupe(item); + if (guid) { + item.id = guid; + } + return item; + } + + clientUnderstands(item) { + return item.applicationID === Services.appinfo.ID; + } + + async syncedByClient(item) { + return ( + !item.original.hidden && + !item.original.isSystem && + !( + item.original.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL + ) && + // No need to await the returned promise explicitely: + // |expr1 && expr2| evaluates to expr2 if expr1 is true. + this.engine.isAddonSyncable(item.original, true) + ); + } +} diff --git a/services/sync/modules/engines/bookmarks.sys.mjs b/services/sync/modules/engines/bookmarks.sys.mjs new file mode 100644 index 0000000000..3c1396f67d --- /dev/null +++ b/services/sync/modules/engines/bookmarks.sys.mjs @@ -0,0 +1,950 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { + Changeset, + Store, + SyncEngine, + Tracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", + Observers: "resource://services-common/observers.sys.mjs", + PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs", + PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Resource: "resource://services-sync/resource.sys.mjs", + SyncedBookmarksMirror: "resource://gre/modules/SyncedBookmarksMirror.sys.mjs", +}); + +const PLACES_MAINTENANCE_INTERVAL_SECONDS = 4 * 60 * 60; // 4 hours. + +const FOLDER_SORTINDEX = 1000000; + +// Roots that should be deleted from the server, instead of applied locally. +// This matches `AndroidBrowserBookmarksRepositorySession::forbiddenGUID`, +// but allows tags because we don't want to reparent tag folders or tag items +// to "unfiled". +const FORBIDDEN_INCOMING_IDS = ["pinned", "places", "readinglist"]; + +// Items with these parents should be deleted from the server. We allow +// children of the Places root, to avoid orphaning left pane queries and other +// descendants of custom roots. +const FORBIDDEN_INCOMING_PARENT_IDS = ["pinned", "readinglist"]; + +// The tracker ignores changes made by import and restore, to avoid bumping the +// score and triggering syncs during the process, as well as changes made by +// Sync. +ChromeUtils.defineLazyGetter(lazy, "IGNORED_SOURCES", () => [ + lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + lazy.PlacesUtils.bookmarks.SOURCES.IMPORT, + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE, + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP, + lazy.PlacesUtils.bookmarks.SOURCES.SYNC_REPARENT_REMOVED_FOLDER_CHILDREN, +]); + +// The validation telemetry version for the engine. Version 1 is collected +// by `bookmark_validator.js`, and checks value as well as structure +// differences. Version 2 is collected by the engine as part of building the +// remote tree, and checks structure differences only. +const BOOKMARK_VALIDATOR_VERSION = 2; + +// The maximum time that the engine should wait before aborting a bookmark +// merge. +const BOOKMARK_APPLY_TIMEOUT_MS = 5 * 60 * 60 * 1000; // 5 minutes + +// The default frecency value to use when not known. +const FRECENCY_UNKNOWN = -1; + +// Returns the constructor for a bookmark record type. +function getTypeObject(type) { + switch (type) { + case "bookmark": + return Bookmark; + case "query": + return BookmarkQuery; + case "folder": + return BookmarkFolder; + case "livemark": + return Livemark; + case "separator": + return BookmarkSeparator; + case "item": + return PlacesItem; + } + return null; +} + +export function PlacesItem(collection, id, type) { + CryptoWrapper.call(this, collection, id); + this.type = type || "item"; +} + +PlacesItem.prototype = { + async decrypt(keyBundle) { + // Do the normal CryptoWrapper decrypt, but change types before returning + let clear = await CryptoWrapper.prototype.decrypt.call(this, keyBundle); + + // Convert the abstract places item to the actual object type + if (!this.deleted) { + Object.setPrototypeOf(this, this.getTypeObject(this.type).prototype); + } + + return clear; + }, + + getTypeObject: function PlacesItem_getTypeObject(type) { + let recordObj = getTypeObject(type); + if (!recordObj) { + throw new Error("Unknown places item object type: " + type); + } + return recordObj; + }, + + _logName: "Sync.Record.PlacesItem", + + // Converts the record to a Sync bookmark object that can be passed to + // `PlacesSyncUtils.bookmarks.{insert, update}`. + toSyncBookmark() { + let result = { + kind: this.type, + recordId: this.id, + parentRecordId: this.parentid, + }; + let dateAdded = lazy.PlacesSyncUtils.bookmarks.ratchetTimestampBackwards( + this.dateAdded, + +this.modified * 1000 + ); + if (dateAdded > 0) { + result.dateAdded = dateAdded; + } + return result; + }, + + // Populates the record from a Sync bookmark object returned from + // `PlacesSyncUtils.bookmarks.fetch`. + fromSyncBookmark(item) { + this.parentid = item.parentRecordId; + this.parentName = item.parentTitle; + if (item.dateAdded) { + this.dateAdded = item.dateAdded; + } + }, +}; + +Object.setPrototypeOf(PlacesItem.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(PlacesItem, "cleartext", [ + "hasDupe", + "parentid", + "parentName", + "type", + "dateAdded", +]); + +export function Bookmark(collection, id, type) { + PlacesItem.call(this, collection, id, type || "bookmark"); +} + +Bookmark.prototype = { + _logName: "Sync.Record.Bookmark", + + toSyncBookmark() { + let info = PlacesItem.prototype.toSyncBookmark.call(this); + info.title = this.title; + info.url = this.bmkUri; + info.description = this.description; + info.tags = this.tags; + info.keyword = this.keyword; + return info; + }, + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.title = item.title; + this.bmkUri = item.url.href; + this.description = item.description; + this.tags = item.tags; + this.keyword = item.keyword; + }, +}; + +Object.setPrototypeOf(Bookmark.prototype, PlacesItem.prototype); + +Utils.deferGetSet(Bookmark, "cleartext", [ + "title", + "bmkUri", + "description", + "tags", + "keyword", +]); + +export function BookmarkQuery(collection, id) { + Bookmark.call(this, collection, id, "query"); +} + +BookmarkQuery.prototype = { + _logName: "Sync.Record.BookmarkQuery", + + toSyncBookmark() { + let info = Bookmark.prototype.toSyncBookmark.call(this); + info.folder = this.folderName || undefined; // empty string -> undefined + info.query = this.queryId; + return info; + }, + + fromSyncBookmark(item) { + Bookmark.prototype.fromSyncBookmark.call(this, item); + this.folderName = item.folder || undefined; // empty string -> undefined + this.queryId = item.query; + }, +}; + +Object.setPrototypeOf(BookmarkQuery.prototype, Bookmark.prototype); + +Utils.deferGetSet(BookmarkQuery, "cleartext", ["folderName", "queryId"]); + +export function BookmarkFolder(collection, id, type) { + PlacesItem.call(this, collection, id, type || "folder"); +} + +BookmarkFolder.prototype = { + _logName: "Sync.Record.Folder", + + toSyncBookmark() { + let info = PlacesItem.prototype.toSyncBookmark.call(this); + info.description = this.description; + info.title = this.title; + return info; + }, + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.title = item.title; + this.description = item.description; + this.children = item.childRecordIds; + }, +}; + +Object.setPrototypeOf(BookmarkFolder.prototype, PlacesItem.prototype); + +Utils.deferGetSet(BookmarkFolder, "cleartext", [ + "description", + "title", + "children", +]); + +export function Livemark(collection, id) { + BookmarkFolder.call(this, collection, id, "livemark"); +} + +Livemark.prototype = { + _logName: "Sync.Record.Livemark", + + toSyncBookmark() { + let info = BookmarkFolder.prototype.toSyncBookmark.call(this); + info.feed = this.feedUri; + info.site = this.siteUri; + return info; + }, + + fromSyncBookmark(item) { + BookmarkFolder.prototype.fromSyncBookmark.call(this, item); + this.feedUri = item.feed.href; + if (item.site) { + this.siteUri = item.site.href; + } + }, +}; + +Object.setPrototypeOf(Livemark.prototype, BookmarkFolder.prototype); + +Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]); + +export function BookmarkSeparator(collection, id) { + PlacesItem.call(this, collection, id, "separator"); +} + +BookmarkSeparator.prototype = { + _logName: "Sync.Record.Separator", + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.pos = item.index; + }, +}; + +Object.setPrototypeOf(BookmarkSeparator.prototype, PlacesItem.prototype); + +Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos"); + +/** + * The bookmarks engine uses a different store that stages downloaded bookmarks + * in a separate database, instead of writing directly to Places. The buffer + * handles reconciliation, so we stub out `_reconcile`, and wait to pull changes + * until we're ready to upload. + */ +export function BookmarksEngine(service) { + SyncEngine.call(this, "Bookmarks", service); +} + +BookmarksEngine.prototype = { + _recordObj: PlacesItem, + _trackerObj: BookmarksTracker, + _storeObj: BookmarksStore, + version: 2, + // Used to override the engine name in telemetry, so that we can distinguish + // this engine from the old, now removed non-buffered engine. + overrideTelemetryName: "bookmarks-buffered", + + // Needed to ensure we don't miss items when resuming a sync that failed or + // aborted early. + _defaultSort: "oldest", + + syncPriority: 4, + allowSkippedRecord: false, + + async _ensureCurrentSyncID(newSyncID) { + await lazy.PlacesSyncUtils.bookmarks.ensureCurrentSyncId(newSyncID); + let buf = await this._store.ensureOpenMirror(); + await buf.ensureCurrentSyncId(newSyncID); + }, + + async ensureCurrentSyncID(newSyncID) { + let shouldWipeRemote = + await lazy.PlacesSyncUtils.bookmarks.shouldWipeRemote(); + if (!shouldWipeRemote) { + this._log.debug( + "Checking if server sync ID ${newSyncID} matches existing", + { newSyncID } + ); + await this._ensureCurrentSyncID(newSyncID); + return newSyncID; + } + // We didn't take the new sync ID because we need to wipe the server + // and other clients after a restore. Send the command, wipe the + // server, and reset our sync ID to reupload everything. + this._log.debug( + "Ignoring server sync ID ${newSyncID} after restore; " + + "wiping server and resetting sync ID", + { newSyncID } + ); + await this.service.clientsEngine.sendCommand( + "wipeEngine", + [this.name], + null, + { reason: "bookmark-restore" } + ); + let assignedSyncID = await this.resetSyncID(); + return assignedSyncID; + }, + + async getSyncID() { + return lazy.PlacesSyncUtils.bookmarks.getSyncId(); + }, + + async resetSyncID() { + await this._deleteServerCollection(); + return this.resetLocalSyncID(); + }, + + async resetLocalSyncID() { + let newSyncID = await lazy.PlacesSyncUtils.bookmarks.resetSyncId(); + this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID }); + let buf = await this._store.ensureOpenMirror(); + await buf.ensureCurrentSyncId(newSyncID); + return newSyncID; + }, + + async getLastSync() { + let mirror = await this._store.ensureOpenMirror(); + return mirror.getCollectionHighWaterMark(); + }, + + async setLastSync(lastSync) { + let mirror = await this._store.ensureOpenMirror(); + await mirror.setCollectionLastModified(lastSync); + // Update the last sync time in Places so that reverting to the original + // bookmarks engine doesn't download records we've already applied. + await lazy.PlacesSyncUtils.bookmarks.setLastSync(lastSync); + }, + + async _syncStartup() { + await super._syncStartup(); + + try { + // For first syncs, back up the user's bookmarks. + let lastSync = await this.getLastSync(); + if (!lastSync) { + this._log.debug("Bookmarks backup starting"); + await lazy.PlacesBackups.create(null, true); + this._log.debug("Bookmarks backup done"); + } + } catch (ex) { + // Failure to create a backup is somewhat bad, but probably not bad + // enough to prevent syncing of bookmarks - so just log the error and + // continue. + this._log.warn( + "Error while backing up bookmarks, but continuing with sync", + ex + ); + } + }, + + async _sync() { + try { + await super._sync(); + if (this._ranMaintenanceOnLastSync) { + // If the last sync failed, we ran maintenance, and this sync succeeded, + // maintenance likely fixed the issue. + this._ranMaintenanceOnLastSync = false; + this.service.recordTelemetryEvent("maintenance", "fix", "bookmarks"); + } + } catch (ex) { + if ( + lazy.Async.isShutdownException(ex) || + ex.status > 0 || + ex.name == "InterruptedError" + ) { + // Don't run maintenance on shutdown or HTTP errors, or if we aborted + // the sync because the user changed their bookmarks during merging. + throw ex; + } + if (ex.name == "MergeConflictError") { + this._log.warn( + "Bookmark syncing ran into a merge conflict error...will retry later" + ); + return; + } + // Run Places maintenance periodically to try to recover from corruption + // that might have caused the sync to fail. We cap the interval because + // persistent failures likely indicate a problem that won't be fixed by + // running maintenance after every failed sync. + let elapsedSinceMaintenance = + Date.now() / 1000 - + Services.prefs.getIntPref("places.database.lastMaintenance", 0); + if (elapsedSinceMaintenance >= PLACES_MAINTENANCE_INTERVAL_SECONDS) { + this._log.error( + "Bookmark sync failed, ${elapsedSinceMaintenance}s " + + "elapsed since last run; running Places maintenance", + { elapsedSinceMaintenance } + ); + await lazy.PlacesDBUtils.maintenanceOnIdle(); + this._ranMaintenanceOnLastSync = true; + this.service.recordTelemetryEvent("maintenance", "run", "bookmarks"); + } else { + this._ranMaintenanceOnLastSync = false; + } + throw ex; + } + }, + + async _syncFinish() { + await SyncEngine.prototype._syncFinish.call(this); + await lazy.PlacesSyncUtils.bookmarks.ensureMobileQuery(); + }, + + async pullAllChanges() { + return this.pullNewChanges(); + }, + + async trackRemainingChanges() { + let changes = this._modified.changes; + await lazy.PlacesSyncUtils.bookmarks.pushChanges(changes); + }, + + _deleteId(id) { + this._noteDeletedId(id); + }, + + // The bookmarks engine rarely calls this method directly, except in tests or + // when handling a `reset{All, Engine}` command from another client. We + // usually reset local Sync metadata on a sync ID mismatch, which both engines + // override with logic that lives in Places and the mirror. + async _resetClient() { + await super._resetClient(); + await lazy.PlacesSyncUtils.bookmarks.reset(); + let buf = await this._store.ensureOpenMirror(); + await buf.reset(); + }, + + // Cleans up the Places root, reading list items (ignored in bug 762118, + // removed in bug 1155684), and pinned sites. + _shouldDeleteRemotely(incomingItem) { + return ( + FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) || + FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid) + ); + }, + + emptyChangeset() { + return new BookmarksChangeset(); + }, + + async _apply() { + let buf = await this._store.ensureOpenMirror(); + let watchdog = this._newWatchdog(); + watchdog.start(BOOKMARK_APPLY_TIMEOUT_MS); + + try { + let recordsToUpload = await buf.apply({ + remoteTimeSeconds: lazy.Resource.serverTime, + signal: watchdog.signal, + }); + this._modified.replace(recordsToUpload); + } finally { + watchdog.stop(); + if (watchdog.abortReason) { + this._log.warn(`Aborting bookmark merge: ${watchdog.abortReason}`); + } + } + }, + + async _processIncoming(newitems) { + await super._processIncoming(newitems); + await this._apply(); + }, + + async _reconcile(item) { + return true; + }, + + async _createRecord(id) { + let record = await this._doCreateRecord(id); + if (!record.deleted) { + // Set hasDupe on all (non-deleted) records since we don't use it and we + // want to minimize the risk of older clients corrupting records. Note + // that the SyncedBookmarksMirror sets it for all records that it created, + // but we would like to ensure that weakly uploaded records are marked as + // hasDupe as well. + record.hasDupe = true; + } + return record; + }, + + async _doCreateRecord(id) { + let change = this._modified.changes[id]; + if (!change) { + this._log.error( + "Creating record for item ${id} not in strong changeset", + { id } + ); + throw new TypeError("Can't create record for unchanged item"); + } + let record = this._recordFromCleartext(id, change.cleartext); + record.sortindex = await this._store._calculateIndex(record); + return record; + }, + + _recordFromCleartext(id, cleartext) { + let recordObj = getTypeObject(cleartext.type); + if (!recordObj) { + this._log.warn( + "Creating record for item ${id} with unknown type ${type}", + { id, type: cleartext.type } + ); + recordObj = PlacesItem; + } + let record = new recordObj(this.name, id); + record.cleartext = cleartext; + return record; + }, + + async pullChanges() { + return {}; + }, + + /** + * Writes successfully uploaded records back to the mirror, so that the + * mirror matches the server. We update the mirror before updating Places, + * which has implications for interrupted syncs. + * + * 1. Sync interrupted during upload; server doesn't support atomic uploads. + * We'll download and reapply everything that we uploaded before the + * interruption. All locally changed items retain their change counters. + * 2. Sync interrupted during upload; atomic uploads enabled. The server + * discards the batch. All changed local items retain their change + * counters, so the next sync resumes cleanly. + * 3. Sync interrupted during upload; outgoing records can't fit in a single + * batch. We'll download and reapply all records through the most recent + * committed batch. This is a variation of (1). + * 4. Sync interrupted after we update the mirror, but before cleanup. The + * mirror matches the server, but locally changed items retain their change + * counters. Reuploading them on the next sync should be idempotent, though + * unnecessary. If another client makes a conflicting remote change before + * we sync again, we may incorrectly prefer the local state. + * 5. Sync completes successfully. We'll update the mirror, and reset the + * change counters for all items. + */ + async _onRecordsWritten(succeeded, failed, serverModifiedTime) { + let records = []; + for (let id of succeeded) { + let change = this._modified.changes[id]; + if (!change) { + // TODO (Bug 1433178): Write weakly uploaded records back to the mirror. + this._log.info("Uploaded record not in strong changeset", id); + continue; + } + if (!change.synced) { + this._log.info("Record in strong changeset not uploaded", id); + continue; + } + let cleartext = change.cleartext; + if (!cleartext) { + this._log.error( + "Missing Sync record cleartext for ${id} in ${change}", + { id, change } + ); + throw new TypeError("Missing cleartext for uploaded Sync record"); + } + let record = this._recordFromCleartext(id, cleartext); + record.modified = serverModifiedTime; + records.push(record); + } + let buf = await this._store.ensureOpenMirror(); + await buf.store(records, { needsMerge: false }); + }, + + async finalize() { + await super.finalize(); + await this._store.finalize(); + }, +}; + +Object.setPrototypeOf(BookmarksEngine.prototype, SyncEngine.prototype); + +/** + * The bookmarks store delegates to the mirror for staging and applying + * records. Most `Store` methods intentionally remain abstract, so you can't use + * this store to create or update bookmarks in Places. All changes must go + * through the mirror, which takes care of merging and producing a valid tree. + */ +function BookmarksStore(name, engine) { + Store.call(this, name, engine); +} + +BookmarksStore.prototype = { + _openMirrorPromise: null, + + // For tests. + _batchChunkSize: 500, + + // Create a record starting from the weave id (places guid) + async createRecord(id, collection) { + let item = await lazy.PlacesSyncUtils.bookmarks.fetch(id); + if (!item) { + // deleted item + let record = new PlacesItem(collection, id); + record.deleted = true; + return record; + } + + let recordObj = getTypeObject(item.kind); + if (!recordObj) { + this._log.warn("Unknown item type, cannot serialize: " + item.kind); + recordObj = PlacesItem; + } + let record = new recordObj(collection, id); + record.fromSyncBookmark(item); + + record.sortindex = await this._calculateIndex(record); + + return record; + }, + + async _calculateIndex(record) { + // Ensure folders have a very high sort index so they're not synced last. + if (record.type == "folder") { + return FOLDER_SORTINDEX; + } + + // For anything directly under the toolbar, give it a boost of more than an + // unvisited bookmark + let index = 0; + if (record.parentid == "toolbar") { + index += 150; + } + + // Add in the bookmark's frecency if we have something. + if (record.bmkUri != null) { + let frecency = FRECENCY_UNKNOWN; + try { + frecency = await lazy.PlacesSyncUtils.history.fetchURLFrecency( + record.bmkUri + ); + } catch (ex) { + this._log.warn( + `Failed to fetch frecency for ${record.id}; assuming default`, + ex + ); + this._log.trace("Record {id} has invalid URL ${bmkUri}", record); + } + if (frecency != FRECENCY_UNKNOWN) { + index += frecency; + } + } + + return index; + }, + + async wipe() { + // Save a backup before clearing out all bookmarks. + await lazy.PlacesBackups.create(null, true); + await lazy.PlacesSyncUtils.bookmarks.wipe(); + }, + + ensureOpenMirror() { + if (!this._openMirrorPromise) { + this._openMirrorPromise = this._openMirror().catch(err => { + // We may have failed to open the mirror temporarily; for example, if + // the database is locked. Clear the promise so that subsequent + // `ensureOpenMirror` calls can try to open the mirror again. + this._openMirrorPromise = null; + throw err; + }); + } + return this._openMirrorPromise; + }, + + async _openMirror() { + let mirrorPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "bookmarks.sqlite" + ); + await IOUtils.makeDirectory(PathUtils.parent(mirrorPath), { + createAncestors: true, + }); + + return lazy.SyncedBookmarksMirror.open({ + path: mirrorPath, + recordStepTelemetry: (name, took, counts) => { + lazy.Observers.notify( + "weave:engine:sync:step", + { + name, + took, + counts, + }, + this.name + ); + }, + recordValidationTelemetry: (took, checked, problems) => { + lazy.Observers.notify( + "weave:engine:validate:finish", + { + version: BOOKMARK_VALIDATOR_VERSION, + took, + checked, + problems, + }, + this.name + ); + }, + }); + }, + + async applyIncomingBatch(records, countTelemetry) { + let buf = await this.ensureOpenMirror(); + for (let chunk of lazy.PlacesUtils.chunkArray( + records, + this._batchChunkSize + )) { + await buf.store(chunk); + } + // Array of failed records. + return []; + }, + + async applyIncoming(record) { + let buf = await this.ensureOpenMirror(); + await buf.store([record]); + }, + + async finalize() { + if (!this._openMirrorPromise) { + return; + } + let buf = await this._openMirrorPromise; + await buf.finalize(); + }, +}; + +Object.setPrototypeOf(BookmarksStore.prototype, Store.prototype); + +// The bookmarks tracker is a special flower. Instead of listening for changes +// via observer notifications, it queries Places for the set of items that have +// changed since the last sync. Because it's a "pull-based" tracker, it ignores +// all concepts of "add a changed ID." However, it still registers an observer +// to bump the score, so that changed bookmarks are synced immediately. +function BookmarksTracker(name, engine) { + Tracker.call(this, name, engine); +} +BookmarksTracker.prototype = { + onStart() { + this._placesListener = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + lazy.PlacesUtils.observers.addListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-guid-changed", + "bookmark-keyword-changed", + "bookmark-tags-changed", + "bookmark-time-changed", + "bookmark-title-changed", + "bookmark-url-changed", + ], + this._placesListener + ); + Svc.Obs.add("bookmarks-restore-begin", this); + Svc.Obs.add("bookmarks-restore-success", this); + Svc.Obs.add("bookmarks-restore-failed", this); + }, + + onStop() { + lazy.PlacesUtils.observers.removeListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-guid-changed", + "bookmark-keyword-changed", + "bookmark-tags-changed", + "bookmark-time-changed", + "bookmark-title-changed", + "bookmark-url-changed", + ], + this._placesListener + ); + Svc.Obs.remove("bookmarks-restore-begin", this); + Svc.Obs.remove("bookmarks-restore-success", this); + Svc.Obs.remove("bookmarks-restore-failed", this); + }, + + async getChangedIDs() { + return lazy.PlacesSyncUtils.bookmarks.pullChanges(); + }, + + observe(subject, topic, data) { + switch (topic) { + case "bookmarks-restore-begin": + this._log.debug("Ignoring changes from importing bookmarks."); + break; + case "bookmarks-restore-success": + this._log.debug("Tracking all items on successful import."); + + if (data == "json") { + this._log.debug( + "Restore succeeded: wiping server and other clients." + ); + // Trigger an immediate sync. `ensureCurrentSyncID` will notice we + // restored, wipe the server and other clients, reset the sync ID, and + // upload the restored tree. + this.score += SCORE_INCREMENT_XLARGE; + } else { + // "html", "html-initial", or "json-append" + this._log.debug("Import succeeded."); + } + break; + case "bookmarks-restore-failed": + this._log.debug("Tracking all items on failed import."); + break; + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + /* Every add/remove/change will trigger a sync for MULTI_DEVICE */ + _upScore: function BMT__upScore() { + this.score += SCORE_INCREMENT_XLARGE; + }, + + handlePlacesEvents(events) { + for (let event of events) { + switch (event.type) { + case "bookmark-added": + case "bookmark-removed": + case "bookmark-moved": + case "bookmark-keyword-changed": + case "bookmark-tags-changed": + case "bookmark-time-changed": + case "bookmark-title-changed": + case "bookmark-url-changed": + if (lazy.IGNORED_SOURCES.includes(event.source)) { + continue; + } + + this._log.trace(`'${event.type}': ${event.id}`); + this._upScore(); + break; + case "bookmark-guid-changed": + if (event.source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC) { + this._log.warn( + "The source of bookmark-guid-changed event shoud be sync." + ); + continue; + } + + this._log.trace(`'${event.type}': ${event.id}`); + this._upScore(); + break; + case "purge-caches": + this._log.trace("purge-caches"); + this._upScore(); + break; + } + } + }, +}; + +Object.setPrototypeOf(BookmarksTracker.prototype, Tracker.prototype); + +/** + * A changeset that stores extra metadata in a change record for each ID. The + * engine updates this metadata when uploading Sync records, and writes it back + * to Places in `BookmarksEngine#trackRemainingChanges`. + * + * The `synced` property on a change record means its corresponding item has + * been uploaded, and we should pretend it doesn't exist in the changeset. + */ +class BookmarksChangeset extends Changeset { + // Only `_reconcile` calls `getModifiedTimestamp` and `has`, and the engine + // does its own reconciliation. + getModifiedTimestamp(id) { + throw new Error("Don't use timestamps to resolve bookmark conflicts"); + } + + has(id) { + throw new Error("Don't use the changeset to resolve bookmark conflicts"); + } + + delete(id) { + let change = this.changes[id]; + if (change) { + // Mark the change as synced without removing it from the set. We do this + // so that we can update Places in `trackRemainingChanges`. + change.synced = true; + } + } + + ids() { + let results = new Set(); + for (let id in this.changes) { + if (!this.changes[id].synced) { + results.add(id); + } + } + return [...results]; + } +} diff --git a/services/sync/modules/engines/clients.sys.mjs b/services/sync/modules/engines/clients.sys.mjs new file mode 100644 index 0000000000..eda92bd75b --- /dev/null +++ b/services/sync/modules/engines/clients.sys.mjs @@ -0,0 +1,1122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * How does the clients engine work? + * + * - We use 2 files - commands.json and commands-syncing.json. + * + * - At sync upload time, we attempt a rename of commands.json to + * commands-syncing.json, and ignore errors (helps for crash during sync!). + * - We load commands-syncing.json and stash the contents in + * _currentlySyncingCommands which lives for the duration of the upload process. + * - We use _currentlySyncingCommands to build the outgoing records + * - Immediately after successful upload, we delete commands-syncing.json from + * disk (and clear _currentlySyncingCommands). We reconcile our local records + * with what we just wrote in the server, and add failed IDs commands + * back in commands.json + * - Any time we need to "save" a command for future syncs, we load + * commands.json, update it, and write it back out. + */ + +import { Async } from "resource://services-common/async.sys.mjs"; + +import { + DEVICE_TYPE_DESKTOP, + DEVICE_TYPE_MOBILE, + SINGLE_USER_THRESHOLD, + SYNC_API_VERSION, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +import { PREF_ACCOUNT_ROOT } from "resource://gre/modules/FxAccountsCommon.sys.mjs"; + +const CLIENTS_TTL = 15552000; // 180 days +const CLIENTS_TTL_REFRESH = 604800; // 7 days +const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days + +// TTL of the message sent to another device when sending a tab +const NOTIFY_TAB_SENT_TTL_SECS = 1 * 3600; // 1 hour + +// How often we force a refresh of the FxA device list. +const REFRESH_FXA_DEVICE_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours + +// Reasons behind sending collection_changed push notifications. +const COLLECTION_MODIFIED_REASON_SENDTAB = "sendtab"; +const COLLECTION_MODIFIED_REASON_FIRSTSYNC = "firstsync"; + +const SUPPORTED_PROTOCOL_VERSIONS = [SYNC_API_VERSION]; +const LAST_MODIFIED_ON_PROCESS_COMMAND_PREF = + "services.sync.clients.lastModifiedOnProcessCommands"; + +function hasDupeCommand(commands, action) { + if (!commands) { + return false; + } + return commands.some( + other => + other.command == action.command && + Utils.deepEquals(other.args, action.args) + ); +} + +export function ClientsRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +ClientsRec.prototype = { + _logName: "Sync.Record.Clients", + ttl: CLIENTS_TTL, +}; +Object.setPrototypeOf(ClientsRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(ClientsRec, "cleartext", [ + "name", + "type", + "commands", + "version", + "protocols", + "formfactor", + "os", + "appPackage", + "application", + "device", + "fxaDeviceId", +]); + +export function ClientEngine(service) { + SyncEngine.call(this, "Clients", service); + + this.fxAccounts = lazy.fxAccounts; + this.addClientCommandQueue = Async.asyncQueueCaller(this._log); + Utils.defineLazyIDProperty(this, "localID", "services.sync.client.GUID"); +} + +ClientEngine.prototype = { + _storeObj: ClientStore, + _recordObj: ClientsRec, + _trackerObj: ClientsTracker, + allowSkippedRecord: false, + _knownStaleFxADeviceIds: null, + _lastDeviceCounts: null, + _lastFxaDeviceRefresh: 0, + + async initialize() { + // Reset the last sync timestamp on every startup so that we fetch all clients + await this.resetLastSync(); + }, + + // These two properties allow us to avoid replaying the same commands + // continuously if we cannot manage to upload our own record. + _localClientLastModified: 0, + get _lastModifiedOnProcessCommands() { + return Services.prefs.getIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, -1); + }, + + set _lastModifiedOnProcessCommands(value) { + Services.prefs.setIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, value); + }, + + get isFirstSync() { + return !this.lastRecordUpload; + }, + + // Always sync client data as it controls other sync behavior + get enabled() { + return true; + }, + + get lastRecordUpload() { + return Svc.PrefBranch.getIntPref(this.name + ".lastRecordUpload", 0); + }, + set lastRecordUpload(value) { + Svc.PrefBranch.setIntPref( + this.name + ".lastRecordUpload", + Math.floor(value) + ); + }, + + get remoteClients() { + // return all non-stale clients for external consumption. + return Object.values(this._store._remoteClients).filter(v => !v.stale); + }, + + remoteClient(id) { + let client = this._store._remoteClients[id]; + return client && !client.stale ? client : null; + }, + + remoteClientExists(id) { + return !!this.remoteClient(id); + }, + + // Aggregate some stats on the composition of clients on this account + get stats() { + let stats = { + hasMobile: this.localType == DEVICE_TYPE_MOBILE, + names: [this.localName], + numClients: 1, + }; + + for (let id in this._store._remoteClients) { + let { name, type, stale } = this._store._remoteClients[id]; + if (!stale) { + stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE; + stats.names.push(name); + stats.numClients++; + } + } + + return stats; + }, + + /** + * Obtain information about device types. + * + * Returns a Map of device types to integer counts. Guaranteed to include + * "desktop" (which will have at least 1 - this device) and "mobile" (which + * may have zero) counts. It almost certainly will include only these 2. + */ + get deviceTypes() { + let counts = new Map(); + + counts.set(this.localType, 1); // currently this must be DEVICE_TYPE_DESKTOP + counts.set(DEVICE_TYPE_MOBILE, 0); + + for (let id in this._store._remoteClients) { + let record = this._store._remoteClients[id]; + if (record.stale) { + continue; // pretend "stale" records don't exist. + } + let type = record.type; + if (!counts.has(type)) { + counts.set(type, 0); + } + + counts.set(type, counts.get(type) + 1); + } + + return counts; + }, + + get brandName() { + let brand = Services.strings.createBundle( + "chrome://branding/locale/brand.properties" + ); + return brand.GetStringFromName("brandShortName"); + }, + + get localName() { + return this.fxAccounts.device.getLocalName(); + }, + set localName(value) { + this.fxAccounts.device.setLocalName(value); + }, + + get localType() { + return this.fxAccounts.device.getLocalType(); + }, + + getClientName(id) { + if (id == this.localID) { + return this.localName; + } + let client = this._store._remoteClients[id]; + if (!client) { + return ""; + } + // Sometimes the sync clients don't always correctly update the device name + // However FxA always does, so try to pull the name from there first + let fxaDevice = this.fxAccounts.device.recentDeviceList?.find( + device => device.id === client.fxaDeviceId + ); + + // should be very rare, but could happen if we have yet to fetch devices, + // or the client recently disconnected + if (!fxaDevice) { + this._log.warn( + "Couldn't find associated FxA device, falling back to client name" + ); + return client.name; + } + return fxaDevice.name; + }, + + getClientFxaDeviceId(id) { + if (this._store._remoteClients[id]) { + return this._store._remoteClients[id].fxaDeviceId; + } + return null; + }, + + getClientByFxaDeviceId(fxaDeviceId) { + for (let id in this._store._remoteClients) { + let client = this._store._remoteClients[id]; + if (client.stale) { + continue; + } + if (client.fxaDeviceId == fxaDeviceId) { + return client; + } + } + return null; + }, + + getClientType(id) { + const client = this._store._remoteClients[id]; + if (client.type == DEVICE_TYPE_DESKTOP) { + return "desktop"; + } + if (client.formfactor && client.formfactor.includes("tablet")) { + return "tablet"; + } + return "phone"; + }, + + async _readCommands() { + let commands = await Utils.jsonLoad("commands", this); + return commands || {}; + }, + + /** + * Low level function, do not use directly (use _addClientCommand instead). + */ + async _saveCommands(commands) { + try { + await Utils.jsonSave("commands", this, commands); + } catch (error) { + this._log.error("Failed to save JSON outgoing commands", error); + } + }, + + async _prepareCommandsForUpload() { + try { + await Utils.jsonMove("commands", "commands-syncing", this); + } catch (e) { + // Ignore errors + } + let commands = await Utils.jsonLoad("commands-syncing", this); + return commands || {}; + }, + + async _deleteUploadedCommands() { + delete this._currentlySyncingCommands; + try { + await Utils.jsonRemove("commands-syncing", this); + } catch (err) { + this._log.error("Failed to delete syncing-commands file", err); + } + }, + + // Gets commands for a client we are yet to write to the server. Doesn't + // include commands for that client which are already on the server. + // We should rename this! + async getClientCommands(clientId) { + const allCommands = await this._readCommands(); + return allCommands[clientId] || []; + }, + + async removeLocalCommand(command) { + // the implementation of this engine is such that adding a command to + // the local client is how commands are deleted! ¯\_(ツ)_/¯ + await this._addClientCommand(this.localID, command); + }, + + async _addClientCommand(clientId, command) { + this.addClientCommandQueue.enqueueCall(async () => { + try { + const localCommands = await this._readCommands(); + const localClientCommands = localCommands[clientId] || []; + const remoteClient = this._store._remoteClients[clientId]; + let remoteClientCommands = []; + if (remoteClient && remoteClient.commands) { + remoteClientCommands = remoteClient.commands; + } + const clientCommands = localClientCommands.concat(remoteClientCommands); + if (hasDupeCommand(clientCommands, command)) { + return false; + } + localCommands[clientId] = localClientCommands.concat(command); + await this._saveCommands(localCommands); + return true; + } catch (e) { + // Failing to save a command should not "break the queue" of pending operations. + this._log.error(e); + return false; + } + }); + + return this.addClientCommandQueue.promiseCallsComplete(); + }, + + async _removeClientCommands(clientId) { + const allCommands = await this._readCommands(); + delete allCommands[clientId]; + await this._saveCommands(allCommands); + }, + + async updateKnownStaleClients() { + this._log.debug("Updating the known stale clients"); + // _fetchFxADevices side effect updates this._knownStaleFxADeviceIds. + await this._fetchFxADevices(); + let localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + // Process newer records first, so that if we hit a record with a device ID + // we've seen before, we can mark it stale immediately. + let clientList = Object.values(this._store._remoteClients).sort( + (a, b) => b.serverLastModified - a.serverLastModified + ); + let seenDeviceIds = new Set([localFxADeviceId]); + for (let client of clientList) { + // Clients might not have an `fxaDeviceId` if they fail the FxA + // registration process. + if (!client.fxaDeviceId) { + continue; + } + if (this._knownStaleFxADeviceIds.includes(client.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${client.id} - in known stale clients list` + ); + client.stale = true; + } else if (seenDeviceIds.has(client.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${client.id}` + + ` - duplicate device id ${client.fxaDeviceId}` + ); + client.stale = true; + } else { + seenDeviceIds.add(client.fxaDeviceId); + } + } + }, + + async _fetchFxADevices() { + // We only force a refresh periodically to keep the load on the servers + // down, and because we expect FxA to have received a push message in + // most cases when the FxA device list would have changed. For this reason + // we still go ahead and check the stale list even if we didn't force a + // refresh. + let now = this.fxAccounts._internal.now(); // tests mock this .now() impl. + if (now - REFRESH_FXA_DEVICE_INTERVAL_MS > this._lastFxaDeviceRefresh) { + this._lastFxaDeviceRefresh = now; + try { + await this.fxAccounts.device.refreshDeviceList(); + } catch (e) { + this._log.error("Could not refresh the FxA device list", e); + } + } + + // We assume that clients not present in the FxA Device Manager list have been + // disconnected and so are stale + this._log.debug("Refreshing the known stale clients list"); + let localClients = Object.values(this._store._remoteClients) + .filter(client => client.fxaDeviceId) // iOS client records don't have fxaDeviceId + .map(client => client.fxaDeviceId); + const fxaClients = this.fxAccounts.device.recentDeviceList + ? this.fxAccounts.device.recentDeviceList.map(device => device.id) + : []; + this._knownStaleFxADeviceIds = Utils.arraySub(localClients, fxaClients); + }, + + async _syncStartup() { + // Reupload new client record periodically. + if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) { + await this._tracker.addChangedID(this.localID); + } + return SyncEngine.prototype._syncStartup.call(this); + }, + + async _processIncoming() { + // Fetch all records from the server. + await this.resetLastSync(); + this._incomingClients = {}; + try { + await SyncEngine.prototype._processIncoming.call(this); + // Update FxA Device list. + await this._fetchFxADevices(); + // Since clients are synced unconditionally, any records in the local store + // that don't exist on the server must be for disconnected clients. Remove + // them, so that we don't upload records with commands for clients that will + // never see them. We also do this to filter out stale clients from the + // tabs collection, since showing their list of tabs is confusing. + for (let id in this._store._remoteClients) { + if (!this._incomingClients[id]) { + this._log.info(`Removing local state for deleted client ${id}`); + await this._removeRemoteClient(id); + } + } + let localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + // Bug 1264498: Mobile clients don't remove themselves from the clients + // collection when the user disconnects Sync, so we mark as stale clients + // with the same name that haven't synced in over a week. + // (Note we can't simply delete them, or we re-apply them next sync - see + // bug 1287687) + this._localClientLastModified = Math.round( + this._incomingClients[this.localID] + ); + delete this._incomingClients[this.localID]; + let names = new Set([this.localName]); + let seenDeviceIds = new Set([localFxADeviceId]); + let idToLastModifiedList = Object.entries(this._incomingClients).sort( + (a, b) => b[1] - a[1] + ); + for (let [id, serverLastModified] of idToLastModifiedList) { + let record = this._store._remoteClients[id]; + // stash the server last-modified time on the record. + record.serverLastModified = serverLastModified; + if ( + record.fxaDeviceId && + this._knownStaleFxADeviceIds.includes(record.fxaDeviceId) + ) { + this._log.info( + `Hiding stale client ${id} - in known stale clients list` + ); + record.stale = true; + } + if (!names.has(record.name)) { + if (record.fxaDeviceId) { + seenDeviceIds.add(record.fxaDeviceId); + } + names.add(record.name); + continue; + } + let remoteAge = Resource.serverTime - this._incomingClients[id]; + if (remoteAge > STALE_CLIENT_REMOTE_AGE) { + this._log.info(`Hiding stale client ${id} with age ${remoteAge}`); + record.stale = true; + continue; + } + if (record.fxaDeviceId && seenDeviceIds.has(record.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${record.id}` + + ` - duplicate device id ${record.fxaDeviceId}` + ); + record.stale = true; + } else if (record.fxaDeviceId) { + seenDeviceIds.add(record.fxaDeviceId); + } + } + } finally { + this._incomingClients = null; + } + }, + + async _uploadOutgoing() { + this._currentlySyncingCommands = await this._prepareCommandsForUpload(); + const clientWithPendingCommands = Object.keys( + this._currentlySyncingCommands + ); + for (let clientId of clientWithPendingCommands) { + if (this._store._remoteClients[clientId] || this.localID == clientId) { + this._modified.set(clientId, 0); + } + } + let updatedIDs = this._modified.ids(); + await SyncEngine.prototype._uploadOutgoing.call(this); + // Record the response time as the server time for each item we uploaded. + let lastSync = await this.getLastSync(); + for (let id of updatedIDs) { + if (id == this.localID) { + this.lastRecordUpload = lastSync; + } else { + this._store._remoteClients[id].serverLastModified = lastSync; + } + } + }, + + async _onRecordsWritten(succeeded, failed) { + // Reconcile the status of the local records with what we just wrote on the + // server + for (let id of succeeded) { + const commandChanges = this._currentlySyncingCommands[id]; + if (id == this.localID) { + if (this.isFirstSync) { + this._log.info( + "Uploaded our client record for the first time, notifying other clients." + ); + this._notifyClientRecordUploaded(); + } + if (this.localCommands) { + this.localCommands = this.localCommands.filter( + command => !hasDupeCommand(commandChanges, command) + ); + } + } else { + const clientRecord = this._store._remoteClients[id]; + if (!commandChanges || !clientRecord) { + // should be impossible, else we wouldn't have been writing it. + this._log.warn( + "No command/No record changes for a client we uploaded" + ); + continue; + } + // fixup the client record, so our copy of _remoteClients matches what we uploaded. + this._store._remoteClients[id] = await this._store.createRecord(id); + // we could do better and pass the reference to the record we just uploaded, + // but this will do for now + } + } + + // Re-add failed commands + for (let id of failed) { + const commandChanges = this._currentlySyncingCommands[id]; + if (!commandChanges) { + continue; + } + await this._addClientCommand(id, commandChanges); + } + + await this._deleteUploadedCommands(); + + // Notify other devices that their own client collection changed + const idsToNotify = succeeded.reduce((acc, id) => { + if (id == this.localID) { + return acc; + } + const fxaDeviceId = this.getClientFxaDeviceId(id); + return fxaDeviceId ? acc.concat(fxaDeviceId) : acc; + }, []); + if (idsToNotify.length) { + this._notifyOtherClientsModified(idsToNotify); + } + }, + + _notifyOtherClientsModified(ids) { + // We are not waiting on this promise on purpose. + this._notifyCollectionChanged( + ids, + NOTIFY_TAB_SENT_TTL_SECS, + COLLECTION_MODIFIED_REASON_SENDTAB + ); + }, + + _notifyClientRecordUploaded() { + // We are not waiting on this promise on purpose. + this._notifyCollectionChanged( + null, + 0, + COLLECTION_MODIFIED_REASON_FIRSTSYNC + ); + }, + + /** + * @param {?string[]} ids FxA Client IDs to notify. null means everyone else. + * @param {number} ttl TTL of the push notification. + * @param {string} reason Reason for sending this push notification. + */ + async _notifyCollectionChanged(ids, ttl, reason) { + const message = { + version: 1, + command: "sync:collection_changed", + data: { + collections: ["clients"], + reason, + }, + }; + let excludedIds = null; + if (!ids) { + const localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + excludedIds = [localFxADeviceId]; + } + try { + await this.fxAccounts.notifyDevices(ids, excludedIds, message, ttl); + } catch (e) { + this._log.error("Could not notify of changes in the collection", e); + } + }, + + async _syncFinish() { + // Record histograms for our device types, and also write them to a pref + // so non-histogram telemetry (eg, UITelemetry) and the sync scheduler + // has easy access to them, and so they are accurate even before we've + // successfully synced the first time after startup. + let deviceTypeCounts = this.deviceTypes; + for (let [deviceType, count] of deviceTypeCounts) { + let hid; + let prefName = this.name + ".devices."; + switch (deviceType) { + case DEVICE_TYPE_DESKTOP: + hid = "WEAVE_DEVICE_COUNT_DESKTOP"; + prefName += "desktop"; + break; + case DEVICE_TYPE_MOBILE: + hid = "WEAVE_DEVICE_COUNT_MOBILE"; + prefName += "mobile"; + break; + default: + this._log.warn( + `Unexpected deviceType "${deviceType}" recording device telemetry.` + ); + continue; + } + Services.telemetry.getHistogramById(hid).add(count); + // Optimization: only write the pref if it changed since our last sync. + if ( + this._lastDeviceCounts == null || + this._lastDeviceCounts.get(prefName) != count + ) { + Svc.PrefBranch.setIntPref(prefName, count); + } + } + this._lastDeviceCounts = deviceTypeCounts; + return SyncEngine.prototype._syncFinish.call(this); + }, + + async _reconcile(item) { + // Every incoming record is reconciled, so we use this to track the + // contents of the collection on the server. + this._incomingClients[item.id] = item.modified; + + if (!(await this._store.itemExists(item.id))) { + return true; + } + // Clients are synced unconditionally, so we'll always have new records. + // Unfortunately, this will cause the scheduler to use the immediate sync + // interval for the multi-device case, instead of the active interval. We + // work around this by updating the record during reconciliation, and + // returning false to indicate that the record doesn't need to be applied + // later. + await this._store.update(item); + return false; + }, + + // Treat reset the same as wiping for locally cached clients + async _resetClient() { + await this._wipeClient(); + }, + + async _wipeClient() { + await SyncEngine.prototype._resetClient.call(this); + this._knownStaleFxADeviceIds = null; + delete this.localCommands; + await this._store.wipe(); + try { + await Utils.jsonRemove("commands", this); + } catch (err) { + this._log.warn("Could not delete commands.json", err); + } + try { + await Utils.jsonRemove("commands-syncing", this); + } catch (err) { + this._log.warn("Could not delete commands-syncing.json", err); + } + }, + + async removeClientData() { + let res = this.service.resource(this.engineURL + "/" + this.localID); + await res.delete(); + }, + + // Override the default behavior to delete bad records from the server. + async handleHMACMismatch(item, mayRetry) { + this._log.debug("Handling HMAC mismatch for " + item.id); + + let base = await SyncEngine.prototype.handleHMACMismatch.call( + this, + item, + mayRetry + ); + if (base != SyncEngine.kRecoveryStrategy.error) { + return base; + } + + // It's a bad client record. Save it to be deleted at the end of the sync. + this._log.debug("Bad client record detected. Scheduling for deletion."); + await this._deleteId(item.id); + + // Neither try again nor error; we're going to delete it. + return SyncEngine.kRecoveryStrategy.ignore; + }, + + /** + * A hash of valid commands that the client knows about. The key is a command + * and the value is a hash containing information about the command such as + * number of arguments, description, and importance (lower importance numbers + * indicate higher importance. + */ + _commands: { + resetAll: { + args: 0, + importance: 0, + desc: "Clear temporary local data for all engines", + }, + resetEngine: { + args: 1, + importance: 0, + desc: "Clear temporary local data for engine", + }, + wipeEngine: { + args: 1, + importance: 0, + desc: "Delete all client data for engine", + }, + logout: { args: 0, importance: 0, desc: "Log out client" }, + }, + + /** + * Sends a command+args pair to a specific client. + * + * @param command Command string + * @param args Array of arguments/data for command + * @param clientId Client to send command to + */ + async _sendCommandToClient(command, args, clientId, telemetryExtra) { + this._log.trace("Sending " + command + " to " + clientId); + + let client = this._store._remoteClients[clientId]; + if (!client) { + throw new Error("Unknown remote client ID: '" + clientId + "'."); + } + if (client.stale) { + throw new Error("Stale remote client ID: '" + clientId + "'."); + } + + let action = { + command, + args, + // We send the flowID to the other client so *it* can report it in its + // telemetry - we record it in ours below. + flowID: telemetryExtra.flowID, + }; + + if (await this._addClientCommand(clientId, action)) { + this._log.trace(`Client ${clientId} got a new action`, [command, args]); + await this._tracker.addChangedID(clientId); + try { + telemetryExtra.deviceID = + this.service.identity.hashedDeviceID(clientId); + } catch (_) {} + + this.service.recordTelemetryEvent( + "sendcommand", + command, + undefined, + telemetryExtra + ); + } else { + this._log.trace(`Client ${clientId} got a duplicate action`, [ + command, + args, + ]); + } + }, + + /** + * Check if the local client has any remote commands and perform them. + * + * @return false to abort sync + */ + async processIncomingCommands() { + return this._notify("clients:process-commands", "", async function () { + if ( + !this.localCommands || + (this._lastModifiedOnProcessCommands == this._localClientLastModified && + !this.ignoreLastModifiedOnProcessCommands) + ) { + return true; + } + this._lastModifiedOnProcessCommands = this._localClientLastModified; + + const clearedCommands = await this._readCommands()[this.localID]; + const commands = this.localCommands.filter( + command => !hasDupeCommand(clearedCommands, command) + ); + let didRemoveCommand = false; + // Process each command in order. + for (let rawCommand of commands) { + let shouldRemoveCommand = true; // most commands are auto-removed. + let { command, args, flowID } = rawCommand; + this._log.debug("Processing command " + command, args); + + this.service.recordTelemetryEvent( + "processcommand", + command, + undefined, + { flowID } + ); + + let engines = [args[0]]; + switch (command) { + case "resetAll": + engines = null; + // Fallthrough + case "resetEngine": + await this.service.resetClient(engines); + break; + case "wipeEngine": + await this.service.wipeClient(engines); + break; + case "logout": + this.service.logout(); + return false; + default: + this._log.warn("Received an unknown command: " + command); + break; + } + // Add the command to the "cleared" commands list + if (shouldRemoveCommand) { + await this.removeLocalCommand(rawCommand); + didRemoveCommand = true; + } + } + if (didRemoveCommand) { + await this._tracker.addChangedID(this.localID); + } + + return true; + })(); + }, + + /** + * Validates and sends a command to a client or all clients. + * + * Calling this does not actually sync the command data to the server. If the + * client already has the command/args pair, it won't receive a duplicate + * command. + * This method is async since it writes the command to a file. + * + * @param command + * Command to invoke on remote clients + * @param args + * Array of arguments to give to the command + * @param clientId + * Client ID to send command to. If undefined, send to all remote + * clients. + * @param flowID + * A unique identifier used to track success for this operation across + * devices. + */ + async sendCommand(command, args, clientId = null, telemetryExtra = {}) { + let commandData = this._commands[command]; + // Don't send commands that we don't know about. + if (!commandData) { + this._log.error("Unknown command to send: " + command); + return; + } else if (!args || args.length != commandData.args) { + // Don't send a command with the wrong number of arguments. + this._log.error( + "Expected " + + commandData.args + + " args for '" + + command + + "', but got " + + args + ); + return; + } + + // We allocate a "flowID" here, so it is used for each client. + telemetryExtra = Object.assign({}, telemetryExtra); // don't clobber the caller's object + if (!telemetryExtra.flowID) { + telemetryExtra.flowID = Utils.makeGUID(); + } + + if (clientId) { + await this._sendCommandToClient(command, args, clientId, telemetryExtra); + } else { + for (let [id, record] of Object.entries(this._store._remoteClients)) { + if (!record.stale) { + await this._sendCommandToClient(command, args, id, telemetryExtra); + } + } + } + }, + + async _removeRemoteClient(id) { + delete this._store._remoteClients[id]; + await this._tracker.removeChangedID(id); + await this._removeClientCommands(id); + this._modified.delete(id); + }, +}; +Object.setPrototypeOf(ClientEngine.prototype, SyncEngine.prototype); + +function ClientStore(name, engine) { + Store.call(this, name, engine); +} +ClientStore.prototype = { + _remoteClients: {}, + + async create(record) { + await this.update(record); + }, + + async update(record) { + if (record.id == this.engine.localID) { + // Only grab commands from the server; local name/type always wins + this.engine.localCommands = record.commands; + } else { + this._remoteClients[record.id] = record.cleartext; + } + }, + + async createRecord(id, collection) { + let record = new ClientsRec(collection, id); + + const commandsChanges = this.engine._currentlySyncingCommands + ? this.engine._currentlySyncingCommands[id] + : []; + + // Package the individual components into a record for the local client + if (id == this.engine.localID) { + try { + record.fxaDeviceId = await this.engine.fxAccounts.device.getLocalId(); + } catch (error) { + this._log.warn("failed to get fxa device id", error); + } + record.name = this.engine.localName; + record.type = this.engine.localType; + record.version = Services.appinfo.version; + record.protocols = SUPPORTED_PROTOCOL_VERSIONS; + + // Substract the commands we recorded that we've already executed + if ( + commandsChanges && + commandsChanges.length && + this.engine.localCommands && + this.engine.localCommands.length + ) { + record.commands = this.engine.localCommands.filter( + command => !hasDupeCommand(commandsChanges, command) + ); + } + + // Optional fields. + record.os = Services.appinfo.OS; // "Darwin" + record.appPackage = Services.appinfo.ID; + record.application = this.engine.brandName; // "Nightly" + + // We can't compute these yet. + // record.device = ""; // Bug 1100723 + // record.formfactor = ""; // Bug 1100722 + } else { + record.cleartext = Object.assign({}, this._remoteClients[id]); + delete record.cleartext.serverLastModified; // serverLastModified is a local only attribute. + + // Add the commands we have to send + if (commandsChanges && commandsChanges.length) { + const recordCommands = record.cleartext.commands || []; + const newCommands = commandsChanges.filter( + command => !hasDupeCommand(recordCommands, command) + ); + record.cleartext.commands = recordCommands.concat(newCommands); + } + + if (record.cleartext.stale) { + // It's almost certainly a logic error for us to upload a record we + // consider stale, so make log noise, but still remove the flag. + this._log.error( + `Preparing to upload record ${id} that we consider stale` + ); + delete record.cleartext.stale; + } + } + if (record.commands) { + const maxPayloadSize = + this.engine.service.getMemcacheMaxRecordPayloadSize(); + let origOrder = new Map(record.commands.map((c, i) => [c, i])); + // we sort first by priority, and second by age (indicated by order in the + // original list) + let commands = record.commands.slice().sort((a, b) => { + let infoA = this.engine._commands[a.command]; + let infoB = this.engine._commands[b.command]; + // Treat unknown command types as highest priority, to allow us to add + // high priority commands in the future without worrying about clients + // removing them on each-other unnecessarially. + let importA = infoA ? infoA.importance : 0; + let importB = infoB ? infoB.importance : 0; + // Higher importantance numbers indicate that we care less, so they + // go to the end of the list where they'll be popped off. + let importDelta = importA - importB; + if (importDelta != 0) { + return importDelta; + } + let origIdxA = origOrder.get(a); + let origIdxB = origOrder.get(b); + // Within equivalent priorities, we put older entries near the end + // of the list, so that they are removed first. + return origIdxB - origIdxA; + }); + let truncatedCommands = Utils.tryFitItems(commands, maxPayloadSize); + if (truncatedCommands.length != record.commands.length) { + this._log.warn( + `Removing commands from client ${id} (from ${record.commands.length} to ${truncatedCommands.length})` + ); + // Restore original order. + record.commands = truncatedCommands.sort( + (a, b) => origOrder.get(a) - origOrder.get(b) + ); + } + } + return record; + }, + + async itemExists(id) { + return id in (await this.getAllIDs()); + }, + + async getAllIDs() { + let ids = {}; + ids[this.engine.localID] = true; + for (let id in this._remoteClients) { + ids[id] = true; + } + return ids; + }, + + async wipe() { + this._remoteClients = {}; + }, +}; +Object.setPrototypeOf(ClientStore.prototype, Store.prototype); + +function ClientsTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +ClientsTracker.prototype = { + _enabled: false, + + onStart() { + Svc.Obs.add("fxaccounts:new_device_id", this.asyncObserver); + Services.prefs.addObserver( + PREF_ACCOUNT_ROOT + "device.name", + this.asyncObserver + ); + }, + onStop() { + Services.prefs.removeObserver( + PREF_ACCOUNT_ROOT + "device.name", + this.asyncObserver + ); + Svc.Obs.remove("fxaccounts:new_device_id", this.asyncObserver); + }, + + async observe(subject, topic, data) { + switch (topic) { + case "nsPref:changed": + this._log.debug("client.name preference changed"); + // Fallthrough intended. + case "fxaccounts:new_device_id": + await this.addChangedID(this.engine.localID); + this.score += SINGLE_USER_THRESHOLD + 1; // ALWAYS SYNC NOW. + break; + } + }, +}; +Object.setPrototypeOf(ClientsTracker.prototype, LegacyTracker.prototype); diff --git a/services/sync/modules/engines/extension-storage.sys.mjs b/services/sync/modules/engines/extension-storage.sys.mjs new file mode 100644 index 0000000000..d2671978c8 --- /dev/null +++ b/services/sync/modules/engines/extension-storage.sys.mjs @@ -0,0 +1,308 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { + BridgedEngine, + BridgeWrapperXPCOM, + LogAdapter, +} from "resource://services-sync/bridged_engine.sys.mjs"; +import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + MULTI_DEVICE_THRESHOLD: "resource://services-sync/constants.sys.mjs", + Observers: "resource://services-common/observers.sys.mjs", + SCORE_INCREMENT_MEDIUM: "resource://services-sync/constants.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", + + extensionStorageSyncKinto: + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "StorageSyncService", + "@mozilla.org/extensions/storage/sync;1", + "nsIInterfaceRequestor" +); + +const PREF_FORCE_ENABLE = "engine.extension-storage.force"; + +// A helper to indicate whether extension-storage is enabled - it's based on +// the "addons" pref. The same logic is shared between both engine impls. +function getEngineEnabled() { + // By default, we sync extension storage if we sync addons. This + // lets us simplify the UX since users probably don't consider + // "extension preferences" a separate category of syncing. + // However, we also respect engine.extension-storage.force, which + // can be set to true or false, if a power user wants to customize + // the behavior despite the lack of UI. + if ( + lazy.Svc.PrefBranch.getPrefType(PREF_FORCE_ENABLE) != + Ci.nsIPrefBranch.PREF_INVALID + ) { + return lazy.Svc.PrefBranch.getBoolPref(PREF_FORCE_ENABLE); + } + return lazy.Svc.PrefBranch.getBoolPref("engine.addons", false); +} + +function setEngineEnabled(enabled) { + // This will be called by the engine manager when declined on another device. + // Things will go a bit pear-shaped if the engine manager tries to end up + // with 'addons' and 'extension-storage' in different states - however, this + // *can* happen given we support the `engine.extension-storage.force` + // preference. So if that pref exists, we set it to this value. If that pref + // doesn't exist, we just ignore it and hope that the 'addons' engine is also + // going to be set to the same state. + if ( + lazy.Svc.PrefBranch.getPrefType(PREF_FORCE_ENABLE) != + Ci.nsIPrefBranch.PREF_INVALID + ) { + lazy.Svc.PrefBranch.setBoolPref(PREF_FORCE_ENABLE, enabled); + } +} + +// A "bridged engine" to our webext-storage component. +export function ExtensionStorageEngineBridge(service) { + this.component = lazy.StorageSyncService.getInterface( + Ci.mozIBridgedSyncEngine + ); + BridgedEngine.call(this, "Extension-Storage", service); + this._bridge = new BridgeWrapperXPCOM(this.component); + + let app_services_logger = Cc["@mozilla.org/appservices/logger;1"].getService( + Ci.mozIAppServicesLogger + ); + let logger_target = "app-services:webext_storage:sync"; + app_services_logger.register(logger_target, new LogAdapter(this._log)); +} + +ExtensionStorageEngineBridge.prototype = { + syncPriority: 10, + + // Used to override the engine name in telemetry, so that we can distinguish . + overrideTelemetryName: "rust-webext-storage", + + _notifyPendingChanges() { + return new Promise(resolve => { + this.component + .QueryInterface(Ci.mozISyncedExtensionStorageArea) + .fetchPendingSyncChanges({ + QueryInterface: ChromeUtils.generateQI([ + "mozIExtensionStorageListener", + "mozIExtensionStorageCallback", + ]), + onChanged: (extId, json) => { + try { + lazy.extensionStorageSync.notifyListeners( + extId, + JSON.parse(json) + ); + } catch (ex) { + this._log.warn( + `Error notifying change listeners for ${extId}`, + ex + ); + } + }, + handleSuccess: resolve, + handleError: (code, message) => { + this._log.warn( + "Error fetching pending synced changes", + message, + code + ); + resolve(); + }, + }); + }); + }, + + _takeMigrationInfo() { + return new Promise((resolve, reject) => { + this.component + .QueryInterface(Ci.mozIExtensionStorageArea) + .takeMigrationInfo({ + QueryInterface: ChromeUtils.generateQI([ + "mozIExtensionStorageCallback", + ]), + handleSuccess: result => { + resolve(result ? JSON.parse(result) : null); + }, + handleError: (code, message) => { + this._log.warn("Error fetching migration info", message, code); + // `takeMigrationInfo` doesn't actually perform the migration, + // just reads (and clears) any data stored in the DB from the + // previous migration. + // + // Any errors here are very likely occurring a good while + // after the migration ran, so we just warn and pretend + // nothing was there. + resolve(null); + }, + }); + }); + }, + + async _syncStartup() { + let result = await super._syncStartup(); + let info = await this._takeMigrationInfo(); + if (info) { + lazy.Observers.notify( + "weave:telemetry:migration", + info, + "webext-storage" + ); + } + return result; + }, + + async _processIncoming() { + await super._processIncoming(); + try { + await this._notifyPendingChanges(); + } catch (ex) { + // Failing to notify `storage.onChanged` observers is bad, but shouldn't + // interrupt syncing. + this._log.warn("Error notifying about synced changes", ex); + } + }, + + get enabled() { + return getEngineEnabled(); + }, + set enabled(enabled) { + setEngineEnabled(enabled); + }, +}; +Object.setPrototypeOf( + ExtensionStorageEngineBridge.prototype, + BridgedEngine.prototype +); + +/** + ***************************************************************************** + * + * Deprecated support for Kinto + * + ***************************************************************************** + */ + +/** + * The Engine that manages syncing for the web extension "storage" + * API, and in particular ext.storage.sync. + * + * ext.storage.sync is implemented using Kinto, so it has mechanisms + * for syncing that we do not need to integrate in the Firefox Sync + * framework, so this is something of a stub. + */ +export function ExtensionStorageEngineKinto(service) { + SyncEngine.call(this, "Extension-Storage", service); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_skipPercentageChance", + "services.sync.extension-storage.skipPercentageChance", + 0 + ); +} + +ExtensionStorageEngineKinto.prototype = { + _trackerObj: ExtensionStorageTracker, + // we don't need these since we implement our own sync logic + _storeObj: undefined, + _recordObj: undefined, + + syncPriority: 10, + allowSkippedRecord: false, + + async _sync() { + return lazy.extensionStorageSyncKinto.syncAll(); + }, + + get enabled() { + return getEngineEnabled(); + }, + // We only need the enabled setter for the edge-case where info/collections + // has `extension-storage` - which could happen if the pref to flip the new + // engine on was once set but no longer is. + set enabled(enabled) { + setEngineEnabled(enabled); + }, + + _wipeClient() { + return lazy.extensionStorageSyncKinto.clearAll(); + }, + + shouldSkipSync(syncReason) { + if (syncReason == "user" || syncReason == "startup") { + this._log.info( + `Not skipping extension storage sync: reason == ${syncReason}` + ); + // Always sync if a user clicks the button, or if we're starting up. + return false; + } + // Ensure this wouldn't cause a resync... + if (this._tracker.score >= lazy.MULTI_DEVICE_THRESHOLD) { + this._log.info( + "Not skipping extension storage sync: Would trigger resync anyway" + ); + return false; + } + + let probability = this._skipPercentageChance / 100.0; + // Math.random() returns a value in the interval [0, 1), so `>` is correct: + // if `probability` is 1 skip every time, and if it's 0, never skip. + let shouldSkip = probability > Math.random(); + + this._log.info( + `Skipping extension-storage sync with a chance of ${probability}: ${shouldSkip}` + ); + return shouldSkip; + }, +}; +Object.setPrototypeOf( + ExtensionStorageEngineKinto.prototype, + SyncEngine.prototype +); + +function ExtensionStorageTracker(name, engine) { + Tracker.call(this, name, engine); + this._ignoreAll = false; +} +ExtensionStorageTracker.prototype = { + get ignoreAll() { + return this._ignoreAll; + }, + + set ignoreAll(value) { + this._ignoreAll = value; + }, + + onStart() { + lazy.Svc.Obs.add("ext.storage.sync-changed", this.asyncObserver); + }, + + onStop() { + lazy.Svc.Obs.remove("ext.storage.sync-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + + if (topic !== "ext.storage.sync-changed") { + return; + } + + // Single adds, removes and changes are not so important on their + // own, so let's just increment score a bit. + this.score += lazy.SCORE_INCREMENT_MEDIUM; + }, +}; +Object.setPrototypeOf(ExtensionStorageTracker.prototype, Tracker.prototype); diff --git a/services/sync/modules/engines/forms.sys.mjs b/services/sync/modules/engines/forms.sys.mjs new file mode 100644 index 0000000000..3516327659 --- /dev/null +++ b/services/sync/modules/engines/forms.sys.mjs @@ -0,0 +1,298 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; + +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { SCORE_INCREMENT_MEDIUM } from "resource://services-sync/constants.sys.mjs"; +import { + CollectionProblemData, + CollectionValidator, +} from "resource://services-sync/collection_validator.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + FormHistory: "resource://gre/modules/FormHistory.sys.mjs", +}); + +const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds. + +export function FormRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +FormRec.prototype = { + _logName: "Sync.Record.Form", + ttl: FORMS_TTL, +}; +Object.setPrototypeOf(FormRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]); + +var FormWrapper = { + _log: Log.repository.getLogger("Sync.Engine.Forms"), + + _getEntryCols: ["fieldname", "value"], + _guidCols: ["guid"], + + _search(terms, searchData) { + return lazy.FormHistory.search(terms, searchData); + }, + + async _update(changes) { + if (!lazy.FormHistory.enabled) { + return; // update isn't going to do anything. + } + await lazy.FormHistory.update(changes).catch(console.error); + }, + + async getEntry(guid) { + let results = await this._search(this._getEntryCols, { guid }); + if (!results.length) { + return null; + } + return { name: results[0].fieldname, value: results[0].value }; + }, + + async getGUID(name, value) { + // Query for the provided entry. + let query = { fieldname: name, value }; + let results = await this._search(this._guidCols, query); + return results.length ? results[0].guid : null; + }, + + async hasGUID(guid) { + // We could probably use a count function here, but search exists... + let results = await this._search(this._guidCols, { guid }); + return !!results.length; + }, + + async replaceGUID(oldGUID, newGUID) { + let changes = { + op: "update", + guid: oldGUID, + newGuid: newGUID, + }; + await this._update(changes); + }, +}; + +export function FormEngine(service) { + SyncEngine.call(this, "Forms", service); +} + +FormEngine.prototype = { + _storeObj: FormStore, + _trackerObj: FormTracker, + _recordObj: FormRec, + + syncPriority: 6, + + get prefName() { + return "history"; + }, + + async _findDupe(item) { + return FormWrapper.getGUID(item.name, item.value); + }, +}; +Object.setPrototypeOf(FormEngine.prototype, SyncEngine.prototype); + +function FormStore(name, engine) { + Store.call(this, name, engine); +} +FormStore.prototype = { + async _processChange(change) { + // If this._changes is defined, then we are applying a batch, so we + // can defer it. + if (this._changes) { + this._changes.push(change); + return; + } + + // Otherwise we must handle the change right now. + await FormWrapper._update(change); + }, + + async applyIncomingBatch(records, countTelemetry) { + Async.checkAppReady(); + // We collect all the changes to be made then apply them all at once. + this._changes = []; + let failures = await Store.prototype.applyIncomingBatch.call( + this, + records, + countTelemetry + ); + if (this._changes.length) { + await FormWrapper._update(this._changes); + } + delete this._changes; + return failures; + }, + + async getAllIDs() { + let results = await FormWrapper._search(["guid"], []); + let guids = {}; + for (let result of results) { + guids[result.guid] = true; + } + return guids; + }, + + async changeItemID(oldID, newID) { + await FormWrapper.replaceGUID(oldID, newID); + }, + + async itemExists(id) { + return FormWrapper.hasGUID(id); + }, + + async createRecord(id, collection) { + let record = new FormRec(collection, id); + let entry = await FormWrapper.getEntry(id); + if (entry != null) { + record.name = entry.name; + record.value = entry.value; + } else { + record.deleted = true; + } + return record; + }, + + async create(record) { + this._log.trace("Adding form record for " + record.name); + let change = { + op: "add", + guid: record.id, + fieldname: record.name, + value: record.value, + }; + await this._processChange(change); + }, + + async remove(record) { + this._log.trace("Removing form record: " + record.id); + let change = { + op: "remove", + guid: record.id, + }; + await this._processChange(change); + }, + + async update(record) { + this._log.trace("Ignoring form record update request!"); + }, + + async wipe() { + let change = { + op: "remove", + }; + await FormWrapper._update(change); + }, +}; +Object.setPrototypeOf(FormStore.prototype, Store.prototype); + +function FormTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +FormTracker.prototype = { + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + onStart() { + Svc.Obs.add("satchel-storage-changed", this.asyncObserver); + }, + + onStop() { + Svc.Obs.remove("satchel-storage-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + switch (topic) { + case "satchel-storage-changed": + if (data == "formhistory-add" || data == "formhistory-remove") { + let guid = subject.QueryInterface(Ci.nsISupportsString).toString(); + await this.trackEntry(guid); + } + break; + } + }, + + async trackEntry(guid) { + const added = await this.addChangedID(guid); + if (added) { + this.score += SCORE_INCREMENT_MEDIUM; + } + }, +}; +Object.setPrototypeOf(FormTracker.prototype, LegacyTracker.prototype); + +class FormsProblemData extends CollectionProblemData { + getSummary() { + // We don't support syncing deleted form data, so "clientMissing" isn't a problem + return super.getSummary().filter(entry => entry.name !== "clientMissing"); + } +} + +export class FormValidator extends CollectionValidator { + constructor() { + super("forms", "id", ["name", "value"]); + this.ignoresMissingClients = true; + } + + emptyProblemData() { + return new FormsProblemData(); + } + + async getClientItems() { + return FormWrapper._search(["guid", "fieldname", "value"], {}); + } + + normalizeClientItem(item) { + return { + id: item.guid, + guid: item.guid, + name: item.fieldname, + fieldname: item.fieldname, + value: item.value, + original: item, + }; + } + + async normalizeServerItem(item) { + let res = Object.assign( + { + guid: item.id, + fieldname: item.name, + original: item, + }, + item + ); + // Missing `name` or `value` causes the getGUID call to throw + if (item.name !== undefined && item.value !== undefined) { + let guid = await FormWrapper.getGUID(item.name, item.value); + if (guid) { + res.guid = guid; + res.id = guid; + res.duped = true; + } + } + + return res; + } +} diff --git a/services/sync/modules/engines/history.sys.mjs b/services/sync/modules/engines/history.sys.mjs new file mode 100644 index 0000000000..44014e4d9e --- /dev/null +++ b/services/sync/modules/engines/history.sys.mjs @@ -0,0 +1,654 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const HISTORY_TTL = 5184000; // 60 days in milliseconds +const THIRTY_DAYS_IN_MS = 2592000000; // 30 days in milliseconds +// Sync may bring new fields from other clients, not yet understood by our engine. +// Unknown fields outside these fields are aggregated into 'unknownFields' and +// safely synced to prevent data loss. +const VALID_HISTORY_FIELDS = ["id", "title", "histUri", "visits"]; +const VALID_VISIT_FIELDS = ["date", "type", "transition"]; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { + MAX_HISTORY_DOWNLOAD, + MAX_HISTORY_UPLOAD, + SCORE_INCREMENT_SMALL, + SCORE_INCREMENT_XLARGE, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +export function HistoryRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +HistoryRec.prototype = { + _logName: "Sync.Record.History", + ttl: HISTORY_TTL, +}; +Object.setPrototypeOf(HistoryRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]); + +export function HistoryEngine(service) { + SyncEngine.call(this, "History", service); +} + +HistoryEngine.prototype = { + _recordObj: HistoryRec, + _storeObj: HistoryStore, + _trackerObj: HistoryTracker, + downloadLimit: MAX_HISTORY_DOWNLOAD, + + syncPriority: 7, + + async getSyncID() { + return lazy.PlacesSyncUtils.history.getSyncId(); + }, + + async ensureCurrentSyncID(newSyncID) { + this._log.debug( + "Checking if server sync ID ${newSyncID} matches existing", + { newSyncID } + ); + await lazy.PlacesSyncUtils.history.ensureCurrentSyncId(newSyncID); + return newSyncID; + }, + + async resetSyncID() { + // First, delete the collection on the server. It's fine if we're + // interrupted here: on the next sync, we'll detect that our old sync ID is + // now stale, and start over as a first sync. + await this._deleteServerCollection(); + // Then, reset our local sync ID. + return this.resetLocalSyncID(); + }, + + async resetLocalSyncID() { + let newSyncID = await lazy.PlacesSyncUtils.history.resetSyncId(); + this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID }); + return newSyncID; + }, + + async getLastSync() { + let lastSync = await lazy.PlacesSyncUtils.history.getLastSync(); + return lastSync; + }, + + async setLastSync(lastSync) { + await lazy.PlacesSyncUtils.history.setLastSync(lastSync); + }, + + shouldSyncURL(url) { + return !url.startsWith("file:"); + }, + + async pullNewChanges() { + const changedIDs = await this._tracker.getChangedIDs(); + let modifiedGUIDs = Object.keys(changedIDs); + if (!modifiedGUIDs.length) { + return {}; + } + + let guidsToRemove = + await lazy.PlacesSyncUtils.history.determineNonSyncableGuids( + modifiedGUIDs + ); + await this._tracker.removeChangedID(...guidsToRemove); + return changedIDs; + }, + + async _resetClient() { + await super._resetClient(); + await lazy.PlacesSyncUtils.history.reset(); + }, +}; +Object.setPrototypeOf(HistoryEngine.prototype, SyncEngine.prototype); + +function HistoryStore(name, engine) { + Store.call(this, name, engine); +} + +HistoryStore.prototype = { + // We try and only update this many visits at one time. + MAX_VISITS_PER_INSERT: 500, + + // Some helper functions to handle GUIDs + async setGUID(uri, guid) { + if (!guid) { + guid = Utils.makeGUID(); + } + + try { + await lazy.PlacesSyncUtils.history.changeGuid(uri, guid); + } catch (e) { + this._log.error("Error setting GUID ${guid} for URI ${uri}", guid, uri); + } + + return guid; + }, + + async GUIDForUri(uri, create) { + // Use the existing GUID if it exists + let guid; + try { + guid = await lazy.PlacesSyncUtils.history.fetchGuidForURL(uri); + } catch (e) { + this._log.error("Error fetching GUID for URL ${uri}", uri); + } + + // If the URI has an existing GUID, return it. + if (guid) { + return guid; + } + + // If the URI doesn't have a GUID and we were indicated to create one. + if (create) { + return this.setGUID(uri); + } + + // If the URI doesn't have a GUID and we didn't create one for it. + return null; + }, + + async changeItemID(oldID, newID) { + let info = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(oldID); + if (!info) { + throw new Error(`Can't change ID for nonexistent history entry ${oldID}`); + } + this.setGUID(info.url, newID); + }, + + async getAllIDs() { + let urls = await lazy.PlacesSyncUtils.history.getAllURLs({ + since: new Date(Date.now() - THIRTY_DAYS_IN_MS), + limit: MAX_HISTORY_UPLOAD, + }); + + let urlsByGUID = {}; + for (let url of urls) { + if (!this.engine.shouldSyncURL(url)) { + continue; + } + let guid = await this.GUIDForUri(url, true); + urlsByGUID[guid] = url; + } + return urlsByGUID; + }, + + async applyIncomingBatch(records, countTelemetry) { + // Convert incoming records to mozIPlaceInfo objects which are applied as + // either history additions or removals. + let failed = []; + let toAdd = []; + let toRemove = []; + let pageGuidsWithUnknownFields = new Map(); + let visitTimesWithUnknownFields = new Map(); + await Async.yieldingForEach(records, async record => { + if (record.deleted) { + toRemove.push(record); + } else { + try { + let pageInfo = await this._recordToPlaceInfo(record); + if (pageInfo) { + toAdd.push(pageInfo); + + // Pull any unknown fields that may have come from other clients + let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields( + record.cleartext, + VALID_HISTORY_FIELDS + ); + if (unknownFields) { + pageGuidsWithUnknownFields.set(pageInfo.guid, { unknownFields }); + } + + // Visits themselves could also contain unknown fields + for (const visit of pageInfo.visits) { + let unknownVisitFields = + lazy.PlacesSyncUtils.extractUnknownFields( + visit, + VALID_VISIT_FIELDS + ); + if (unknownVisitFields) { + // Visits don't have an id at the time of sync so we'll need + // to use the time instead until it's inserted in the DB + visitTimesWithUnknownFields.set(visit.date.getTime(), { + unknownVisitFields, + }); + } + } + } + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to create a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + } + }); + if (toAdd.length || toRemove.length) { + if (toRemove.length) { + // PlacesUtils.history.remove takes an array of visits to remove, + // but the error semantics are tricky - a single "bad" entry will cause + // an exception before anything is removed. So we do remove them one at + // a time. + await Async.yieldingForEach(toRemove, async record => { + try { + await this.remove(record); + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to delete a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + }); + } + for (let chunk of this._generateChunks(toAdd)) { + // Per bug 1415560, we ignore any exceptions returned by insertMany + // as they are likely to be spurious. We do supply an onError handler + // and log the exceptions seen there as they are likely to be + // informative, but we still never abort the sync based on them. + let unknownFieldsToInsert = []; + try { + await lazy.PlacesUtils.history.insertMany( + chunk, + result => { + const placeToUpdate = pageGuidsWithUnknownFields.get(result.guid); + // Extract the placeId from this result so we can add the unknownFields + // to the proper table + if (placeToUpdate) { + unknownFieldsToInsert.push({ + placeId: result.placeId, + unknownFields: placeToUpdate.unknownFields, + }); + } + // same for visits + result.visits.forEach(visit => { + let visitToUpdate = visitTimesWithUnknownFields.get( + visit.date.getTime() + ); + if (visitToUpdate) { + unknownFieldsToInsert.push({ + visitId: visit.visitId, + unknownFields: visitToUpdate.unknownVisitFields, + }); + } + }); + }, + failedVisit => { + this._log.info( + "Failed to insert a history record", + failedVisit.guid + ); + this._log.trace("The record that failed", failedVisit); + failed.push(failedVisit.guid); + } + ); + } catch (ex) { + this._log.info("Failed to insert history records", ex); + countTelemetry.addIncomingFailedReason(ex.message); + } + + // All the top level places or visits that had unknown fields are sent + // to be added to the appropiate tables + await lazy.PlacesSyncUtils.history.updateUnknownFieldsBatch( + unknownFieldsToInsert + ); + } + } + + return failed; + }, + + /** + * Returns a generator that splits records into sanely sized chunks suitable + * for passing to places to prevent places doing bad things at shutdown. + */ + *_generateChunks(records) { + // We chunk based on the number of *visits* inside each record. However, + // we do not split a single record into multiple records, because at some + // time in the future, we intend to ensure these records are ordered by + // lastModified, and advance the engine's timestamp as we process them, + // meaning we can resume exactly where we left off next sync - although + // currently that's not done, so we will retry the entire batch next sync + // if interrupted. + // ie, this means that if a single record has more than MAX_VISITS_PER_INSERT + // visits, we will call insertMany() with exactly 1 record, but with + // more than MAX_VISITS_PER_INSERT visits. + let curIndex = 0; + this._log.debug(`adding ${records.length} records to history`); + while (curIndex < records.length) { + Async.checkAppReady(); // may throw if we are shutting down. + let toAdd = []; // what we are going to insert. + let count = 0; // a counter which tells us when toAdd is full. + do { + let record = records[curIndex]; + curIndex += 1; + toAdd.push(record); + count += record.visits.length; + } while ( + curIndex < records.length && + count + records[curIndex].visits.length <= this.MAX_VISITS_PER_INSERT + ); + this._log.trace(`adding ${toAdd.length} items in this chunk`); + yield toAdd; + } + }, + + /* An internal helper to determine if we can add an entry to places. + Exists primarily so tests can override it. + */ + _canAddURI(uri) { + return lazy.PlacesUtils.history.canAddURI(uri); + }, + + /** + * Converts a Sync history record to a mozIPlaceInfo. + * + * Throws if an invalid record is encountered (invalid URI, etc.), + * returns a new PageInfo object if the record is to be applied, null + * otherwise (no visits to add, etc.), + */ + async _recordToPlaceInfo(record) { + // Sort out invalid URIs and ones Places just simply doesn't want. + record.url = lazy.PlacesUtils.normalizeToURLOrGUID(record.histUri); + record.uri = CommonUtils.makeURI(record.histUri); + + if (!Utils.checkGUID(record.id)) { + this._log.warn("Encountered record with invalid GUID: " + record.id); + return null; + } + record.guid = record.id; + + if ( + !this._canAddURI(record.uri) || + !this.engine.shouldSyncURL(record.uri.spec) + ) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": can't add this URI." + ); + return null; + } + + // We dupe visits by date and type. So an incoming visit that has + // the same timestamp and type as a local one won't get applied. + // To avoid creating new objects, we rewrite the query result so we + // can simply check for containment below. + let curVisitsAsArray = []; + let curVisits = new Set(); + try { + curVisitsAsArray = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + } + let oldestAllowed = + lazy.PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP; + if (curVisitsAsArray.length == 20) { + let oldestVisit = curVisitsAsArray[curVisitsAsArray.length - 1]; + oldestAllowed = lazy.PlacesSyncUtils.history.clampVisitDate( + lazy.PlacesUtils.toDate(oldestVisit.date).getTime() + ); + } + + let i, k; + for (i = 0; i < curVisitsAsArray.length; i++) { + // Same logic as used in the loop below to generate visitKey. + let { date, type } = curVisitsAsArray[i]; + let dateObj = lazy.PlacesUtils.toDate(date); + let millis = lazy.PlacesSyncUtils.history + .clampVisitDate(dateObj) + .getTime(); + curVisits.add(`${millis},${type}`); + } + + // Walk through the visits, make sure we have sound data, and eliminate + // dupes. The latter is done by rewriting the array in-place. + for (i = 0, k = 0; i < record.visits.length; i++) { + let visit = (record.visits[k] = record.visits[i]); + + if ( + !visit.date || + typeof visit.date != "number" || + !Number.isInteger(visit.date) + ) { + this._log.warn( + "Encountered record with invalid visit date: " + visit.date + ); + continue; + } + + if ( + !visit.type || + !Object.values(lazy.PlacesUtils.history.TRANSITIONS).includes( + visit.type + ) + ) { + this._log.warn( + "Encountered record with invalid visit type: " + + visit.type + + "; ignoring." + ); + continue; + } + + // Dates need to be integers. Future and far past dates are clamped to the + // current date and earliest sensible date, respectively. + let originalVisitDate = lazy.PlacesUtils.toDate(Math.round(visit.date)); + visit.date = + lazy.PlacesSyncUtils.history.clampVisitDate(originalVisitDate); + + if (visit.date.getTime() < oldestAllowed) { + // Visit is older than the oldest visit we have, and we have so many + // visits for this uri that we hit our limit when inserting. + continue; + } + let visitKey = `${visit.date.getTime()},${visit.type}`; + if (curVisits.has(visitKey)) { + // Visit is a dupe, don't increment 'k' so the element will be + // overwritten. + continue; + } + + // Note the visit key, so that we don't add duplicate visits with + // clamped timestamps. + curVisits.add(visitKey); + + visit.transition = visit.type; + k += 1; + } + record.visits.length = k; // truncate array + + // No update if there aren't any visits to apply. + // History wants at least one visit. + // In any case, the only thing we could change would be the title + // and that shouldn't change without a visit. + if (!record.visits.length) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": no visits to add." + ); + return null; + } + + // PageInfo is validated using validateItemProperties which does a shallow + // copy of the properties. Since record uses getters some of the properties + // are not copied over. Thus we create and return a new object. + let pageInfo = { + title: record.title, + url: record.url, + guid: record.guid, + visits: record.visits, + }; + + return pageInfo; + }, + + async remove(record) { + this._log.trace("Removing page: " + record.id); + let removed = await lazy.PlacesUtils.history.remove(record.id); + if (removed) { + this._log.trace("Removed page: " + record.id); + } else { + this._log.debug("Page already removed: " + record.id); + } + }, + + async itemExists(id) { + return !!(await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id)); + }, + + async createRecord(id, collection) { + let foo = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id); + let record = new HistoryRec(collection, id); + if (foo) { + record.histUri = foo.url; + record.title = foo.title; + record.sortindex = foo.frecency; + + // If we had any unknown fields, ensure we put it back on the + // top-level record + if (foo.unknownFields) { + let unknownFields = JSON.parse(foo.unknownFields); + Object.assign(record.cleartext, unknownFields); + } + + try { + record.visits = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + record.visits = []; + } + } else { + record.deleted = true; + } + + return record; + }, + + async wipe() { + return lazy.PlacesSyncUtils.history.wipe(); + }, +}; +Object.setPrototypeOf(HistoryStore.prototype, Store.prototype); + +function HistoryTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +HistoryTracker.prototype = { + onStart() { + this._log.info("Adding Places observer."); + this._placesObserver = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + PlacesObservers.addListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + }, + + onStop() { + this._log.info("Removing Places observer."); + if (this._placesObserver) { + PlacesObservers.removeListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + handlePlacesEvents(aEvents) { + this.asyncObserver.enqueueCall(() => this._handlePlacesEvents(aEvents)); + }, + + async _handlePlacesEvents(aEvents) { + if (this.ignoreAll) { + this._log.trace( + "ignoreAll: ignoring visits [" + + aEvents.map(v => v.guid).join(",") + + "]" + ); + return; + } + for (let event of aEvents) { + switch (event.type) { + case "page-visited": { + this._log.trace("'page-visited': " + event.url); + if ( + this.engine.shouldSyncURL(event.url) && + (await this.addChangedID(event.pageGuid)) + ) { + this.score += SCORE_INCREMENT_SMALL; + } + break; + } + case "history-cleared": { + this._log.trace("history-cleared"); + // Note that we're going to trigger a sync, but none of the cleared + // pages are tracked, so the deletions will not be propagated. + // See Bug 578694. + this.score += SCORE_INCREMENT_XLARGE; + break; + } + case "page-removed": { + if (event.reason === PlacesVisitRemoved.REASON_EXPIRED) { + return; + } + + this._log.trace( + "page-removed: " + event.url + ", reason " + event.reason + ); + const added = await this.addChangedID(event.pageGuid); + if (added) { + this.score += event.isRemovedFromStore + ? SCORE_INCREMENT_XLARGE + : SCORE_INCREMENT_SMALL; + } + break; + } + } + } + }, +}; +Object.setPrototypeOf(HistoryTracker.prototype, LegacyTracker.prototype); diff --git a/services/sync/modules/engines/passwords.sys.mjs b/services/sync/modules/engines/passwords.sys.mjs new file mode 100644 index 0000000000..8dea5664be --- /dev/null +++ b/services/sync/modules/engines/passwords.sys.mjs @@ -0,0 +1,546 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs"; +import { + Changeset, + Store, + SyncEngine, + Tracker, +} from "resource://services-sync/engines.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +// These are valid fields the server could have for a logins record +// we mainly use this to detect if there are any unknownFields and +// store (but don't process) those fields to roundtrip them back +const VALID_LOGIN_FIELDS = [ + "id", + "displayOrigin", + "formSubmitURL", + "formActionOrigin", + "httpRealm", + "hostname", + "origin", + "password", + "passwordField", + "timeCreated", + "timeLastUsed", + "timePasswordChanged", + "timesUsed", + "username", + "usernameField", + "everSynced", + "syncCounter", + "unknownFields", +]; + +import { LoginManagerStorage } from "resource://passwordmgr/passwordstorage.sys.mjs"; + +// Sync and many tests rely on having an time that is rounded to the nearest +// 100th of a second otherwise tests can fail intermittently. +function roundTimeForSync(time) { + return Math.round(time / 10) / 100; +} + +export function LoginRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +LoginRec.prototype = { + _logName: "Sync.Record.Login", + + cleartextToString() { + let o = Object.assign({}, this.cleartext); + if (o.password) { + o.password = "X".repeat(o.password.length); + } + return JSON.stringify(o); + }, +}; +Object.setPrototypeOf(LoginRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(LoginRec, "cleartext", [ + "hostname", + "formSubmitURL", + "httpRealm", + "username", + "password", + "usernameField", + "passwordField", + "timeCreated", + "timePasswordChanged", +]); + +export function PasswordEngine(service) { + SyncEngine.call(this, "Passwords", service); +} + +PasswordEngine.prototype = { + _storeObj: PasswordStore, + _trackerObj: PasswordTracker, + _recordObj: LoginRec, + + syncPriority: 2, + + emptyChangeset() { + return new PasswordsChangeset(); + }, + + async ensureCurrentSyncID(newSyncID) { + return Services.logins.ensureCurrentSyncID(newSyncID); + }, + + async getLastSync() { + let legacyValue = await super.getLastSync(); + if (legacyValue) { + await this.setLastSync(legacyValue); + Svc.PrefBranch.clearUserPref(this.name + ".lastSync"); + this._log.debug( + `migrated timestamp of ${legacyValue} to the logins store` + ); + return legacyValue; + } + return this._store.storage.getLastSync(); + }, + + async setLastSync(timestamp) { + await this._store.storage.setLastSync(timestamp); + }, + + // Testing function to emulate that a login has been synced. + async markSynced(guid) { + this._store.storage.resetSyncCounter(guid, 0); + }, + + async pullAllChanges() { + return this._getChangedIDs(true); + }, + + async getChangedIDs() { + return this._getChangedIDs(false); + }, + + async _getChangedIDs(getAll) { + let changes = {}; + + let logins = await this._store.storage.getAllLogins(true); + for (let login of logins) { + if (getAll || login.syncCounter > 0) { + if (Utils.getSyncCredentialsHosts().has(login.origin)) { + continue; + } + + changes[login.guid] = { + counter: login.syncCounter, // record the initial counter value + modified: roundTimeForSync(login.timePasswordChanged), + deleted: this._store.storage.loginIsDeleted(login.guid), + }; + } + } + + return changes; + }, + + async trackRemainingChanges() { + // Reset the syncCounter on the items that were changed. + for (let [guid, { counter, synced }] of Object.entries( + this._modified.changes + )) { + if (synced) { + this._store.storage.resetSyncCounter(guid, counter); + } + } + }, + + async _findDupe(item) { + let login = this._store._nsLoginInfoFromRecord(item); + if (!login) { + return null; + } + + let logins = await this._store.storage.searchLoginsAsync({ + origin: login.origin, + formActionOrigin: login.formActionOrigin, + httpRealm: login.httpRealm, + }); + + // Look for existing logins that match the origin, but ignore the password. + for (let local of logins) { + if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) { + return local.guid; + } + } + + return null; + }, + + _deleteId(id) { + this._noteDeletedId(id); + }, + + getValidator() { + return new PasswordValidator(); + }, +}; +Object.setPrototypeOf(PasswordEngine.prototype, SyncEngine.prototype); + +function PasswordStore(name, engine) { + Store.call(this, name, engine); + this._nsLoginInfo = new Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" + ); + this.storage = LoginManagerStorage.create(); +} +PasswordStore.prototype = { + _newPropertyBag() { + return Cc["@mozilla.org/hash-property-bag;1"].createInstance( + Ci.nsIWritablePropertyBag2 + ); + }, + + // Returns an stringified object of any fields not "known" by this client + // mainly used to to prevent data loss for other clients by roundtripping + // these fields without processing them + _processUnknownFields(record) { + let unknownFields = {}; + let keys = Object.keys(record); + keys + .filter(key => !VALID_LOGIN_FIELDS.includes(key)) + .forEach(key => { + unknownFields[key] = record[key]; + }); + // If we found some unknown fields, we stringify it to be able + // to properly encrypt it for roundtripping since we can't know if + // it contained sensitive fields or not + if (Object.keys(unknownFields).length) { + return JSON.stringify(unknownFields); + } + return null; + }, + + /** + * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo). + */ + _nsLoginInfoFromRecord(record) { + function nullUndefined(x) { + return x == undefined ? null : x; + } + + function stringifyNullUndefined(x) { + return x == undefined || x == null ? "" : x; + } + + if (record.formSubmitURL && record.httpRealm) { + this._log.warn( + "Record " + + record.id + + " has both formSubmitURL and httpRealm. Skipping." + ); + return null; + } + + // Passing in "undefined" results in an empty string, which later + // counts as a value. Explicitly `|| null` these fields according to JS + // truthiness. Records with empty strings or null will be unmolested. + let info = new this._nsLoginInfo( + record.hostname, + nullUndefined(record.formSubmitURL), + nullUndefined(record.httpRealm), + stringifyNullUndefined(record.username), + record.password, + record.usernameField, + record.passwordField + ); + + info.QueryInterface(Ci.nsILoginMetaInfo); + info.guid = record.id; + if (record.timeCreated && !isNaN(new Date(record.timeCreated).getTime())) { + info.timeCreated = record.timeCreated; + } + if ( + record.timePasswordChanged && + !isNaN(new Date(record.timePasswordChanged).getTime()) + ) { + info.timePasswordChanged = record.timePasswordChanged; + } + + // Check the record if there are any unknown fields from other clients + // that we want to roundtrip during sync to prevent data loss + let unknownFields = this._processUnknownFields(record.cleartext); + if (unknownFields) { + info.unknownFields = unknownFields; + } + return info; + }, + + async _getLoginFromGUID(guid) { + let logins = await this.storage.searchLoginsAsync({ guid }, true); + if (logins.length) { + this._log.trace(logins.length + " items matching " + guid + " found."); + return logins[0]; + } + + this._log.trace("No items matching " + guid + " found. Ignoring"); + return null; + }, + + async applyIncoming(record) { + if (record.deleted) { + // Need to supply the sourceSync flag. + await this.remove(record, { sourceSync: true }); + return; + } + + await super.applyIncoming(record); + }, + + async getAllIDs() { + let items = {}; + let logins = await this.storage.getAllLogins(true); + + for (let i = 0; i < logins.length; i++) { + // Skip over Weave password/passphrase entries. + let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo); + if (Utils.getSyncCredentialsHosts().has(metaInfo.origin)) { + continue; + } + + items[metaInfo.guid] = metaInfo; + } + + return items; + }, + + async changeItemID(oldID, newID) { + this._log.trace("Changing item ID: " + oldID + " to " + newID); + + if (!(await this.itemExists(oldID))) { + this._log.trace("Can't change item ID: item doesn't exist"); + return; + } + if (await this._getLoginFromGUID(newID)) { + this._log.trace("Can't change item ID: new ID already in use"); + return; + } + + let prop = this._newPropertyBag(); + prop.setPropertyAsAUTF8String("guid", newID); + + let oldLogin = await this._getLoginFromGUID(oldID); + this.storage.modifyLogin(oldLogin, prop, true); + }, + + async itemExists(id) { + let login = await this._getLoginFromGUID(id); + return login && !this.storage.loginIsDeleted(id); + }, + + async createRecord(id, collection) { + let record = new LoginRec(collection, id); + let login = await this._getLoginFromGUID(id); + + if (!login || this.storage.loginIsDeleted(id)) { + record.deleted = true; + return record; + } + + record.hostname = login.origin; + record.formSubmitURL = login.formActionOrigin; + record.httpRealm = login.httpRealm; + record.username = login.username; + record.password = login.password; + record.usernameField = login.usernameField; + record.passwordField = login.passwordField; + + // Optional fields. + login.QueryInterface(Ci.nsILoginMetaInfo); + record.timeCreated = login.timeCreated; + record.timePasswordChanged = login.timePasswordChanged; + + // put the unknown fields back to the top-level record + // during upload + if (login.unknownFields) { + let unknownFields = JSON.parse(login.unknownFields); + if (unknownFields) { + Object.keys(unknownFields).forEach(key => { + // We have to manually add it to the cleartext since that's + // what gets processed during upload + record.cleartext[key] = unknownFields[key]; + }); + } + } + + return record; + }, + + async create(record) { + let login = this._nsLoginInfoFromRecord(record); + if (!login) { + return; + } + + login.everSynced = true; + + this._log.trace("Adding login for " + record.hostname); + this._log.trace( + "httpRealm: " + + JSON.stringify(login.httpRealm) + + "; " + + "formSubmitURL: " + + JSON.stringify(login.formActionOrigin) + ); + await Services.logins.addLoginAsync(login); + }, + + async remove(record, { sourceSync = false } = {}) { + this._log.trace("Removing login " + record.id); + + let loginItem = await this._getLoginFromGUID(record.id); + if (!loginItem) { + this._log.trace("Asked to remove record that doesn't exist, ignoring"); + return; + } + + this.storage.removeLogin(loginItem, sourceSync); + }, + + async update(record) { + let loginItem = await this._getLoginFromGUID(record.id); + if (!loginItem || this.storage.loginIsDeleted(record.id)) { + this._log.trace("Skipping update for unknown item: " + record.hostname); + return; + } + + this._log.trace("Updating " + record.hostname); + let newinfo = this._nsLoginInfoFromRecord(record); + if (!newinfo) { + return; + } + + loginItem.everSynced = true; + + this.storage.modifyLogin(loginItem, newinfo, true); + }, + + async wipe() { + this.storage.removeAllUserFacingLogins(true); + }, +}; +Object.setPrototypeOf(PasswordStore.prototype, Store.prototype); + +function PasswordTracker(name, engine) { + Tracker.call(this, name, engine); +} +PasswordTracker.prototype = { + onStart() { + Svc.Obs.add("passwordmgr-storage-changed", this.asyncObserver); + }, + + onStop() { + Svc.Obs.remove("passwordmgr-storage-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + + switch (data) { + case "modifyLogin": + // The syncCounter should have been incremented only for + // those items that need to be sycned. + if ( + subject.QueryInterface(Ci.nsIArrayExtensions).GetElementAt(1) + .syncCounter > 0 + ) { + this.score += SCORE_INCREMENT_XLARGE; + } + break; + + case "addLogin": + case "removeLogin": + case "importLogins": + this.score += SCORE_INCREMENT_XLARGE; + break; + + case "removeAllLogins": + this.score += + SCORE_INCREMENT_XLARGE * + (subject.QueryInterface(Ci.nsIArrayExtensions).Count() + 1); + break; + } + }, +}; +Object.setPrototypeOf(PasswordTracker.prototype, Tracker.prototype); + +export class PasswordValidator extends CollectionValidator { + constructor() { + super("passwords", "id", [ + "hostname", + "formSubmitURL", + "httpRealm", + "password", + "passwordField", + "username", + "usernameField", + ]); + } + + async getClientItems() { + let logins = await Services.logins.getAllLogins(); + let syncHosts = Utils.getSyncCredentialsHosts(); + let result = logins + .map(l => l.QueryInterface(Ci.nsILoginMetaInfo)) + .filter(l => !syncHosts.has(l.origin)); + return Promise.resolve(result); + } + + normalizeClientItem(item) { + return { + id: item.guid, + guid: item.guid, + hostname: item.hostname, + formSubmitURL: item.formSubmitURL, + httpRealm: item.httpRealm, + password: item.password, + passwordField: item.passwordField, + username: item.username, + usernameField: item.usernameField, + original: item, + }; + } + + async normalizeServerItem(item) { + return Object.assign({ guid: item.id }, item); + } +} + +export class PasswordsChangeset extends Changeset { + getModifiedTimestamp(id) { + return this.changes[id].modified; + } + + has(id) { + let change = this.changes[id]; + if (change) { + return !change.synced; + } + return false; + } + + delete(id) { + let change = this.changes[id]; + if (change) { + // Mark the change as synced without removing it from the set. + // This allows the sync counter to be reset when sync is complete + // within trackRemainingChanges. + change.synced = true; + } + } +} diff --git a/services/sync/modules/engines/prefs.sys.mjs b/services/sync/modules/engines/prefs.sys.mjs new file mode 100644 index 0000000000..f29a9e7b59 --- /dev/null +++ b/services/sync/modules/engines/prefs.sys.mjs @@ -0,0 +1,503 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Prefs which start with this prefix are our "control" prefs - they indicate +// which preferences should be synced. +const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; + +// Prefs which have a default value are usually not synced - however, if the +// preference exists under this prefix and the value is: +// * `true`, then we do sync default values. +// * `false`, then as soon as we ever sync a non-default value out, or sync +// any value in, then we toggle the value to `true`. +// +// We never explicitly set this pref back to false, so it's one-shot. +// Some preferences which are known to have a different default value on +// different platforms have this preference with a default value of `false`, +// so they don't sync until one device changes to the non-default value, then +// that value forever syncs, even if it gets reset back to the default. +// Note that preferences handled this way *must also* have the "normal" +// control pref set. +// A possible future enhancement would be to sync these prefs so that +// other distributions can flag them if they change the default, but that +// doesn't seem worthwhile until we can be confident they'd actually create +// this special control pref at the same time they flip the default. +const PREF_SYNC_SEEN_PREFIX = "services.sync.prefs.sync-seen."; + +import { + Store, + SyncEngine, + Tracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "PREFS_GUID", () => + CommonUtils.encodeBase64URL(Services.appinfo.ID) +); + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", +}); + +// In bug 1538015, we decided that it isn't always safe to allow all "incoming" +// preferences to be applied locally. So we introduced another preference to control +// this for backward compatibility. We removed that capability in bug 1854698, but in the +// interests of working well between different versions of Firefox, we still forever +// want to prevent this preference from syncing. +// This was the name of the "control" pref. +const PREF_SYNC_PREFS_ARBITRARY = + "services.sync.prefs.dangerously_allow_arbitrary"; + +// Check for a local control pref or PREF_SYNC_PREFS_ARBITRARY +function isAllowedPrefName(prefName) { + if (prefName == PREF_SYNC_PREFS_ARBITRARY) { + return false; // never allow this. + } + // The pref must already have a control pref set, although it doesn't matter + // here whether that value is true or false. We can't use prefHasUserValue + // here because we also want to check prefs still with default values. + try { + Services.prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + prefName); + // pref exists! + return true; + } catch (_) { + return false; + } +} + +export function PrefRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +PrefRec.prototype = { + _logName: "Sync.Record.Pref", +}; +Object.setPrototypeOf(PrefRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(PrefRec, "cleartext", ["value"]); + +export function PrefsEngine(service) { + SyncEngine.call(this, "Prefs", service); +} + +PrefsEngine.prototype = { + _storeObj: PrefStore, + _trackerObj: PrefTracker, + _recordObj: PrefRec, + version: 2, + + syncPriority: 1, + allowSkippedRecord: false, + + async getChangedIDs() { + // No need for a proper timestamp (no conflict resolution needed). + let changedIDs = {}; + if (this._tracker.modified) { + changedIDs[lazy.PREFS_GUID] = 0; + } + return changedIDs; + }, + + async _wipeClient() { + await SyncEngine.prototype._wipeClient.call(this); + this.justWiped = true; + }, + + async _reconcile(item) { + // Apply the incoming item if we don't care about the local data + if (this.justWiped) { + this.justWiped = false; + return true; + } + return SyncEngine.prototype._reconcile.call(this, item); + }, + + async _uploadOutgoing() { + try { + await SyncEngine.prototype._uploadOutgoing.call(this); + } finally { + this._store._incomingPrefs = null; + } + }, + + async trackRemainingChanges() { + if (this._modified.count() > 0) { + this._tracker.modified = true; + } + }, +}; +Object.setPrototypeOf(PrefsEngine.prototype, SyncEngine.prototype); + +// We don't use services.sync.engine.tabs.filteredSchemes since it includes +// about: pages and the like, which we want to be syncable in preferences. +// Blob, moz-extension, data and file uris are never safe to sync, +// so we limit our check to those. +const UNSYNCABLE_URL_REGEXP = /^(moz-extension|blob|data|file):/i; +function isUnsyncableURLPref(prefName) { + if (Services.prefs.getPrefType(prefName) != Ci.nsIPrefBranch.PREF_STRING) { + return false; + } + const prefValue = Services.prefs.getStringPref(prefName, ""); + return UNSYNCABLE_URL_REGEXP.test(prefValue); +} + +function PrefStore(name, engine) { + Store.call(this, name, engine); + Svc.Obs.add( + "profile-before-change", + function () { + this.__prefs = null; + }, + this + ); +} +PrefStore.prototype = { + __prefs: null, + // used just for logging so we can work out why we chose to re-upload + _incomingPrefs: null, + get _prefs() { + if (!this.__prefs) { + this.__prefs = Services.prefs.getBranch(""); + } + return this.__prefs; + }, + + _getSyncPrefs() { + let syncPrefs = Services.prefs + .getBranch(PREF_SYNC_PREFS_PREFIX) + .getChildList("") + .filter(pref => isAllowedPrefName(pref) && !isUnsyncableURLPref(pref)); + // Also sync preferences that determine which prefs get synced. + let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref); + return controlPrefs.concat(syncPrefs); + }, + + _isSynced(pref) { + if (pref.startsWith(PREF_SYNC_PREFS_PREFIX)) { + // this is an incoming control pref, which is ignored if there's not already + // a local control pref for the preference. + let controlledPref = pref.slice(PREF_SYNC_PREFS_PREFIX.length); + return isAllowedPrefName(controlledPref); + } + + // This is the pref itself - it must be both allowed, and have a control + // pref which is true. + if (!this._prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + pref, false)) { + return false; + } + return isAllowedPrefName(pref); + }, + + // Given a preference name, returns either a string, bool, number or null. + _getPrefValue(pref) { + switch (this._prefs.getPrefType(pref)) { + case Ci.nsIPrefBranch.PREF_STRING: + return this._prefs.getStringPref(pref); + case Ci.nsIPrefBranch.PREF_INT: + return this._prefs.getIntPref(pref); + case Ci.nsIPrefBranch.PREF_BOOL: + return this._prefs.getBoolPref(pref); + // case Ci.nsIPrefBranch.PREF_INVALID: handled by the fallthrough + } + return null; + }, + + _getAllPrefs() { + let values = {}; + for (let pref of this._getSyncPrefs()) { + // Note: _isSynced doesn't call isUnsyncableURLPref since it would cause + // us not to apply (syncable) changes to preferences that are set locally + // which have unsyncable urls. + if (this._isSynced(pref) && !isUnsyncableURLPref(pref)) { + let isSet = this._prefs.prefHasUserValue(pref); + // Missing and default prefs get the null value, unless that `seen` + // pref is set, in which case it always gets the value. + let forceValue = this._prefs.getBoolPref( + PREF_SYNC_SEEN_PREFIX + pref, + false + ); + if (isSet || forceValue) { + values[pref] = this._getPrefValue(pref); + } else { + values[pref] = null; + } + // If incoming and outgoing don't match then either the user toggled a + // pref that doesn't match an incoming non-default value for that pref + // during a sync (unlikely!) or it refused to stick and is behaving oddly. + if (this._incomingPrefs) { + let inValue = this._incomingPrefs[pref]; + let outValue = values[pref]; + if (inValue != null && outValue != null && inValue != outValue) { + this._log.debug(`Incoming pref '${pref}' refused to stick?`); + this._log.trace(`Incoming: '${inValue}', outgoing: '${outValue}'`); + } + } + // If this is a special "sync-seen" pref, and it's not the default value, + // set the seen pref to true. + if ( + isSet && + this._prefs.getBoolPref(PREF_SYNC_SEEN_PREFIX + pref, false) === false + ) { + this._log.trace(`toggling sync-seen pref for '${pref}' to true`); + this._prefs.setBoolPref(PREF_SYNC_SEEN_PREFIX + pref, true); + } + } + } + return values; + }, + + _maybeLogPrefChange(pref, incomingValue, existingValue) { + if (incomingValue != existingValue) { + this._log.debug(`Adjusting preference "${pref}" to the incoming value`); + // values are PII, so must only be logged at trace. + this._log.trace(`Existing: ${existingValue}. Incoming: ${incomingValue}`); + } + }, + + _setAllPrefs(values) { + const selectedThemeIDPref = "extensions.activeThemeID"; + let selectedThemeIDBefore = this._prefs.getStringPref( + selectedThemeIDPref, + null + ); + let selectedThemeIDAfter = selectedThemeIDBefore; + + // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise + // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device). + let prefs = Object.keys(values).sort( + a => -a.indexOf(PREF_SYNC_PREFS_PREFIX) + ); + for (let pref of prefs) { + let value = values[pref]; + if (!this._isSynced(pref)) { + // It's unusual for us to find an incoming preference (ie, a pref some other + // instance thinks is syncable) which we don't think is syncable. + this._log.trace(`Ignoring incoming unsyncable preference "${pref}"`); + continue; + } + + if (typeof value == "string" && UNSYNCABLE_URL_REGEXP.test(value)) { + this._log.trace(`Skipping incoming unsyncable url for pref: ${pref}`); + continue; + } + + switch (pref) { + // Some special prefs we don't want to set directly. + case selectedThemeIDPref: + selectedThemeIDAfter = value; + break; + + // default is to just set the pref + default: + if (value == null) { + // Pref has gone missing. The best we can do is reset it. + if (this._prefs.prefHasUserValue(pref)) { + this._log.debug(`Clearing existing local preference "${pref}"`); + this._log.trace( + `Existing local value for preference: ${this._getPrefValue( + pref + )}` + ); + } + this._prefs.clearUserPref(pref); + } else { + try { + switch (typeof value) { + case "string": + this._maybeLogPrefChange( + pref, + value, + this._prefs.getStringPref(pref, undefined) + ); + this._prefs.setStringPref(pref, value); + break; + case "number": + this._maybeLogPrefChange( + pref, + value, + this._prefs.getIntPref(pref, undefined) + ); + this._prefs.setIntPref(pref, value); + break; + case "boolean": + this._maybeLogPrefChange( + pref, + value, + this._prefs.getBoolPref(pref, undefined) + ); + this._prefs.setBoolPref(pref, value); + break; + } + } catch (ex) { + this._log.trace(`Failed to set pref: ${pref}`, ex); + } + } + // If there's a "sync-seen" pref for this it gets toggled to true + // regardless of the value. + let seenPref = PREF_SYNC_SEEN_PREFIX + pref; + if ( + this._prefs.getPrefType(seenPref) != Ci.nsIPrefBranch.PREF_INVALID + ) { + this._prefs.setBoolPref(PREF_SYNC_SEEN_PREFIX + pref, true); + } + } + } + // Themes are a little messy. Themes which have been installed are handled + // by the addons engine - but default themes aren't seen by that engine. + // So if there's a new default theme ID and that ID corresponds to a + // system addon, then we arrange to enable that addon here. + if (selectedThemeIDBefore != selectedThemeIDAfter) { + this._maybeEnableBuiltinTheme(selectedThemeIDAfter).catch(e => { + this._log.error("Failed to maybe update the default theme", e); + }); + } + }, + + async _maybeEnableBuiltinTheme(themeId) { + let addon = null; + try { + addon = await lazy.AddonManager.getAddonByID(themeId); + } catch (ex) { + this._log.trace( + `There's no addon with ID '${themeId} - it can't be a builtin theme` + ); + return; + } + if (addon && addon.isBuiltin && addon.type == "theme") { + this._log.trace(`Enabling builtin theme '${themeId}'`); + await addon.enable(); + } else { + this._log.trace( + `Have incoming theme ID of '${themeId}' but it's not a builtin theme` + ); + } + }, + + async getAllIDs() { + /* We store all prefs in just one WBO, with just one GUID */ + let allprefs = {}; + allprefs[lazy.PREFS_GUID] = true; + return allprefs; + }, + + async changeItemID(oldID, newID) { + this._log.trace("PrefStore GUID is constant!"); + }, + + async itemExists(id) { + return id === lazy.PREFS_GUID; + }, + + async createRecord(id, collection) { + let record = new PrefRec(collection, id); + + if (id == lazy.PREFS_GUID) { + record.value = this._getAllPrefs(); + } else { + record.deleted = true; + } + + return record; + }, + + async create(record) { + this._log.trace("Ignoring create request"); + }, + + async remove(record) { + this._log.trace("Ignoring remove request"); + }, + + async update(record) { + // Silently ignore pref updates that are for other apps. + if (record.id != lazy.PREFS_GUID) { + return; + } + + this._log.trace("Received pref updates, applying..."); + this._incomingPrefs = record.value; + this._setAllPrefs(record.value); + }, + + async wipe() { + this._log.trace("Ignoring wipe request"); + }, +}; +Object.setPrototypeOf(PrefStore.prototype, Store.prototype); + +function PrefTracker(name, engine) { + Tracker.call(this, name, engine); + this._ignoreAll = false; + Svc.Obs.add("profile-before-change", this.asyncObserver); +} +PrefTracker.prototype = { + get ignoreAll() { + return this._ignoreAll; + }, + + set ignoreAll(value) { + this._ignoreAll = value; + }, + + get modified() { + return Svc.PrefBranch.getBoolPref("engine.prefs.modified", false); + }, + set modified(value) { + Svc.PrefBranch.setBoolPref("engine.prefs.modified", value); + }, + + clearChangedIDs: function clearChangedIDs() { + this.modified = false; + }, + + __prefs: null, + get _prefs() { + if (!this.__prefs) { + this.__prefs = Services.prefs.getBranch(""); + } + return this.__prefs; + }, + + onStart() { + Services.prefs.addObserver("", this.asyncObserver); + }, + + onStop() { + this.__prefs = null; + Services.prefs.removeObserver("", this.asyncObserver); + }, + + async observe(subject, topic, data) { + switch (topic) { + case "profile-before-change": + await this.stop(); + break; + case "nsPref:changed": + if (this.ignoreAll) { + break; + } + // Trigger a sync for MULTI-DEVICE for a change that determines + // which prefs are synced or a regular pref change. + if ( + data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 || + this._prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + data, false) + ) { + this.score += SCORE_INCREMENT_XLARGE; + this.modified = true; + this._log.trace("Preference " + data + " changed"); + } + break; + } + }, +}; +Object.setPrototypeOf(PrefTracker.prototype, Tracker.prototype); + +export function getPrefsGUIDForTest() { + return lazy.PREFS_GUID; +} diff --git a/services/sync/modules/engines/tabs.sys.mjs b/services/sync/modules/engines/tabs.sys.mjs new file mode 100644 index 0000000000..861e051d1a --- /dev/null +++ b/services/sync/modules/engines/tabs.sys.mjs @@ -0,0 +1,625 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const STORAGE_VERSION = 1; // This needs to be kept in-sync with the rust storage version + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; +import { + SCORE_INCREMENT_SMALL, + STATUS_OK, + URI_LENGTH_MAX, +} from "resource://services-sync/constants.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Async } from "resource://services-common/async.sys.mjs"; +import { + SyncRecord, + SyncTelemetry, +} from "resource://services-sync/telemetry.sys.mjs"; +import { BridgedEngine } from "resource://services-sync/bridged_engine.sys.mjs"; + +const FAR_FUTURE = 4102405200000; // 2100/01/01 + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + ReaderMode: "resource://gre/modules/ReaderMode.sys.mjs", + TabsStore: "resource://gre/modules/RustTabs.sys.mjs", + RemoteTabRecord: "resource://gre/modules/RustTabs.sys.mjs", +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "TABS_FILTERED_SCHEMES", + "services.sync.engine.tabs.filteredSchemes", + "", + null, + val => { + return new Set(val.split("|")); + } +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "SYNC_AFTER_DELAY_MS", + "services.sync.syncedTabs.syncDelayAfterTabChange", + 0 +); + +// A "bridged engine" to our tabs component. +export function TabEngine(service) { + BridgedEngine.call(this, "Tabs", service); +} + +TabEngine.prototype = { + _trackerObj: TabTracker, + syncPriority: 3, + + async prepareTheBridge(isQuickWrite) { + let clientsEngine = this.service.clientsEngine; + // Tell the bridged engine about clients. + // This is the same shape as ClientData in app-services. + // schema: https://github.com/mozilla/application-services/blob/a1168751231ed4e88c44d85f6dccc09c3b412bd2/components/sync15/src/client_types.rs#L14 + let clientData = { + local_client_id: clientsEngine.localID, + recent_clients: {}, + }; + + // We shouldn't upload tabs past what the server will accept + let tabs = await this.getTabsWithinPayloadSize(); + await this._rustStore.setLocalTabs( + tabs.map(tab => { + // rust wants lastUsed in MS but the provider gives it in seconds + tab.lastUsed = tab.lastUsed * 1000; + return new lazy.RemoteTabRecord(tab); + }) + ); + + for (let remoteClient of clientsEngine.remoteClients) { + let id = remoteClient.id; + if (!id) { + throw new Error("Remote client somehow did not have an id"); + } + let client = { + fxa_device_id: remoteClient.fxaDeviceId, + // device_name and device_type are soft-deprecated - every client + // prefers what's in the FxA record. But fill them correctly anyway. + device_name: clientsEngine.getClientName(id) ?? "", + device_type: clientsEngine.getClientType(id), + }; + clientData.recent_clients[id] = client; + } + + // put ourself in there too so we record the correct device info in our sync record. + clientData.recent_clients[clientsEngine.localID] = { + fxa_device_id: await clientsEngine.fxAccounts.device.getLocalId(), + device_name: clientsEngine.localName, + device_type: clientsEngine.localType, + }; + + // Quick write needs to adjust the lastSync so we can POST to the server + // see quickWrite() for details + if (isQuickWrite) { + await this.setLastSync(FAR_FUTURE); + await this._bridge.prepareForSync(JSON.stringify(clientData)); + return; + } + + // Just incase we crashed while the lastSync timestamp was FAR_FUTURE, we + // reset it to zero + if ((await this.getLastSync()) === FAR_FUTURE) { + await this._bridge.setLastSync(0); + } + await this._bridge.prepareForSync(JSON.stringify(clientData)); + }, + + async _syncStartup() { + await super._syncStartup(); + await this.prepareTheBridge(); + }, + + async initialize() { + await SyncEngine.prototype.initialize.call(this); + + let path = PathUtils.join(PathUtils.profileDir, "synced-tabs.db"); + this._rustStore = await lazy.TabsStore.init(path); + this._bridge = await this._rustStore.bridgedEngine(); + + // Uniffi doesn't currently only support async methods, so we'll need to hardcode + // these values for now (which is fine for now as these hardly ever change) + this._bridge.storageVersion = STORAGE_VERSION; + this._bridge.allowSkippedRecord = true; + + this._log.info("Got a bridged engine!"); + this._tracker.modified = true; + }, + + async getChangedIDs() { + // No need for a proper timestamp (no conflict resolution needed). + let changedIDs = {}; + if (this._tracker.modified) { + changedIDs[this.service.clientsEngine.localID] = 0; + } + return changedIDs; + }, + + // API for use by Sync UI code to give user choices of tabs to open. + async getAllClients() { + let remoteTabs = await this._rustStore.getAll(); + let remoteClientTabs = []; + for (let remoteClient of this.service.clientsEngine.remoteClients) { + // We get the some client info from the rust tabs engine and some from + // the clients engine. + let rustClient = remoteTabs.find( + x => x.clientId === remoteClient.fxaDeviceId + ); + if (!rustClient) { + continue; + } + let client = { + // rust gives us ms but js uses seconds, so fix them up. + tabs: rustClient.remoteTabs.map(tab => { + tab.lastUsed = tab.lastUsed / 1000; + return tab; + }), + lastModified: rustClient.lastModified / 1000, + ...remoteClient, + }; + remoteClientTabs.push(client); + } + return remoteClientTabs; + }, + + async removeClientData() { + let url = this.engineURL + "/" + this.service.clientsEngine.localID; + await this.service.resource(url).delete(); + }, + + async trackRemainingChanges() { + if (this._modified.count() > 0) { + this._tracker.modified = true; + } + }, + + async getTabsWithinPayloadSize() { + const maxPayloadSize = this.service.getMaxRecordPayloadSize(); + // See bug 535326 comment 8 for an explanation of the estimation + const maxSerializedSize = (maxPayloadSize / 4) * 3 - 1500; + return TabProvider.getAllTabsWithEstimatedMax(true, maxSerializedSize); + }, + + // Support for "quick writes" + _engineLock: Utils.lock, + _engineLocked: false, + + // Tabs has a special lock to help support its "quick write" + get locked() { + return this._engineLocked; + }, + lock() { + if (this._engineLocked) { + return false; + } + this._engineLocked = true; + return true; + }, + unlock() { + this._engineLocked = false; + }, + + // Quickly do a POST of our current tabs if possible. + // This does things that would be dangerous for other engines - eg, posting + // without checking what's on the server could cause data-loss for other + // engines, but because each device exclusively owns exactly 1 tabs record + // with a known ID, it's safe here. + // Returns true if we successfully synced, false otherwise (either on error + // or because we declined to sync for any reason.) The return value is + // primarily for tests. + async quickWrite() { + if (!this.enabled) { + // this should be very rare, and only if tabs are disabled after the + // timer is created. + this._log.info("Can't do a quick-sync as tabs is disabled"); + return false; + } + // This quick-sync doesn't drive the login state correctly, so just + // decline to sync if out status is bad + if (this.service.status.checkSetup() != STATUS_OK) { + this._log.info( + "Can't do a quick-sync due to the service status", + this.service.status.toString() + ); + return false; + } + if (!this.service.serverConfiguration) { + this._log.info("Can't do a quick sync before the first full sync"); + return false; + } + try { + return await this._engineLock("tabs.js: quickWrite", async () => { + // We want to restore the lastSync timestamp when complete so next sync + // takes tabs written by other devices since our last real sync. + // And for this POST we don't want the protections offered by + // X-If-Unmodified-Since - we want the POST to work even if the remote + // has moved on and we will catch back up next full sync. + const origLastSync = await this.getLastSync(); + try { + return this._doQuickWrite(); + } finally { + // set the lastSync to it's original value for regular sync + await this.setLastSync(origLastSync); + } + })(); + } catch (ex) { + if (!Utils.isLockException(ex)) { + throw ex; + } + this._log.info( + "Can't do a quick-write as another tab sync is in progress" + ); + return false; + } + }, + + // The guts of the quick-write sync, after we've taken the lock, checked + // the service status etc. + async _doQuickWrite() { + // We need to track telemetry for these syncs too! + const name = "tabs"; + let telemetryRecord = new SyncRecord( + SyncTelemetry.allowedEngines, + "quick-write" + ); + telemetryRecord.onEngineStart(name); + try { + Async.checkAppReady(); + // We need to prep the bridge before we try to POST since it grabs + // the most recent local client id and properly sets a lastSync + // which is needed for a proper POST request + await this.prepareTheBridge(true); + this._tracker.clearChangedIDs(); + this._tracker.resetScore(); + + Async.checkAppReady(); + // now just the "upload" part of a sync, + // which for a rust engine is not obvious. + // We need to do is ask the rust engine for the changes. Although + // this is kinda abusing the bridged-engine interface, we know the tabs + // implementation of it works ok + let outgoing = await this._bridge.apply(); + // We know we always have exactly 1 record. + let mine = outgoing[0]; + this._log.trace("outgoing bso", mine); + // `this._recordObj` is a `BridgedRecord`, which isn't exported. + let record = this._recordObj.fromOutgoingBso(this.name, JSON.parse(mine)); + let changeset = {}; + changeset[record.id] = { synced: false, record }; + this._modified.replace(changeset); + + Async.checkAppReady(); + await this._uploadOutgoing(); + telemetryRecord.onEngineStop(name, null); + return true; + } catch (ex) { + this._log.warn("quicksync sync failed", ex); + telemetryRecord.onEngineStop(name, ex); + return false; + } finally { + // The top-level sync is never considered to fail here, just the engine + telemetryRecord.finished(null); + SyncTelemetry.takeTelemetryRecord(telemetryRecord); + } + }, + + async _sync() { + try { + await this._engineLock("tabs.js: fullSync", async () => { + await super._sync(); + })(); + } catch (ex) { + if (!Utils.isLockException(ex)) { + throw ex; + } + this._log.info( + "Can't do full tabs sync as a quick-write is currently running" + ); + } + }, +}; +Object.setPrototypeOf(TabEngine.prototype, BridgedEngine.prototype); + +export const TabProvider = { + getWindowEnumerator() { + return Services.wm.getEnumerator("navigator:browser"); + }, + + shouldSkipWindow(win) { + return win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win); + }, + + getAllBrowserTabs() { + let tabs = []; + for (let win of this.getWindowEnumerator()) { + if (this.shouldSkipWindow(win)) { + continue; + } + // Get all the tabs from the browser + for (let tab of win.gBrowser.tabs) { + tabs.push(tab); + } + } + + return tabs.sort(function (a, b) { + return b.lastAccessed - a.lastAccessed; + }); + }, + + // This function creates tabs records up to a specified amount of bytes + // It is an "estimation" since we don't accurately calculate how much the + // favicon and JSON overhead is and give a rough estimate (for optimization purposes) + async getAllTabsWithEstimatedMax(filter, bytesMax) { + let log = Log.repository.getLogger(`Sync.Engine.Tabs.Provider`); + let tabRecords = []; + let iconPromises = []; + let runningByteLength = 0; + let encoder = new TextEncoder(); + + // Fetch all the tabs the user has open + let winTabs = this.getAllBrowserTabs(); + + for (let tab of winTabs) { + // We don't want to process any more tabs than we can sync + if (runningByteLength >= bytesMax) { + log.warn( + `Can't fit all tabs in sync payload: have ${winTabs.length}, + but can only fit ${tabRecords.length}.` + ); + break; + } + + // Note that we used to sync "tab history" (ie, the "back button") state, + // but in practice this hasn't been used - only the current URI is of + // interest to clients. + // We stopped recording this in bug 1783991. + if (!tab?.linkedBrowser) { + continue; + } + let acceptable = !filter + ? url => url + : url => + url && + !lazy.TABS_FILTERED_SCHEMES.has(Services.io.extractScheme(url)); + + let url = tab.linkedBrowser.currentURI?.spec; + // Special case for reader mode. + if (url && url.startsWith("about:reader?")) { + url = lazy.ReaderMode.getOriginalUrl(url); + } + // We ignore the tab completely if the current entry url is + // not acceptable (we need something accurate to open). + if (!acceptable(url)) { + continue; + } + + if (url.length > URI_LENGTH_MAX) { + log.trace("Skipping over-long URL."); + continue; + } + + let thisTab = new lazy.RemoteTabRecord({ + title: tab.linkedBrowser.contentTitle || "", + urlHistory: [url], + icon: "", + lastUsed: Math.floor((tab.lastAccessed || 0) / 1000), + }); + tabRecords.push(thisTab); + + // we don't want to wait for each favicon to resolve to get the bytes + // so we estimate a conservative 100 chars for the favicon and json overhead + // Rust will further optimize and trim if we happened to be wildly off + runningByteLength += + encoder.encode(thisTab.title + thisTab.lastUsed + url).byteLength + 100; + + // Use the favicon service for the icon url - we can wait for the promises at the end. + let iconPromise = lazy.PlacesUtils.promiseFaviconData(url) + .then(iconData => { + thisTab.icon = iconData.uri.spec; + }) + .catch(ex => { + log.trace( + `Failed to fetch favicon for ${url}`, + thisTab.urlHistory[0] + ); + }); + iconPromises.push(iconPromise); + } + + await Promise.allSettled(iconPromises); + return tabRecords; + }, +}; + +function TabTracker(name, engine) { + Tracker.call(this, name, engine); + + // Make sure "this" pointer is always set correctly for event listeners. + this.onTab = Utils.bind2(this, this.onTab); + this._unregisterListeners = Utils.bind2(this, this._unregisterListeners); +} +TabTracker.prototype = { + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), + + clearChangedIDs() { + this.modified = false; + }, + + // We do not track TabSelect because that almost always triggers + // the web progress listeners (onLocationChange), which we already track + _topics: ["TabOpen", "TabClose"], + + _registerListenersForWindow(window) { + this._log.trace("Registering tab listeners in window"); + for (let topic of this._topics) { + window.addEventListener(topic, this.onTab); + } + window.addEventListener("unload", this._unregisterListeners); + // If it's got a tab browser we can listen for things like navigation. + if (window.gBrowser) { + window.gBrowser.addProgressListener(this); + } + }, + + _unregisterListeners(event) { + this._unregisterListenersForWindow(event.target); + }, + + _unregisterListenersForWindow(window) { + this._log.trace("Removing tab listeners in window"); + window.removeEventListener("unload", this._unregisterListeners); + for (let topic of this._topics) { + window.removeEventListener(topic, this.onTab); + } + if (window.gBrowser) { + window.gBrowser.removeProgressListener(this); + } + }, + + onStart() { + Svc.Obs.add("domwindowopened", this.asyncObserver); + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._registerListenersForWindow(win); + } + }, + + onStop() { + Svc.Obs.remove("domwindowopened", this.asyncObserver); + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._unregisterListenersForWindow(win); + } + }, + + async observe(subject, topic, data) { + switch (topic) { + case "domwindowopened": + let onLoad = () => { + subject.removeEventListener("load", onLoad); + // Only register after the window is done loading to avoid unloads. + this._registerListenersForWindow(subject); + }; + + // Add tab listeners now that a window has opened. + subject.addEventListener("load", onLoad); + break; + } + }, + + onTab(event) { + if (event.originalTarget.linkedBrowser) { + let browser = event.originalTarget.linkedBrowser; + if ( + lazy.PrivateBrowsingUtils.isBrowserPrivate(browser) && + !lazy.PrivateBrowsingUtils.permanentPrivateBrowsing + ) { + this._log.trace("Ignoring tab event from private browsing."); + return; + } + } + this._log.trace("onTab event: " + event.type); + + switch (event.type) { + case "TabOpen": + /* We do not have a reliable way of checking the URI on the TabOpen + * so we will rely on the other methods (onLocationChange, getAllTabsWithEstimatedMax) + * to filter these when going through sync + */ + this.callScheduleSync(SCORE_INCREMENT_SMALL); + break; + case "TabClose": + // If event target has `linkedBrowser`, the event target can be assumed <tab> element. + // Else, event target is assumed <browser> element, use the target as it is. + const tab = event.target.linkedBrowser || event.target; + + // TabClose means the tab has already loaded and we can check the URI + // and ignore if it's a scheme we don't care about + if (lazy.TABS_FILTERED_SCHEMES.has(tab.currentURI.scheme)) { + return; + } + this.callScheduleSync(SCORE_INCREMENT_SMALL); + break; + } + }, + + // web progress listeners. + onLocationChange(webProgress, request, locationURI, flags) { + // We only care about top-level location changes. We do want location changes in the + // same document because if a page uses the `pushState()` API, they *appear* as though + // they are in the same document even if the URL changes. It also doesn't hurt to accurately + // reflect the fragment changing - so we allow LOCATION_CHANGE_SAME_DOCUMENT + if ( + flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_RELOAD || + !webProgress.isTopLevel || + !locationURI + ) { + return; + } + + // We can't filter out tabs that we don't sync here, because we might be + // navigating from a tab that we *did* sync to one we do not, and that + // tab we *did* sync should no longer be synced. + this.callScheduleSync(); + }, + + callScheduleSync(scoreIncrement) { + this.modified = true; + let { scheduler } = this.engine.service; + let delayInMs = lazy.SYNC_AFTER_DELAY_MS; + + // Schedule a sync once we detect a tab change + // to ensure the server always has the most up to date tabs + if ( + delayInMs > 0 && + scheduler.numClients > 1 // Only schedule quick syncs for multi client users + ) { + if (this.tabsQuickWriteTimer) { + this._log.debug( + "Detected a tab change, but a quick-write is already scheduled" + ); + return; + } + this._log.debug( + "Detected a tab change: scheduling a quick-write in " + delayInMs + "ms" + ); + CommonUtils.namedTimer( + () => { + this._log.trace("tab quick-sync timer fired."); + this.engine + .quickWrite() + .then(() => { + this._log.trace("tab quick-sync done."); + }) + .catch(ex => { + this._log.error("tab quick-sync failed.", ex); + }); + }, + delayInMs, + this, + "tabsQuickWriteTimer" + ); + } else if (scoreIncrement) { + this._log.debug( + "Detected a tab change, but conditions aren't met for a quick write - bumping score" + ); + this.score += scoreIncrement; + } else { + this._log.debug( + "Detected a tab change, but conditions aren't met for a quick write or a score bump" + ); + } + }, +}; +Object.setPrototypeOf(TabTracker.prototype, Tracker.prototype); diff --git a/services/sync/modules/keys.sys.mjs b/services/sync/modules/keys.sys.mjs new file mode 100644 index 0000000000..b6a1dce19a --- /dev/null +++ b/services/sync/modules/keys.sys.mjs @@ -0,0 +1,166 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Weave } from "resource://services-sync/main.sys.mjs"; + +/** + * Represents a pair of keys. + * + * Each key stored in a key bundle is 256 bits. One key is used for symmetric + * encryption. The other is used for HMAC. + * + * A KeyBundle by itself is just an anonymous pair of keys. Other types + * deriving from this one add semantics, such as associated collections or + * generating a key bundle via HKDF from another key. + */ +function KeyBundle() { + this._encrypt = null; + this._encryptB64 = null; + this._hmac = null; + this._hmacB64 = null; +} +KeyBundle.prototype = { + _encrypt: null, + _encryptB64: null, + _hmac: null, + _hmacB64: null, + + equals: function equals(bundle) { + return ( + bundle && + bundle.hmacKey == this.hmacKey && + bundle.encryptionKey == this.encryptionKey + ); + }, + + /* + * Accessors for the two keys. + */ + get encryptionKey() { + return this._encrypt; + }, + + set encryptionKey(value) { + if (!value || typeof value != "string") { + throw new Error("Encryption key can only be set to string values."); + } + + if (value.length < 16) { + throw new Error("Encryption key must be at least 128 bits long."); + } + + this._encrypt = value; + this._encryptB64 = btoa(value); + }, + + get encryptionKeyB64() { + return this._encryptB64; + }, + + get hmacKey() { + return this._hmac; + }, + + set hmacKey(value) { + if (!value || typeof value != "string") { + throw new Error("HMAC key can only be set to string values."); + } + + if (value.length < 16) { + throw new Error("HMAC key must be at least 128 bits long."); + } + + this._hmac = value; + this._hmacB64 = btoa(value); + }, + + get hmacKeyB64() { + return this._hmacB64; + }, + + /** + * Populate this key pair with 2 new, randomly generated keys. + */ + async generateRandom() { + // Compute both at that same time + let [generatedHMAC, generatedEncr] = await Promise.all([ + Weave.Crypto.generateRandomKey(), + Weave.Crypto.generateRandomKey(), + ]); + this.keyPairB64 = [generatedEncr, generatedHMAC]; + }, +}; + +/** + * Represents a KeyBundle associated with a collection. + * + * This is just a KeyBundle with a collection attached. + */ +export function BulkKeyBundle(collection) { + let log = Log.repository.getLogger("Sync.BulkKeyBundle"); + log.info("BulkKeyBundle being created for " + collection); + KeyBundle.call(this); + + this._collection = collection; +} + +BulkKeyBundle.fromHexKey = function (hexKey) { + let key = CommonUtils.hexToBytes(hexKey); + let bundle = new BulkKeyBundle(); + // [encryptionKey, hmacKey] + bundle.keyPair = [key.slice(0, 32), key.slice(32, 64)]; + return bundle; +}; + +BulkKeyBundle.fromJWK = function (jwk) { + if (!jwk || !jwk.k || jwk.kty !== "oct") { + throw new Error("Invalid JWK provided to BulkKeyBundle.fromJWK"); + } + return BulkKeyBundle.fromHexKey(CommonUtils.base64urlToHex(jwk.k)); +}; + +BulkKeyBundle.prototype = { + get collection() { + return this._collection; + }, + + /** + * Obtain the key pair in this key bundle. + * + * The returned keys are represented as raw byte strings. + */ + get keyPair() { + return [this.encryptionKey, this.hmacKey]; + }, + + set keyPair(value) { + if (!Array.isArray(value) || value.length != 2) { + throw new Error("BulkKeyBundle.keyPair value must be array of 2 keys."); + } + + this.encryptionKey = value[0]; + this.hmacKey = value[1]; + }, + + get keyPairB64() { + return [this.encryptionKeyB64, this.hmacKeyB64]; + }, + + set keyPairB64(value) { + if (!Array.isArray(value) || value.length != 2) { + throw new Error( + "BulkKeyBundle.keyPairB64 value must be an array of 2 keys." + ); + } + + this.encryptionKey = CommonUtils.safeAtoB(value[0]); + this.hmacKey = CommonUtils.safeAtoB(value[1]); + }, +}; + +Object.setPrototypeOf(BulkKeyBundle.prototype, KeyBundle.prototype); diff --git a/services/sync/modules/main.sys.mjs b/services/sync/modules/main.sys.mjs new file mode 100644 index 0000000000..838da15742 --- /dev/null +++ b/services/sync/modules/main.sys.mjs @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export { lazy as Weave }; + +const lazy = {}; + +// We want these to be lazily loaded, which helps performance and also tests +// to not have these loaded before they are ready. +ChromeUtils.defineESModuleGetters(lazy, { + Service: "resource://services-sync/service.sys.mjs", + Status: "resource://services-sync/status.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + Utils: "resource://services-sync/util.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "Crypto", () => { + let { WeaveCrypto } = ChromeUtils.importESModule( + "resource://services-crypto/WeaveCrypto.sys.mjs" + ); + return new WeaveCrypto(); +}); diff --git a/services/sync/modules/policies.sys.mjs b/services/sync/modules/policies.sys.mjs new file mode 100644 index 0000000000..290e793b8e --- /dev/null +++ b/services/sync/modules/policies.sys.mjs @@ -0,0 +1,1055 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { + CREDENTIALS_CHANGED, + ENGINE_APPLY_FAIL, + ENGINE_UNKNOWN_FAIL, + IDLE_OBSERVER_BACK_DELAY, + LOGIN_FAILED_INVALID_PASSPHRASE, + LOGIN_FAILED_LOGIN_REJECTED, + LOGIN_FAILED_NETWORK_ERROR, + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_SUCCEEDED, + MASTER_PASSWORD_LOCKED, + MASTER_PASSWORD_LOCKED_RETRY_INTERVAL, + MAX_ERROR_COUNT_BEFORE_BACKOFF, + MINIMUM_BACKOFF_INTERVAL, + MULTI_DEVICE_THRESHOLD, + NO_SYNC_NODE_FOUND, + NO_SYNC_NODE_INTERVAL, + OVER_QUOTA, + RESPONSE_OVER_QUOTA, + SCORE_UPDATE_DELAY, + SERVER_MAINTENANCE, + SINGLE_USER_THRESHOLD, + STATUS_OK, + SYNC_FAILED_PARTIAL, + SYNC_SUCCEEDED, + kSyncBackoffNotMet, + kSyncMasterPasswordLocked, +} from "resource://services-sync/constants.sys.mjs"; + +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { logManager } from "resource://gre/modules/FxAccountsCommon.sys.mjs"; +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + Status: "resource://services-sync/status.sys.mjs", +}); +ChromeUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); +XPCOMUtils.defineLazyServiceGetter( + lazy, + "IdleService", + "@mozilla.org/widget/useridleservice;1", + "nsIUserIdleService" +); +XPCOMUtils.defineLazyServiceGetter( + lazy, + "CaptivePortalService", + "@mozilla.org/network/captive-portal-service;1", + "nsICaptivePortalService" +); + +// Get the value for an interval that's stored in preferences. To save users +// from themselves (and us from them!) the minimum time they can specify +// is 60s. +function getThrottledIntervalPreference(prefName) { + return Math.max(Svc.PrefBranch.getIntPref(prefName), 60) * 1000; +} + +export function SyncScheduler(service) { + this.service = service; + this.init(); +} + +SyncScheduler.prototype = { + _log: Log.repository.getLogger("Sync.SyncScheduler"), + + _fatalLoginStatus: [ + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_FAILED_INVALID_PASSPHRASE, + LOGIN_FAILED_LOGIN_REJECTED, + ], + + /** + * The nsITimer object that schedules the next sync. See scheduleNextSync(). + */ + syncTimer: null, + + setDefaults: function setDefaults() { + this._log.trace("Setting SyncScheduler policy values to defaults."); + + this.singleDeviceInterval = getThrottledIntervalPreference( + "scheduler.fxa.singleDeviceInterval" + ); + this.idleInterval = getThrottledIntervalPreference( + "scheduler.idleInterval" + ); + this.activeInterval = getThrottledIntervalPreference( + "scheduler.activeInterval" + ); + this.immediateInterval = getThrottledIntervalPreference( + "scheduler.immediateInterval" + ); + + // A user is non-idle on startup by default. + this.idle = false; + + this.hasIncomingItems = false; + // This is the last number of clients we saw when previously updating the + // client mode. If this != currentNumClients (obtained from prefs written + // by the clients engine) then we need to transition to and from + // single and multi-device mode. + this.numClientsLastSync = 0; + + this._resyncs = 0; + + this.clearSyncTriggers(); + }, + + // nextSync is in milliseconds, but prefs can't hold that much + get nextSync() { + return Svc.PrefBranch.getIntPref("nextSync", 0) * 1000; + }, + set nextSync(value) { + Svc.PrefBranch.setIntPref("nextSync", Math.floor(value / 1000)); + }, + + get missedFxACommandsFetchInterval() { + return Services.prefs.getIntPref( + "identity.fxaccounts.commands.missed.fetch_interval" + ); + }, + + get missedFxACommandsLastFetch() { + return Services.prefs.getIntPref( + "identity.fxaccounts.commands.missed.last_fetch", + 0 + ); + }, + + set missedFxACommandsLastFetch(val) { + Services.prefs.setIntPref( + "identity.fxaccounts.commands.missed.last_fetch", + val + ); + }, + + get syncInterval() { + return this._syncInterval; + }, + set syncInterval(value) { + if (value != this._syncInterval) { + Services.prefs.setIntPref("services.sync.syncInterval", value); + } + }, + + get syncThreshold() { + return this._syncThreshold; + }, + set syncThreshold(value) { + if (value != this._syncThreshold) { + Services.prefs.setIntPref("services.sync.syncThreshold", value); + } + }, + + get globalScore() { + return this._globalScore; + }, + set globalScore(value) { + if (this._globalScore != value) { + Services.prefs.setIntPref("services.sync.globalScore", value); + } + }, + + // Managed by the clients engine (by way of prefs) + get numClients() { + return this.numDesktopClients + this.numMobileClients; + }, + set numClients(value) { + throw new Error("Don't set numClients - the clients engine manages it."); + }, + + get offline() { + // Services.io.offline has slowly become fairly useless over the years - it + // no longer attempts to track the actual network state by default, but one + // thing stays true: if it says we're offline then we are definitely not online. + // + // We also ask the captive portal service if we are behind a locked captive + // portal. + // + // We don't check on the NetworkLinkService however, because it gave us + // false positives in the past in a vm environment. + try { + if ( + Services.io.offline || + lazy.CaptivePortalService.state == + lazy.CaptivePortalService.LOCKED_PORTAL + ) { + return true; + } + } catch (ex) { + this._log.warn("Could not determine network status.", ex); + } + return false; + }, + + _initPrefGetters() { + XPCOMUtils.defineLazyPreferenceGetter( + this, + "idleTime", + "services.sync.scheduler.idleTime" + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "maxResyncs", + "services.sync.maxResyncs", + 0 + ); + + // The number of clients we have is maintained in preferences via the + // clients engine, and only updated after a successsful sync. + XPCOMUtils.defineLazyPreferenceGetter( + this, + "numDesktopClients", + "services.sync.clients.devices.desktop", + 0 + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "numMobileClients", + "services.sync.clients.devices.mobile", + 0 + ); + + // Scheduler state that seems to be read more often than it's written. + // We also check if the value has changed before writing in the setters. + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_syncThreshold", + "services.sync.syncThreshold", + SINGLE_USER_THRESHOLD + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_syncInterval", + "services.sync.syncInterval", + this.singleDeviceInterval + ); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_globalScore", + "services.sync.globalScore", + 0 + ); + }, + + init: function init() { + this._log.manageLevelFromPref("services.sync.log.logger.service.main"); + this.setDefaults(); + this._initPrefGetters(); + Svc.Obs.add("weave:engine:score:updated", this); + Svc.Obs.add("network:offline-status-changed", this); + Svc.Obs.add("network:link-status-changed", this); + Svc.Obs.add("captive-portal-detected", this); + Svc.Obs.add("weave:service:sync:start", this); + Svc.Obs.add("weave:service:sync:finish", this); + Svc.Obs.add("weave:engine:sync:finish", this); + Svc.Obs.add("weave:engine:sync:error", this); + Svc.Obs.add("weave:service:login:error", this); + Svc.Obs.add("weave:service:logout:finish", this); + Svc.Obs.add("weave:service:sync:error", this); + Svc.Obs.add("weave:service:backoff:interval", this); + Svc.Obs.add("weave:engine:sync:applied", this); + Svc.Obs.add("weave:service:setup-complete", this); + Svc.Obs.add("weave:service:start-over", this); + Svc.Obs.add("FxA:hawk:backoff:interval", this); + + if (lazy.Status.checkSetup() == STATUS_OK) { + Svc.Obs.add("wake_notification", this); + Svc.Obs.add("captive-portal-login-success", this); + Svc.Obs.add("sleep_notification", this); + lazy.IdleService.addIdleObserver(this, this.idleTime); + } + }, + + // eslint-disable-next-line complexity + observe: function observe(subject, topic, data) { + this._log.trace("Handling " + topic); + switch (topic) { + case "weave:engine:score:updated": + if (lazy.Status.login == LOGIN_SUCCEEDED) { + CommonUtils.namedTimer( + this.calculateScore, + SCORE_UPDATE_DELAY, + this, + "_scoreTimer" + ); + } + break; + case "network:link-status-changed": + // Note: NetworkLinkService is unreliable, we get false negatives for it + // in cases such as VMs (bug 1420802), so we don't want to use it in + // `get offline`, but we assume that it's probably reliable if we're + // getting status changed events. (We might be wrong about this, but + // if that's true, then the only downside is that we won't sync as + // promptly). + let isOffline = this.offline; + this._log.debug( + `Network link status changed to "${data}". Offline?`, + isOffline + ); + // Data may be one of `up`, `down`, `change`, or `unknown`. We only want + // to sync if it's "up". + if (data == "up" && !isOffline) { + this._log.debug("Network link looks up. Syncing."); + this.scheduleNextSync(0, { why: topic }); + } else if (data == "down") { + // Unschedule pending syncs if we know we're going down. We don't do + // this via `checkSyncStatus`, since link status isn't reflected in + // `this.offline`. + this.clearSyncTriggers(); + } + break; + case "network:offline-status-changed": + case "captive-portal-detected": + // Whether online or offline, we'll reschedule syncs + this._log.trace("Network offline status change: " + data); + this.checkSyncStatus(); + break; + case "weave:service:sync:start": + // Clear out any potentially pending syncs now that we're syncing + this.clearSyncTriggers(); + + // reset backoff info, if the server tells us to continue backing off, + // we'll handle that later + lazy.Status.resetBackoff(); + + this.globalScore = 0; + break; + case "weave:service:sync:finish": + this.nextSync = 0; + this.adjustSyncInterval(); + + if ( + lazy.Status.service == SYNC_FAILED_PARTIAL && + this.requiresBackoff + ) { + this.requiresBackoff = false; + this.handleSyncError(); + return; + } + + let sync_interval; + let nextSyncReason = "schedule"; + this.updateGlobalScore(); + if ( + this.globalScore > this.syncThreshold && + lazy.Status.service == STATUS_OK + ) { + // The global score should be 0 after a sync. If it's not, either + // items were changed during the last sync (and we should schedule an + // immediate follow-up sync), or an engine skipped + this._resyncs++; + if (this._resyncs <= this.maxResyncs) { + sync_interval = 0; + nextSyncReason = "resync"; + } else { + this._log.warn( + `Resync attempt ${this._resyncs} exceeded ` + + `maximum ${this.maxResyncs}` + ); + Svc.Obs.notify("weave:service:resyncs-finished"); + } + } else { + this._resyncs = 0; + Svc.Obs.notify("weave:service:resyncs-finished"); + } + + this._syncErrors = 0; + if (lazy.Status.sync == NO_SYNC_NODE_FOUND) { + // If we don't have a Sync node, override the interval, even if we've + // scheduled a follow-up sync. + this._log.trace("Scheduling a sync at interval NO_SYNC_NODE_FOUND."); + sync_interval = NO_SYNC_NODE_INTERVAL; + } + this.scheduleNextSync(sync_interval, { why: nextSyncReason }); + break; + case "weave:engine:sync:finish": + if (data == "clients") { + // Update the client mode because it might change what we sync. + this.updateClientMode(); + } + break; + case "weave:engine:sync:error": + // `subject` is the exception thrown by an engine's sync() method. + let exception = subject; + if (exception.status >= 500 && exception.status <= 504) { + this.requiresBackoff = true; + } + break; + case "weave:service:login:error": + this.clearSyncTriggers(); + + if (lazy.Status.login == MASTER_PASSWORD_LOCKED) { + // Try again later, just as if we threw an error... only without the + // error count. + this._log.debug("Couldn't log in: master password is locked."); + this._log.trace( + "Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL" + ); + this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL); + } else if (!this._fatalLoginStatus.includes(lazy.Status.login)) { + // Not a fatal login error, just an intermittent network or server + // issue. Keep on syncin'. + this.checkSyncStatus(); + } + break; + case "weave:service:logout:finish": + // Start or cancel the sync timer depending on if + // logged in or logged out + this.checkSyncStatus(); + break; + case "weave:service:sync:error": + // There may be multiple clients but if the sync fails, client mode + // should still be updated so that the next sync has a correct interval. + this.updateClientMode(); + this.adjustSyncInterval(); + this.nextSync = 0; + this.handleSyncError(); + break; + case "FxA:hawk:backoff:interval": + case "weave:service:backoff:interval": + let requested_interval = subject * 1000; + this._log.debug( + "Got backoff notification: " + requested_interval + "ms" + ); + // Leave up to 25% more time for the back off. + let interval = requested_interval * (1 + Math.random() * 0.25); + lazy.Status.backoffInterval = interval; + lazy.Status.minimumNextSync = Date.now() + requested_interval; + this._log.debug( + "Fuzzed minimum next sync: " + lazy.Status.minimumNextSync + ); + break; + case "weave:engine:sync:applied": + let numItems = subject.succeeded; + this._log.trace( + "Engine " + data + " successfully applied " + numItems + " items." + ); + // Bug 1800186 - the tabs engine always reports incoming items, so we don't + // want special scheduling in this scenario. + // (However, even when we fix the underlying cause of that, we probably still can + // ignore tabs here - new incoming tabs don't need to trigger the extra syncs we do + // based on this flag.) + if (data != "tabs" && numItems) { + this.hasIncomingItems = true; + } + if (subject.newFailed) { + this._log.error( + `Engine ${data} found ${subject.newFailed} new records that failed to apply` + ); + } + break; + case "weave:service:setup-complete": + Services.prefs.savePrefFile(null); + lazy.IdleService.addIdleObserver(this, this.idleTime); + Svc.Obs.add("wake_notification", this); + Svc.Obs.add("captive-portal-login-success", this); + Svc.Obs.add("sleep_notification", this); + break; + case "weave:service:start-over": + this.setDefaults(); + try { + lazy.IdleService.removeIdleObserver(this, this.idleTime); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_FAILURE) { + throw ex; + } + // In all likelihood we didn't have an idle observer registered yet. + // It's all good. + } + break; + case "idle": + this._log.trace("We're idle."); + this.idle = true; + // Adjust the interval for future syncs. This won't actually have any + // effect until the next pending sync (which will happen soon since we + // were just active.) + this.adjustSyncInterval(); + break; + case "active": + this._log.trace("Received notification that we're back from idle."); + this.idle = false; + CommonUtils.namedTimer( + function onBack() { + if (this.idle) { + this._log.trace( + "... and we're idle again. " + + "Ignoring spurious back notification." + ); + return; + } + + this._log.trace("Genuine return from idle. Syncing."); + // Trigger a sync if we have multiple clients. + if (this.numClients > 1) { + this.scheduleNextSync(0, { why: topic }); + } + }, + IDLE_OBSERVER_BACK_DELAY, + this, + "idleDebouncerTimer" + ); + break; + case "wake_notification": + this._log.debug("Woke from sleep."); + CommonUtils.nextTick(() => { + // Trigger a sync if we have multiple clients. We give it 2 seconds + // so the browser can recover from the wake and do more important + // operations first (timers etc). + if (this.numClients > 1) { + if (!this.offline) { + this._log.debug("Online, will sync in 2s."); + this.scheduleNextSync(2000, { why: topic }); + } + } + }); + break; + case "captive-portal-login-success": + this._log.debug("Captive portal login success. Scheduling a sync."); + CommonUtils.nextTick(() => { + this.scheduleNextSync(3000, { why: topic }); + }); + break; + case "sleep_notification": + if (this.service.engineManager.get("tabs")._tracker.modified) { + this._log.debug("Going to sleep, doing a quick sync."); + this.scheduleNextSync(0, { engines: ["tabs"], why: "sleep" }); + } + break; + } + }, + + adjustSyncInterval: function adjustSyncInterval() { + if (this.numClients <= 1) { + this._log.trace("Adjusting syncInterval to singleDeviceInterval."); + this.syncInterval = this.singleDeviceInterval; + return; + } + + // Only MULTI_DEVICE clients will enter this if statement + // since SINGLE_USER clients will be handled above. + if (this.idle) { + this._log.trace("Adjusting syncInterval to idleInterval."); + this.syncInterval = this.idleInterval; + return; + } + + if (this.hasIncomingItems) { + this._log.trace("Adjusting syncInterval to immediateInterval."); + this.hasIncomingItems = false; + this.syncInterval = this.immediateInterval; + } else { + this._log.trace("Adjusting syncInterval to activeInterval."); + this.syncInterval = this.activeInterval; + } + }, + + updateGlobalScore() { + let engines = [this.service.clientsEngine].concat( + this.service.engineManager.getEnabled() + ); + let globalScore = this.globalScore; + for (let i = 0; i < engines.length; i++) { + this._log.trace(engines[i].name + ": score: " + engines[i].score); + globalScore += engines[i].score; + engines[i]._tracker.resetScore(); + } + this.globalScore = globalScore; + this._log.trace("Global score updated: " + globalScore); + }, + + calculateScore() { + this.updateGlobalScore(); + this.checkSyncStatus(); + }, + + /** + * Query the number of known clients to figure out what mode to be in + */ + updateClientMode: function updateClientMode() { + // Nothing to do if it's the same amount + let numClients = this.numClients; + if (numClients == this.numClientsLastSync) { + return; + } + + this._log.debug( + `Client count: ${this.numClientsLastSync} -> ${numClients}` + ); + this.numClientsLastSync = numClients; + + if (numClients <= 1) { + this._log.trace("Adjusting syncThreshold to SINGLE_USER_THRESHOLD"); + this.syncThreshold = SINGLE_USER_THRESHOLD; + } else { + this._log.trace("Adjusting syncThreshold to MULTI_DEVICE_THRESHOLD"); + this.syncThreshold = MULTI_DEVICE_THRESHOLD; + } + this.adjustSyncInterval(); + }, + + /** + * Check if we should be syncing and schedule the next sync, if it's not scheduled + */ + checkSyncStatus: function checkSyncStatus() { + // Should we be syncing now, if not, cancel any sync timers and return + // if we're in backoff, we'll schedule the next sync. + let ignore = [kSyncBackoffNotMet, kSyncMasterPasswordLocked]; + let skip = this.service._checkSync(ignore); + this._log.trace('_checkSync returned "' + skip + '".'); + if (skip) { + this.clearSyncTriggers(); + return; + } + + let why = "schedule"; + // Only set the wait time to 0 if we need to sync right away + let wait; + if (this.globalScore > this.syncThreshold) { + this._log.debug("Global Score threshold hit, triggering sync."); + wait = 0; + why = "score"; + } + this.scheduleNextSync(wait, { why }); + }, + + /** + * Call sync() if Master Password is not locked. + * + * Otherwise, reschedule a sync for later. + */ + syncIfMPUnlocked(engines, why) { + // No point if we got kicked out by the master password dialog. + if (lazy.Status.login == MASTER_PASSWORD_LOCKED && Utils.mpLocked()) { + this._log.debug( + "Not initiating sync: Login status is " + lazy.Status.login + ); + + // If we're not syncing now, we need to schedule the next one. + this._log.trace( + "Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL" + ); + this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL); + return; + } + + if (!Async.isAppReady()) { + this._log.debug("Not initiating sync: app is shutting down"); + return; + } + Services.tm.dispatchToMainThread(() => { + this.service.sync({ engines, why }); + const now = Math.round(new Date().getTime() / 1000); + // Only fetch missed messages in a "scheduled" sync so we don't race against + // the Push service reconnecting on a network link change for example. + if ( + why == "schedule" && + now >= + this.missedFxACommandsLastFetch + this.missedFxACommandsFetchInterval + ) { + lazy.fxAccounts.commands + .pollDeviceCommands() + .then(() => { + this.missedFxACommandsLastFetch = now; + }) + .catch(e => { + this._log.error("Fetching missed remote commands failed.", e); + }); + } + }); + }, + + /** + * Set a timer for the next sync + */ + scheduleNextSync(interval, { engines = null, why = null } = {}) { + // If no interval was specified, use the current sync interval. + if (interval == null) { + interval = this.syncInterval; + } + + // Ensure the interval is set to no less than the backoff. + if (lazy.Status.backoffInterval && interval < lazy.Status.backoffInterval) { + this._log.trace( + "Requested interval " + + interval + + " ms is smaller than the backoff interval. " + + "Using backoff interval " + + lazy.Status.backoffInterval + + " ms instead." + ); + interval = lazy.Status.backoffInterval; + } + let nextSync = this.nextSync; + if (nextSync != 0) { + // There's already a sync scheduled. Don't reschedule if there's already + // a timer scheduled for sooner than requested. + let currentInterval = nextSync - Date.now(); + this._log.trace( + "There's already a sync scheduled in " + currentInterval + " ms." + ); + if (currentInterval < interval && this.syncTimer) { + this._log.trace( + "Ignoring scheduling request for next sync in " + interval + " ms." + ); + return; + } + } + + // Start the sync right away if we're already late. + if (interval <= 0) { + this._log.trace(`Requested sync should happen right away. (why=${why})`); + this.syncIfMPUnlocked(engines, why); + return; + } + + this._log.debug(`Next sync in ${interval} ms. (why=${why})`); + CommonUtils.namedTimer( + () => { + this.syncIfMPUnlocked(engines, why); + }, + interval, + this, + "syncTimer" + ); + + // Save the next sync time in-case sync is disabled (logout/offline/etc.) + this.nextSync = Date.now() + interval; + }, + + /** + * Incorporates the backoff/retry logic used in error handling and elective + * non-syncing. + */ + scheduleAtInterval: function scheduleAtInterval(minimumInterval) { + let interval = Utils.calculateBackoff( + this._syncErrors, + MINIMUM_BACKOFF_INTERVAL, + lazy.Status.backoffInterval + ); + if (minimumInterval) { + interval = Math.max(minimumInterval, interval); + } + + this._log.debug( + "Starting client-initiated backoff. Next sync in " + interval + " ms." + ); + this.scheduleNextSync(interval, { why: "client-backoff-schedule" }); + }, + + autoConnect: function autoConnect() { + if (this.service._checkSetup() == STATUS_OK && !this.service._checkSync()) { + // Schedule a sync based on when a previous sync was scheduled. + // scheduleNextSync() will do the right thing if that time lies in + // the past. + this.scheduleNextSync(this.nextSync - Date.now(), { why: "startup" }); + } + }, + + _syncErrors: 0, + /** + * Deal with sync errors appropriately + */ + handleSyncError: function handleSyncError() { + this._log.trace("In handleSyncError. Error count: " + this._syncErrors); + this._syncErrors++; + + // Do nothing on the first couple of failures, if we're not in + // backoff due to 5xx errors. + if (!lazy.Status.enforceBackoff) { + if (this._syncErrors < MAX_ERROR_COUNT_BEFORE_BACKOFF) { + this.scheduleNextSync(null, { why: "reschedule" }); + return; + } + this._log.debug( + "Sync error count has exceeded " + + MAX_ERROR_COUNT_BEFORE_BACKOFF + + "; enforcing backoff." + ); + lazy.Status.enforceBackoff = true; + } + + this.scheduleAtInterval(); + }, + + /** + * Remove any timers/observers that might trigger a sync + */ + clearSyncTriggers: function clearSyncTriggers() { + this._log.debug("Clearing sync triggers and the global score."); + this.globalScore = this.nextSync = 0; + + // Clear out any scheduled syncs + if (this.syncTimer) { + this.syncTimer.clear(); + } + }, +}; + +export function ErrorHandler(service) { + this.service = service; + this.init(); +} + +ErrorHandler.prototype = { + init() { + Svc.Obs.add("weave:engine:sync:applied", this); + Svc.Obs.add("weave:engine:sync:error", this); + Svc.Obs.add("weave:service:login:error", this); + Svc.Obs.add("weave:service:sync:error", this); + Svc.Obs.add("weave:service:sync:finish", this); + Svc.Obs.add("weave:service:start-over:finish", this); + + this.initLogs(); + }, + + initLogs: function initLogs() { + // Set the root Sync logger level based on a pref. All other logs will + // inherit this level unless they specifically override it. + Log.repository + .getLogger("Sync") + .manageLevelFromPref(`services.sync.log.logger`); + // And allow our specific log to have a custom level via a pref. + this._log = Log.repository.getLogger("Sync.ErrorHandler"); + this._log.manageLevelFromPref("services.sync.log.logger.service.main"); + }, + + observe(subject, topic, data) { + this._log.trace("Handling " + topic); + switch (topic) { + case "weave:engine:sync:applied": + if (subject.newFailed) { + // An engine isn't able to apply one or more incoming records. + // We don't fail hard on this, but it usually indicates a bug, + // so for now treat it as sync error (c.f. Service._syncEngine()) + lazy.Status.engines = [data, ENGINE_APPLY_FAIL]; + this._log.debug(data + " failed to apply some records."); + } + break; + case "weave:engine:sync:error": { + let exception = subject; // exception thrown by engine's sync() method + let engine_name = data; // engine name that threw the exception + + this.checkServerError(exception); + + lazy.Status.engines = [ + engine_name, + exception.failureCode || ENGINE_UNKNOWN_FAIL, + ]; + if (Async.isShutdownException(exception)) { + this._log.debug( + engine_name + + " was interrupted due to the application shutting down" + ); + } else { + this._log.debug(engine_name + " failed", exception); + } + break; + } + case "weave:service:login:error": + this._log.error("Sync encountered a login error"); + this.resetFileLog(); + break; + case "weave:service:sync:error": { + if (lazy.Status.sync == CREDENTIALS_CHANGED) { + this.service.logout(); + } + + let exception = subject; + if (Async.isShutdownException(exception)) { + // If we are shutting down we just log the fact, attempt to flush + // the log file and get out of here! + this._log.error( + "Sync was interrupted due to the application shutting down" + ); + this.resetFileLog(); + break; + } + + // Not a shutdown related exception... + this._log.error("Sync encountered an error", exception); + this.resetFileLog(); + break; + } + case "weave:service:sync:finish": + this._log.trace("Status.service is " + lazy.Status.service); + + // Check both of these status codes: in the event of a failure in one + // engine, Status.service will be SYNC_FAILED_PARTIAL despite + // Status.sync being SYNC_SUCCEEDED. + // *facepalm* + if ( + lazy.Status.sync == SYNC_SUCCEEDED && + lazy.Status.service == STATUS_OK + ) { + // Great. Let's clear our mid-sync 401 note. + this._log.trace("Clearing lastSyncReassigned."); + Svc.PrefBranch.clearUserPref("lastSyncReassigned"); + } + + if (lazy.Status.service == SYNC_FAILED_PARTIAL) { + this._log.error("Some engines did not sync correctly."); + } + this.resetFileLog(); + break; + case "weave:service:start-over:finish": + // ensure we capture any logs between the last sync and the reset completing. + this.resetFileLog() + .then(() => { + // although for privacy reasons we also delete all logs (but we allow + // a preference to avoid this to help with debugging.) + if (!Svc.PrefBranch.getBoolPref("log.keepLogsOnReset", false)) { + return logManager.removeAllLogs().then(() => { + Svc.Obs.notify("weave:service:remove-file-log"); + }); + } + return null; + }) + .catch(err => { + // So we failed to delete the logs - take the ironic option of + // writing this error to the logs we failed to delete! + this._log.error("Failed to delete logs on reset", err); + }); + break; + } + }, + + async _dumpAddons() { + // Just dump the items that sync may be concerned with. Specifically, + // active extensions that are not hidden. + let addons = []; + try { + addons = await lazy.AddonManager.getAddonsByTypes(["extension"]); + } catch (e) { + this._log.warn("Failed to dump addons", e); + } + + let relevantAddons = addons.filter(x => x.isActive && !x.hidden); + this._log.trace("Addons installed", relevantAddons.length); + for (let addon of relevantAddons) { + this._log.trace(" - ${name}, version ${version}, id ${id}", addon); + } + }, + + /** + * Generate a log file for the sync that just completed + * and refresh the input & output streams. + */ + async resetFileLog() { + // If we're writing an error log, dump extensions that may be causing problems. + if (logManager.sawError) { + await this._dumpAddons(); + } + const logType = await logManager.resetFileLog(); + if (logType == logManager.ERROR_LOG_WRITTEN) { + console.error( + "Sync encountered an error - see about:sync-log for the log file." + ); + } + Svc.Obs.notify("weave:service:reset-file-log"); + }, + + /** + * Handle HTTP response results or exceptions and set the appropriate + * Status.* bits. + * + * This method also looks for "side-channel" warnings. + */ + checkServerError(resp) { + // In this case we were passed a resolved value of Resource#_doRequest. + switch (resp.status) { + case 400: + if (resp == RESPONSE_OVER_QUOTA) { + lazy.Status.sync = OVER_QUOTA; + } + break; + + case 401: + this.service.logout(); + this._log.info("Got 401 response; resetting clusterURL."); + this.service.clusterURL = null; + + let delay = 0; + if (Svc.PrefBranch.getBoolPref("lastSyncReassigned", false)) { + // We got a 401 in the middle of the previous sync, and we just got + // another. Login must have succeeded in order for us to get here, so + // the password should be correct. + // This is likely to be an intermittent server issue, so back off and + // give it time to recover. + this._log.warn("Last sync also failed for 401. Delaying next sync."); + delay = MINIMUM_BACKOFF_INTERVAL; + } else { + this._log.debug("New mid-sync 401 failure. Making a note."); + Svc.PrefBranch.setBoolPref("lastSyncReassigned", true); + } + this._log.info("Attempting to schedule another sync."); + this.service.scheduler.scheduleNextSync(delay, { why: "reschedule" }); + break; + + case 500: + case 502: + case 503: + case 504: + lazy.Status.enforceBackoff = true; + if (resp.status == 503 && resp.headers["retry-after"]) { + let retryAfter = resp.headers["retry-after"]; + this._log.debug("Got Retry-After: " + retryAfter); + if (this.service.isLoggedIn) { + lazy.Status.sync = SERVER_MAINTENANCE; + } else { + lazy.Status.login = SERVER_MAINTENANCE; + } + Svc.Obs.notify( + "weave:service:backoff:interval", + parseInt(retryAfter, 10) + ); + } + break; + } + + // In this other case we were passed a rejection value. + switch (resp.result) { + case Cr.NS_ERROR_UNKNOWN_HOST: + case Cr.NS_ERROR_CONNECTION_REFUSED: + case Cr.NS_ERROR_NET_TIMEOUT: + case Cr.NS_ERROR_NET_RESET: + case Cr.NS_ERROR_NET_INTERRUPT: + case Cr.NS_ERROR_PROXY_CONNECTION_REFUSED: + // The constant says it's about login, but in fact it just + // indicates general network error. + if (this.service.isLoggedIn) { + lazy.Status.sync = LOGIN_FAILED_NETWORK_ERROR; + } else { + lazy.Status.login = LOGIN_FAILED_NETWORK_ERROR; + } + break; + } + }, +}; diff --git a/services/sync/modules/record.sys.mjs b/services/sync/modules/record.sys.mjs new file mode 100644 index 0000000000..7d5918a8ca --- /dev/null +++ b/services/sync/modules/record.sys.mjs @@ -0,0 +1,1335 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const CRYPTO_COLLECTION = "crypto"; +const KEYS_WBO = "keys"; + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { + DEFAULT_DOWNLOAD_BATCH_SIZE, + DEFAULT_KEYBUNDLE_NAME, +} from "resource://services-sync/constants.sys.mjs"; +import { BulkKeyBundle } from "resource://services-sync/keys.sys.mjs"; +import { Weave } from "resource://services-sync/main.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs"; + +/** + * The base class for all Sync basic storage objects (BSOs). This is the format + * used to store all records on the Sync server. In an earlier version of the + * Sync protocol, BSOs used to be called WBOs, or Weave Basic Objects. This + * class retains the old name. + * + * @class + * @param {String} collection The collection name for this BSO. + * @param {String} id The ID of this BSO. + */ +export function WBORecord(collection, id) { + this.data = {}; + this.payload = {}; + this.collection = collection; // Optional. + this.id = id; // Optional. +} + +WBORecord.prototype = { + _logName: "Sync.Record.WBO", + + get sortindex() { + if (this.data.sortindex) { + return this.data.sortindex; + } + return 0; + }, + + // Get thyself from your URI, then deserialize. + // Set thine 'response' field. + async fetch(resource) { + if (!(resource instanceof Resource)) { + throw new Error("First argument must be a Resource instance."); + } + + let r = await resource.get(); + if (r.success) { + this.deserialize(r.obj); // Warning! Muffles exceptions! + } + this.response = r; + return this; + }, + + upload(resource) { + if (!(resource instanceof Resource)) { + throw new Error("First argument must be a Resource instance."); + } + + return resource.put(this); + }, + + // Take a base URI string, with trailing slash, and return the URI of this + // WBO based on collection and ID. + uri(base) { + if (this.collection && this.id) { + let url = CommonUtils.makeURI(base + this.collection + "/" + this.id); + url.QueryInterface(Ci.nsIURL); + return url; + } + return null; + }, + + deserialize: function deserialize(json) { + if (!json || typeof json !== "object") { + throw new TypeError("Can't deserialize record from: " + json); + } + this.data = json; + try { + // The payload is likely to be JSON, but if not, keep it as a string + this.payload = JSON.parse(this.payload); + } catch (ex) {} + }, + + toJSON: function toJSON() { + // Copy fields from data to be stringified, making sure payload is a string + let obj = {}; + for (let [key, val] of Object.entries(this.data)) { + obj[key] = key == "payload" ? JSON.stringify(val) : val; + } + if (this.ttl) { + obj.ttl = this.ttl; + } + return obj; + }, + + toString: function toString() { + return ( + "{ " + + "id: " + + this.id + + " " + + "index: " + + this.sortindex + + " " + + "modified: " + + this.modified + + " " + + "ttl: " + + this.ttl + + " " + + "payload: " + + JSON.stringify(this.payload) + + " }" + ); + }, +}; + +Utils.deferGetSet(WBORecord, "data", [ + "id", + "modified", + "sortindex", + "payload", +]); + +/** + * An encrypted BSO record. This subclass handles encrypting and decrypting the + * BSO payload, but doesn't parse or interpret the cleartext string. Subclasses + * must override `transformBeforeEncrypt` and `transformAfterDecrypt` to process + * the cleartext. + * + * This class is only exposed for bridged engines, which handle serialization + * and deserialization in Rust. Sync engines implemented in JS should subclass + * `CryptoWrapper` instead, which takes care of transforming the cleartext into + * an object, and ensuring its contents are valid. + * + * @class + * @template Cleartext + * @param {String} collection The collection name for this BSO. + * @param {String} id The ID of this BSO. + */ +export function RawCryptoWrapper(collection, id) { + // Setting properties before calling the superclass constructor isn't allowed + // in new-style classes (`class MyRecord extends RawCryptoWrapper`), but + // allowed with plain functions. This is also why `defaultCleartext` is a + // method, and not simply set in the subclass constructor. + this.cleartext = this.defaultCleartext(); + WBORecord.call(this, collection, id); + this.ciphertext = null; +} + +RawCryptoWrapper.prototype = { + _logName: "Sync.Record.RawCryptoWrapper", + + /** + * Returns the default empty cleartext for this record type. This is exposed + * as a method so that subclasses can override it, and access the default + * cleartext in their constructors. `CryptoWrapper`, for example, overrides + * this to return an empty object, so that initializing the `id` in its + * constructor calls its overridden `id` setter. + * + * @returns {Cleartext} An empty cleartext. + */ + defaultCleartext() { + return null; + }, + + /** + * Transforms the cleartext into a string that can be encrypted and wrapped + * in a BSO payload. This is called before uploading the record to the server. + * + * @param {Cleartext} outgoingCleartext The cleartext to upload. + * @returns {String} The serialized cleartext. + */ + transformBeforeEncrypt(outgoingCleartext) { + throw new TypeError("Override to stringify outgoing records"); + }, + + /** + * Transforms an incoming cleartext string into an instance of the + * `Cleartext` type. This is called when fetching the record from the + * server. + * + * @param {String} incomingCleartext The decrypted cleartext string. + * @returns {Cleartext} The parsed cleartext. + */ + transformAfterDecrypt(incomingCleartext) { + throw new TypeError("Override to parse incoming records"); + }, + + ciphertextHMAC: async function ciphertextHMAC(keyBundle) { + let hmacKeyByteString = keyBundle.hmacKey; + if (!hmacKeyByteString) { + throw new Error("Cannot compute HMAC without an HMAC key."); + } + let hmacKey = CommonUtils.byteStringToArrayBuffer(hmacKeyByteString); + // NB: this.ciphertext is a base64-encoded string. For some reason this + // implementation computes the HMAC on the encoded value. + let data = CommonUtils.byteStringToArrayBuffer(this.ciphertext); + let hmac = await CryptoUtils.hmac("SHA-256", hmacKey, data); + return CommonUtils.bytesAsHex(CommonUtils.arrayBufferToByteString(hmac)); + }, + + /* + * Don't directly use the sync key. Instead, grab a key for this + * collection, which is decrypted with the sync key. + * + * Cache those keys; invalidate the cache if the time on the keys collection + * changes, or other auth events occur. + * + * Optional key bundle overrides the collection key lookup. + */ + async encrypt(keyBundle) { + if (!keyBundle) { + throw new Error("A key bundle must be supplied to encrypt."); + } + + this.IV = Weave.Crypto.generateRandomIV(); + this.ciphertext = await Weave.Crypto.encrypt( + this.transformBeforeEncrypt(this.cleartext), + keyBundle.encryptionKeyB64, + this.IV + ); + this.hmac = await this.ciphertextHMAC(keyBundle); + this.cleartext = null; + }, + + // Optional key bundle. + async decrypt(keyBundle) { + if (!this.ciphertext) { + throw new Error("No ciphertext: nothing to decrypt?"); + } + + if (!keyBundle) { + throw new Error("A key bundle must be supplied to decrypt."); + } + + // Authenticate the encrypted blob with the expected HMAC + let computedHMAC = await this.ciphertextHMAC(keyBundle); + + if (computedHMAC != this.hmac) { + Utils.throwHMACMismatch(this.hmac, computedHMAC); + } + + let cleartext = await Weave.Crypto.decrypt( + this.ciphertext, + keyBundle.encryptionKeyB64, + this.IV + ); + this.cleartext = this.transformAfterDecrypt(cleartext); + this.ciphertext = null; + + return this.cleartext; + }, +}; + +Object.setPrototypeOf(RawCryptoWrapper.prototype, WBORecord.prototype); + +Utils.deferGetSet(RawCryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]); + +/** + * An encrypted BSO record with a JSON payload. All engines implemented in JS + * should subclass this class to describe their own record types. + * + * @class + * @param {String} collection The collection name for this BSO. + * @param {String} id The ID of this BSO. + */ +export function CryptoWrapper(collection, id) { + RawCryptoWrapper.call(this, collection, id); +} + +CryptoWrapper.prototype = { + _logName: "Sync.Record.CryptoWrapper", + + defaultCleartext() { + return {}; + }, + + transformBeforeEncrypt(cleartext) { + return JSON.stringify(cleartext); + }, + + transformAfterDecrypt(cleartext) { + // Handle invalid data here. Elsewhere we assume that cleartext is an object. + let json_result = JSON.parse(cleartext); + + if (!(json_result && json_result instanceof Object)) { + throw new Error( + `Decryption failed: result is <${json_result}>, not an object.` + ); + } + + // If the payload has an encrypted id ensure it matches the requested record's id. + if (json_result.id && json_result.id != this.id) { + throw new Error(`Record id mismatch: ${json_result.id} != ${this.id}`); + } + + return json_result; + }, + + cleartextToString() { + return JSON.stringify(this.cleartext); + }, + + toString: function toString() { + let payload = this.deleted ? "DELETED" : this.cleartextToString(); + + return ( + "{ " + + "id: " + + this.id + + " " + + "index: " + + this.sortindex + + " " + + "modified: " + + this.modified + + " " + + "ttl: " + + this.ttl + + " " + + "payload: " + + payload + + " " + + "collection: " + + (this.collection || "undefined") + + " }" + ); + }, + + // The custom setter below masks the parent's getter, so explicitly call it :( + get id() { + return super.id; + }, + + // Keep both plaintext and encrypted versions of the id to verify integrity + set id(val) { + super.id = val; + this.cleartext.id = val; + }, +}; + +Object.setPrototypeOf(CryptoWrapper.prototype, RawCryptoWrapper.prototype); + +Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted"); + +/** + * An interface and caching layer for records. + */ +export function RecordManager(service) { + this.service = service; + + this._log = Log.repository.getLogger(this._logName); + this._records = {}; +} + +RecordManager.prototype = { + _recordType: CryptoWrapper, + _logName: "Sync.RecordManager", + + async import(url) { + this._log.trace("Importing record: " + (url.spec ? url.spec : url)); + try { + // Clear out the last response with empty object if GET fails + this.response = {}; + this.response = await this.service.resource(url).get(); + + // Don't parse and save the record on failure + if (!this.response.success) { + return null; + } + + let record = new this._recordType(url); + record.deserialize(this.response.obj); + + return this.set(url, record); + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.debug("Failed to import record", ex); + return null; + } + }, + + get(url) { + // Use a url string as the key to the hash + let spec = url.spec ? url.spec : url; + if (spec in this._records) { + return Promise.resolve(this._records[spec]); + } + return this.import(url); + }, + + set: function RecordMgr_set(url, record) { + let spec = url.spec ? url.spec : url; + return (this._records[spec] = record); + }, + + contains: function RecordMgr_contains(url) { + if ((url.spec || url) in this._records) { + return true; + } + return false; + }, + + clearCache: function recordMgr_clearCache() { + this._records = {}; + }, + + del: function RecordMgr_del(url) { + delete this._records[url]; + }, +}; + +/** + * Keeps track of mappings between collection names ('tabs') and KeyBundles. + * + * You can update this thing simply by giving it /info/collections. It'll + * use the last modified time to bring itself up to date. + */ +export function CollectionKeyManager(lastModified, default_, collections) { + this.lastModified = lastModified || 0; + this._default = default_ || null; + this._collections = collections || {}; + + this._log = Log.repository.getLogger("Sync.CollectionKeyManager"); +} + +// TODO: persist this locally as an Identity. Bug 610913. +// Note that the last modified time needs to be preserved. +CollectionKeyManager.prototype = { + /** + * Generate a new CollectionKeyManager that has the same attributes + * as this one. + */ + clone() { + const newCollections = {}; + for (let c in this._collections) { + newCollections[c] = this._collections[c]; + } + + return new CollectionKeyManager( + this.lastModified, + this._default, + newCollections + ); + }, + + // Return information about old vs new keys: + // * same: true if two collections are equal + // * changed: an array of collection names that changed. + _compareKeyBundleCollections: function _compareKeyBundleCollections(m1, m2) { + let changed = []; + + function process(m1, m2) { + for (let k1 in m1) { + let v1 = m1[k1]; + let v2 = m2[k1]; + if (!(v1 && v2 && v1.equals(v2))) { + changed.push(k1); + } + } + } + + // Diffs both ways. + process(m1, m2); + process(m2, m1); + + // Return a sorted, unique array. + changed.sort(); + let last; + changed = changed.filter(x => x != last && (last = x)); + return { same: !changed.length, changed }; + }, + + get isClear() { + return !this._default; + }, + + clear: function clear() { + this._log.info("Clearing collection keys..."); + this.lastModified = 0; + this._collections = {}; + this._default = null; + }, + + keyForCollection(collection) { + if (collection && this._collections[collection]) { + return this._collections[collection]; + } + + return this._default; + }, + + /** + * If `collections` (an array of strings) is provided, iterate + * over it and generate random keys for each collection. + * Create a WBO for the given data. + */ + _makeWBO(collections, defaultBundle) { + let wbo = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); + let c = {}; + for (let k in collections) { + c[k] = collections[k].keyPairB64; + } + wbo.cleartext = { + default: defaultBundle ? defaultBundle.keyPairB64 : null, + collections: c, + collection: CRYPTO_COLLECTION, + id: KEYS_WBO, + }; + return wbo; + }, + + /** + * Create a WBO for the current keys. + */ + asWBO(collection, id) { + return this._makeWBO(this._collections, this._default); + }, + + /** + * Compute a new default key, and new keys for any specified collections. + */ + async newKeys(collections) { + let newDefaultKeyBundle = await this.newDefaultKeyBundle(); + + let newColls = {}; + if (collections) { + for (let c of collections) { + let b = new BulkKeyBundle(c); + await b.generateRandom(); + newColls[c] = b; + } + } + return [newDefaultKeyBundle, newColls]; + }, + + /** + * Generates new keys, but does not replace our local copy. Use this to + * verify an upload before storing. + */ + async generateNewKeysWBO(collections) { + let newDefaultKey, newColls; + [newDefaultKey, newColls] = await this.newKeys(collections); + + return this._makeWBO(newColls, newDefaultKey); + }, + + /** + * Create a new default key. + * + * @returns {BulkKeyBundle} + */ + async newDefaultKeyBundle() { + const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME); + await key.generateRandom(); + return key; + }, + + /** + * Create a new default key and store it as this._default, since without one you cannot use setContents. + */ + async generateDefaultKey() { + this._default = await this.newDefaultKeyBundle(); + }, + + /** + * Return true if keys are already present for each of the given + * collections. + */ + hasKeysFor(collections) { + // We can't use filter() here because sometimes collections is an iterator. + for (let collection of collections) { + if (!this._collections[collection]) { + return false; + } + } + return true; + }, + + /** + * Return a new CollectionKeyManager that has keys for each of the + * given collections (creating new ones for collections where we + * don't already have keys). + */ + async ensureKeysFor(collections) { + const newKeys = Object.assign({}, this._collections); + for (let c of collections) { + if (newKeys[c]) { + continue; // don't replace existing keys + } + + const b = new BulkKeyBundle(c); + await b.generateRandom(); + newKeys[c] = b; + } + return new CollectionKeyManager(this.lastModified, this._default, newKeys); + }, + + // Take the fetched info/collections WBO, checking the change + // time of the crypto collection. + updateNeeded(info_collections) { + this._log.info( + "Testing for updateNeeded. Last modified: " + this.lastModified + ); + + // No local record of modification time? Need an update. + if (!this.lastModified) { + return true; + } + + // No keys on the server? We need an update, though our + // update handling will be a little more drastic... + if (!(CRYPTO_COLLECTION in info_collections)) { + return true; + } + + // Otherwise, we need an update if our modification time is stale. + return info_collections[CRYPTO_COLLECTION] > this.lastModified; + }, + + // + // Set our keys and modified time to the values fetched from the server. + // Returns one of three values: + // + // * If the default key was modified, return true. + // * If the default key was not modified, but per-collection keys were, + // return an array of such. + // * Otherwise, return false -- we were up-to-date. + // + setContents: function setContents(payload, modified) { + let self = this; + + this._log.info( + "Setting collection keys contents. Our last modified: " + + this.lastModified + + ", input modified: " + + modified + + "." + ); + + if (!payload) { + throw new Error("No payload in CollectionKeyManager.setContents()."); + } + + if (!payload.default) { + this._log.warn("No downloaded default key: this should not occur."); + this._log.warn("Not clearing local keys."); + throw new Error( + "No default key in CollectionKeyManager.setContents(). Cannot proceed." + ); + } + + // Process the incoming default key. + let b = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME); + b.keyPairB64 = payload.default; + let newDefault = b; + + // Process the incoming collections. + let newCollections = {}; + if ("collections" in payload) { + this._log.info("Processing downloaded per-collection keys."); + let colls = payload.collections; + for (let k in colls) { + let v = colls[k]; + if (v) { + let keyObj = new BulkKeyBundle(k); + keyObj.keyPairB64 = v; + newCollections[k] = keyObj; + } + } + } + + // Check to see if these are already our keys. + let sameDefault = this._default && this._default.equals(newDefault); + let collComparison = this._compareKeyBundleCollections( + newCollections, + this._collections + ); + let sameColls = collComparison.same; + + if (sameDefault && sameColls) { + self._log.info("New keys are the same as our old keys!"); + if (modified) { + self._log.info("Bumped local modified time."); + self.lastModified = modified; + } + return false; + } + + // Make sure things are nice and tidy before we set. + this.clear(); + + this._log.info("Saving downloaded keys."); + this._default = newDefault; + this._collections = newCollections; + + // Always trust the server. + if (modified) { + self._log.info("Bumping last modified to " + modified); + self.lastModified = modified; + } + + return sameDefault ? collComparison.changed : true; + }, + + async updateContents(syncKeyBundle, storage_keys) { + let log = this._log; + log.info("Updating collection keys..."); + + // storage_keys is a WBO, fetched from storage/crypto/keys. + // Its payload is the default key, and a map of collections to keys. + // We lazily compute the key objects from the strings we're given. + + let payload; + try { + payload = await storage_keys.decrypt(syncKeyBundle); + } catch (ex) { + log.warn("Got exception decrypting storage keys with sync key.", ex); + log.info("Aborting updateContents. Rethrowing."); + throw ex; + } + + let r = this.setContents(payload, storage_keys.modified); + log.info("Collection keys updated."); + return r; + }, +}; + +export function Collection(uri, recordObj, service) { + if (!service) { + throw new Error("Collection constructor requires a service."); + } + + Resource.call(this, uri); + + // This is a bit hacky, but gets the job done. + let res = service.resource(uri); + this.authenticator = res.authenticator; + + this._recordObj = recordObj; + this._service = service; + + this._full = false; + this._ids = null; + this._limit = 0; + this._older = 0; + this._newer = 0; + this._data = []; + // optional members used by batch upload operations. + this._batch = null; + this._commit = false; + // Used for batch download operations -- note that this is explicitly an + // opaque value and not (necessarily) a number. + this._offset = null; +} + +Collection.prototype = { + _logName: "Sync.Collection", + + _rebuildURL: function Coll__rebuildURL() { + // XXX should consider what happens if it's not a URL... + this.uri.QueryInterface(Ci.nsIURL); + + let args = []; + if (this.older) { + args.push("older=" + this.older); + } + if (this.newer) { + args.push("newer=" + this.newer); + } + if (this.full) { + args.push("full=1"); + } + if (this.sort) { + args.push("sort=" + this.sort); + } + if (this.ids != null) { + args.push("ids=" + this.ids); + } + if (this.limit > 0 && this.limit != Infinity) { + args.push("limit=" + this.limit); + } + if (this._batch) { + args.push("batch=" + encodeURIComponent(this._batch)); + } + if (this._commit) { + args.push("commit=true"); + } + if (this._offset) { + args.push("offset=" + encodeURIComponent(this._offset)); + } + + this.uri = this.uri + .mutate() + .setQuery(args.length ? "?" + args.join("&") : "") + .finalize(); + }, + + // get full items + get full() { + return this._full; + }, + set full(value) { + this._full = value; + this._rebuildURL(); + }, + + // Apply the action to a certain set of ids + get ids() { + return this._ids; + }, + set ids(value) { + this._ids = value; + this._rebuildURL(); + }, + + // Limit how many records to get + get limit() { + return this._limit; + }, + set limit(value) { + this._limit = value; + this._rebuildURL(); + }, + + // get only items modified before some date + get older() { + return this._older; + }, + set older(value) { + this._older = value; + this._rebuildURL(); + }, + + // get only items modified since some date + get newer() { + return this._newer; + }, + set newer(value) { + this._newer = value; + this._rebuildURL(); + }, + + // get items sorted by some criteria. valid values: + // oldest (oldest first) + // newest (newest first) + // index + get sort() { + return this._sort; + }, + set sort(value) { + if (value && value != "oldest" && value != "newest" && value != "index") { + throw new TypeError( + `Illegal value for sort: "${value}" (should be "oldest", "newest", or "index").` + ); + } + this._sort = value; + this._rebuildURL(); + }, + + get offset() { + return this._offset; + }, + set offset(value) { + this._offset = value; + this._rebuildURL(); + }, + + // Set information about the batch for this request. + get batch() { + return this._batch; + }, + set batch(value) { + this._batch = value; + this._rebuildURL(); + }, + + get commit() { + return this._commit; + }, + set commit(value) { + this._commit = value && true; + this._rebuildURL(); + }, + + // Similar to get(), but will page through the items `batchSize` at a time, + // deferring calling the record handler until we've gotten them all. + // + // Returns the last response processed, and doesn't run the record handler + // on any items if a non-success status is received while downloading the + // records (or if a network error occurs). + async getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) { + let totalLimit = Number(this.limit) || Infinity; + if (batchSize <= 0 || batchSize >= totalLimit) { + throw new Error("Invalid batch size"); + } + + if (!this.full) { + throw new Error("getBatched is unimplemented for guid-only GETs"); + } + + // _onComplete and _onProgress are reset after each `get` by Resource. + let { _onComplete, _onProgress } = this; + let recordBuffer = []; + let resp; + try { + let lastModifiedTime; + this.limit = batchSize; + + do { + this._onProgress = _onProgress; + this._onComplete = _onComplete; + if (batchSize + recordBuffer.length > totalLimit) { + this.limit = totalLimit - recordBuffer.length; + } + this._log.trace("Performing batched GET", { + limit: this.limit, + offset: this.offset, + }); + // Actually perform the request + resp = await this.get(); + if (!resp.success) { + recordBuffer = []; + break; + } + for (let json of resp.obj) { + let record = new this._recordObj(); + record.deserialize(json); + recordBuffer.push(record); + } + + // Initialize last modified, or check that something broken isn't happening. + let lastModified = resp.headers["x-last-modified"]; + if (!lastModifiedTime) { + lastModifiedTime = lastModified; + this.setHeader("X-If-Unmodified-Since", lastModified); + } else if (lastModified != lastModifiedTime) { + // Should be impossible -- We'd get a 412 in this case. + throw new Error( + "X-Last-Modified changed in the middle of a download batch! " + + `${lastModified} => ${lastModifiedTime}` + ); + } + + // If this is missing, we're finished. + this.offset = resp.headers["x-weave-next-offset"]; + } while (this.offset && totalLimit > recordBuffer.length); + } finally { + // Ensure we undo any temporary state so that subsequent calls to get() + // or getBatched() work properly. We do this before calling the record + // handler so that we can more convincingly pretend to be a normal get() + // call. Note: we're resetting these to the values they had before this + // function was called. + this._limit = totalLimit; + this._offset = null; + delete this._headers["x-if-unmodified-since"]; + this._rebuildURL(); + } + return { response: resp, records: recordBuffer }; + }, + + // This object only supports posting via the postQueue object. + post() { + throw new Error( + "Don't directly post to a collection - use newPostQueue instead" + ); + }, + + newPostQueue(log, timestamp, postCallback) { + let poster = (data, headers, batch, commit) => { + this.batch = batch; + this.commit = commit; + for (let [header, value] of headers) { + this.setHeader(header, value); + } + return Resource.prototype.post.call(this, data); + }; + return new PostQueue( + poster, + timestamp, + this._service.serverConfiguration || {}, + log, + postCallback + ); + }, +}; + +Object.setPrototypeOf(Collection.prototype, Resource.prototype); + +// These are limits for requests provided by the server at the +// info/configuration endpoint -- server documentation is available here: +// http://moz-services-docs.readthedocs.io/en/latest/storage/apis-1.5.html#api-instructions +// +// All are optional, however we synthesize (non-infinite) default values for the +// "max_request_bytes" and "max_record_payload_bytes" options. For the others, +// we ignore them (we treat the limit is infinite) if they're missing. +// +// These are also the only ones that all servers (even batching-disabled +// servers) should support, at least once this sync-serverstorage patch is +// everywhere https://github.com/mozilla-services/server-syncstorage/pull/74 +// +// Batching enabled servers also limit the amount of payload data and number +// of and records we can send in a single post as well as in the whole batch. +// Note that the byte limits for these there are just with respect to the +// *payload* data, e.g. the data appearing in the payload property (a +// string) of the object. +// +// Note that in practice, these limits should be sensible, but the code makes +// no assumptions about this. If we hit any of the limits, we perform the +// corresponding action (e.g. submit a request, possibly committing the +// current batch). +const DefaultPostQueueConfig = Object.freeze({ + // Number of total bytes allowed in a request + max_request_bytes: 260 * 1024, + + // Maximum number of bytes allowed in the "payload" property of a record. + max_record_payload_bytes: 256 * 1024, + + // The limit for how many bytes worth of data appearing in "payload" + // properties are allowed in a single post. + max_post_bytes: Infinity, + + // The limit for the number of records allowed in a single post. + max_post_records: Infinity, + + // The limit for how many bytes worth of data appearing in "payload" + // properties are allowed in a batch. (Same as max_post_bytes, but for + // batches). + max_total_bytes: Infinity, + + // The limit for the number of records allowed in a single post. (Same + // as max_post_records, but for batches). + max_total_records: Infinity, +}); + +// Manages a pair of (byte, count) limits for a PostQueue, such as +// (max_post_bytes, max_post_records) or (max_total_bytes, max_total_records). +class LimitTracker { + constructor(maxBytes, maxRecords) { + this.maxBytes = maxBytes; + this.maxRecords = maxRecords; + this.curBytes = 0; + this.curRecords = 0; + } + + clear() { + this.curBytes = 0; + this.curRecords = 0; + } + + canAddRecord(payloadSize) { + // The record counts are inclusive, but depending on the version of the + // server, the byte counts may or may not be inclusive (See + // https://github.com/mozilla-services/server-syncstorage/issues/73). + return ( + this.curRecords + 1 <= this.maxRecords && + this.curBytes + payloadSize < this.maxBytes + ); + } + + canNeverAdd(recordSize) { + return recordSize >= this.maxBytes; + } + + didAddRecord(recordSize) { + if (!this.canAddRecord(recordSize)) { + // This is a bug, caller is expected to call canAddRecord first. + throw new Error( + "LimitTracker.canAddRecord must be checked before adding record" + ); + } + this.curRecords += 1; + this.curBytes += recordSize; + } +} + +/* A helper to manage the posting of records while respecting the various + size limits. + + This supports the concept of a server-side "batch". The general idea is: + * We queue as many records as allowed in memory, then make a single POST. + * This first POST (optionally) gives us a batch ID, which we use for + all subsequent posts, until... + * At some point we hit a batch-maximum, and jump through a few hoops to + commit the current batch (ie, all previous POSTs) and start a new one. + * Eventually commit the final batch. + + In most cases we expect there to be exactly 1 batch consisting of possibly + multiple POSTs. +*/ +export function PostQueue(poster, timestamp, serverConfig, log, postCallback) { + // The "post" function we should use when it comes time to do the post. + this.poster = poster; + this.log = log; + + let config = Object.assign({}, DefaultPostQueueConfig, serverConfig); + + if (!serverConfig.max_request_bytes && serverConfig.max_post_bytes) { + // Use max_post_bytes for max_request_bytes if it's missing. Only needed + // until server-syncstorage/pull/74 is everywhere, and even then it's + // unnecessary if the server limits are configured sanely (there's no + // guarantee of -- at least before that is fully deployed) + config.max_request_bytes = serverConfig.max_post_bytes; + } + + this.log.trace("new PostQueue config (after defaults): ", config); + + // The callback we make with the response when we do get around to making the + // post (which could be during any of the enqueue() calls or the final flush()) + // This callback may be called multiple times and must not add new items to + // the queue. + // The second argument passed to this callback is a boolean value that is true + // if we're in the middle of a batch, and false if either the batch is + // complete, or it's a post to a server that does not understand batching. + this.postCallback = postCallback; + + // Tracks the count and combined payload size for the records we've queued + // so far but are yet to POST. + this.postLimits = new LimitTracker( + config.max_post_bytes, + config.max_post_records + ); + + // As above, but for the batch size. + this.batchLimits = new LimitTracker( + config.max_total_bytes, + config.max_total_records + ); + + // Limit for the size of `this.queued` before we do a post. + this.maxRequestBytes = config.max_request_bytes; + + // Limit for the size of incoming record payloads. + this.maxPayloadBytes = config.max_record_payload_bytes; + + // The string where we are capturing the stringified version of the records + // queued so far. It will always be invalid JSON as it is always missing the + // closing bracket. It's also used to track whether or not we've gone past + // maxRequestBytes. + this.queued = ""; + + // The ID of our current batch. Can be undefined (meaning we are yet to make + // the first post of a patch, so don't know if we have a batch), null (meaning + // we've made the first post but the server response indicated no batching + // semantics), otherwise we have made the first post and it holds the batch ID + // returned from the server. + this.batchID = undefined; + + // Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET. + this.lastModified = timestamp; +} + +PostQueue.prototype = { + async enqueue(record) { + // We want to ensure the record has a .toJSON() method defined - even + // though JSON.stringify() would implicitly call it, the stringify might + // still work even if it isn't defined, which isn't what we want. + let jsonRepr = record.toJSON(); + if (!jsonRepr) { + throw new Error( + "You must only call this with objects that explicitly support JSON" + ); + } + + let bytes = JSON.stringify(jsonRepr); + + // We use the payload size for the LimitTrackers, since that's what the + // byte limits other than max_request_bytes refer to. + let payloadLength = jsonRepr.payload.length; + + // The `+ 2` is to account for the 2-byte (maximum) overhead (one byte for + // the leading comma or "[", which all records will have, and the other for + // the final trailing "]", only present for the last record). + let encodedLength = bytes.length + 2; + + // Check first if there's some limit that indicates we cannot ever enqueue + // this record. + let isTooBig = + this.postLimits.canNeverAdd(payloadLength) || + this.batchLimits.canNeverAdd(payloadLength) || + encodedLength >= this.maxRequestBytes || + payloadLength >= this.maxPayloadBytes; + + if (isTooBig) { + return { + enqueued: false, + error: new Error("Single record too large to submit to server"), + }; + } + + let canPostRecord = this.postLimits.canAddRecord(payloadLength); + let canBatchRecord = this.batchLimits.canAddRecord(payloadLength); + let canSendRecord = + this.queued.length + encodedLength < this.maxRequestBytes; + + if (!canPostRecord || !canBatchRecord || !canSendRecord) { + this.log.trace("PostQueue flushing: ", { + canPostRecord, + canSendRecord, + canBatchRecord, + }); + // We need to write the queue out before handling this one, but we only + // commit the batch (and thus start a new one) if the record couldn't fit + // inside the batch. + await this.flush(!canBatchRecord); + } + + this.postLimits.didAddRecord(payloadLength); + this.batchLimits.didAddRecord(payloadLength); + + // Either a ',' or a '[' depending on whether this is the first record. + this.queued += this.queued.length ? "," : "["; + this.queued += bytes; + return { enqueued: true }; + }, + + async flush(finalBatchPost) { + if (!this.queued) { + // nothing queued - we can't be in a batch, and something has gone very + // bad if we think we are. + if (this.batchID) { + throw new Error( + `Flush called when no queued records but we are in a batch ${this.batchID}` + ); + } + return; + } + // the batch query-param and headers we'll send. + let batch; + let headers = []; + if (this.batchID === undefined) { + // First commit in a (possible) batch. + batch = "true"; + } else if (this.batchID) { + // We have an existing batch. + batch = this.batchID; + } else { + // Not the first post and we know we have no batch semantics. + batch = null; + } + + headers.push(["x-if-unmodified-since", this.lastModified]); + + let numQueued = this.postLimits.curRecords; + this.log.info( + `Posting ${numQueued} records of ${ + this.queued.length + 1 + } bytes with batch=${batch}` + ); + let queued = this.queued + "]"; + if (finalBatchPost) { + this.batchLimits.clear(); + } + this.postLimits.clear(); + this.queued = ""; + let response = await this.poster( + queued, + headers, + batch, + !!(finalBatchPost && this.batchID !== null) + ); + + if (!response.success) { + this.log.trace("Server error response during a batch", response); + // not clear what we should do here - we expect the consumer of this to + // abort by throwing in the postCallback below. + await this.postCallback(this, response, !finalBatchPost); + return; + } + + if (finalBatchPost) { + this.log.trace("Committed batch", this.batchID); + this.batchID = undefined; // we are now in "first post for the batch" state. + this.lastModified = response.headers["x-last-modified"]; + await this.postCallback(this, response, false); + return; + } + + if (response.status != 202) { + if (this.batchID) { + throw new Error( + "Server responded non-202 success code while a batch was in progress" + ); + } + this.batchID = null; // no batch semantics are in place. + this.lastModified = response.headers["x-last-modified"]; + await this.postCallback(this, response, false); + return; + } + + // this response is saying the server has batch semantics - we should + // always have a batch ID in the response. + let responseBatchID = response.obj.batch; + this.log.trace("Server responsed 202 with batch", responseBatchID); + if (!responseBatchID) { + this.log.error( + "Invalid server response: 202 without a batch ID", + response + ); + throw new Error("Invalid server response: 202 without a batch ID"); + } + + if (this.batchID === undefined) { + this.batchID = responseBatchID; + if (!this.lastModified) { + this.lastModified = response.headers["x-last-modified"]; + if (!this.lastModified) { + throw new Error("Batch response without x-last-modified"); + } + } + } + + if (this.batchID != responseBatchID) { + throw new Error( + `Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}` + ); + } + + await this.postCallback(this, response, true); + }, +}; diff --git a/services/sync/modules/resource.sys.mjs b/services/sync/modules/resource.sys.mjs new file mode 100644 index 0000000000..537ffd5219 --- /dev/null +++ b/services/sync/modules/resource.sys.mjs @@ -0,0 +1,292 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Observers } from "resource://services-common/observers.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; +import { setTimeout, clearTimeout } from "resource://gre/modules/Timer.sys.mjs"; + +/* global AbortController */ + +/* + * Resource represents a remote network resource, identified by a URI. + * Create an instance like so: + * + * let resource = new Resource("http://foobar.com/path/to/resource"); + * + * The 'resource' object has the following methods to issue HTTP requests + * of the corresponding HTTP methods: + * + * get(callback) + * put(data, callback) + * post(data, callback) + * delete(callback) + */ +export function Resource(uri) { + this._log = Log.repository.getLogger(this._logName); + this._log.manageLevelFromPref("services.sync.log.logger.network.resources"); + this.uri = uri; + this._headers = {}; +} + +// (static) Caches the latest server timestamp (X-Weave-Timestamp header). +Resource.serverTime = null; + +XPCOMUtils.defineLazyPreferenceGetter( + Resource, + "SEND_VERSION_INFO", + "services.sync.sendVersionInfo", + true +); +Resource.prototype = { + _logName: "Sync.Resource", + + /** + * Callback to be invoked at request time to add authentication details. + * If the callback returns a promise, it will be awaited upon. + * + * By default, a global authenticator is provided. If this is set, it will + * be used instead of the global one. + */ + authenticator: null, + + // Wait 5 minutes before killing a request. + ABORT_TIMEOUT: 300000, + + // Headers to be included when making a request for the resource. + // Note: Header names should be all lower case, there's no explicit + // check for duplicates due to case! + get headers() { + return this._headers; + }, + set headers(_) { + throw new Error("headers can't be mutated directly. Please use setHeader."); + }, + setHeader(header, value) { + this._headers[header.toLowerCase()] = value; + }, + + // URI representing this resource. + get uri() { + return this._uri; + }, + set uri(value) { + if (typeof value == "string") { + this._uri = CommonUtils.makeURI(value); + } else { + this._uri = value; + } + }, + + // Get the string representation of the URI. + get spec() { + if (this._uri) { + return this._uri.spec; + } + return null; + }, + + /** + * @param {string} method HTTP method + * @returns {Headers} + */ + async _buildHeaders(method) { + const headers = new Headers(this._headers); + + if (Resource.SEND_VERSION_INFO) { + headers.append("user-agent", Utils.userAgent); + } + + if (this.authenticator) { + const result = await this.authenticator(this, method); + if (result && result.headers) { + for (const [k, v] of Object.entries(result.headers)) { + headers.append(k.toLowerCase(), v); + } + } + } else { + this._log.debug("No authenticator found."); + } + + // PUT and POST are treated differently because they have payload data. + if (("PUT" == method || "POST" == method) && !headers.has("content-type")) { + headers.append("content-type", "text/plain"); + } + + if (this._log.level <= Log.Level.Trace) { + for (const [k, v] of headers) { + if (k == "authorization" || k == "x-client-state") { + this._log.trace(`HTTP Header ${k}: ***** (suppressed)`); + } else { + this._log.trace(`HTTP Header ${k}: ${v}`); + } + } + } + + if (!headers.has("accept")) { + headers.append("accept", "application/json;q=0.9,*/*;q=0.2"); + } + + return headers; + }, + + /** + * @param {string} method HTTP method + * @param {string} data HTTP body + * @param {object} signal AbortSignal instance + * @returns {Request} + */ + async _createRequest(method, data, signal) { + const headers = await this._buildHeaders(method); + const init = { + cache: "no-store", // No cache. + headers, + method, + signal, + mozErrors: true, // Return nsresult error codes instead of a generic + // NetworkError when fetch rejects. + }; + + if (data) { + if (!(typeof data == "string" || data instanceof String)) { + data = JSON.stringify(data); + } + this._log.debug(`${method} Length: ${data.length}`); + this._log.trace(`${method} Body: ${data}`); + init.body = data; + } + return new Request(this.uri.spec, init); + }, + + /** + * @param {string} method HTTP method + * @param {string} [data] HTTP body + * @returns {Response} + */ + async _doRequest(method, data = null) { + const controller = new AbortController(); + const request = await this._createRequest(method, data, controller.signal); + const responsePromise = fetch(request); // Rejects on network failure. + let didTimeout = false; + const timeoutId = setTimeout(() => { + didTimeout = true; + this._log.error( + `Request timed out after ${this.ABORT_TIMEOUT}ms. Aborting.` + ); + controller.abort(); + }, this.ABORT_TIMEOUT); + let response; + try { + response = await responsePromise; + } catch (e) { + this._log.warn(`${method} request to ${this.uri.spec} failed`, e); + if (!didTimeout) { + throw e; + } + throw Components.Exception( + "Request aborted (timeout)", + Cr.NS_ERROR_NET_TIMEOUT + ); + } finally { + clearTimeout(timeoutId); + } + return this._processResponse(response, method); + }, + + async _processResponse(response, method) { + const data = await response.text(); + this._logResponse(response, method, data); + this._processResponseHeaders(response); + + const ret = { + data, + url: response.url, + status: response.status, + success: response.ok, + headers: {}, + }; + for (const [k, v] of response.headers) { + ret.headers[k] = v; + } + + // Make a lazy getter to convert the json response into an object. + // Note that this can cause a parse error to be thrown far away from the + // actual fetch, so be warned! + ChromeUtils.defineLazyGetter(ret, "obj", () => { + try { + return JSON.parse(ret.data); + } catch (ex) { + this._log.warn("Got exception parsing response body", ex); + // Stringify to avoid possibly printing non-printable characters. + this._log.debug( + "Parse fail: Response body starts", + (ret.data + "").slice(0, 100) + ); + throw ex; + } + }); + + return ret; + }, + + _logResponse(response, method, data) { + const { status, ok: success, url } = response; + + // Log the status of the request. + this._log.debug( + `${method} ${success ? "success" : "fail"} ${status} ${url}` + ); + + // Additionally give the full response body when Trace logging. + if (this._log.level <= Log.Level.Trace) { + this._log.trace(`${method} body`, data); + } + + if (!success) { + this._log.warn( + `${method} request to ${url} failed with status ${status}` + ); + } + }, + + _processResponseHeaders({ headers, ok: success }) { + if (headers.has("x-weave-timestamp")) { + Resource.serverTime = parseFloat(headers.get("x-weave-timestamp")); + } + // This is a server-side safety valve to allow slowing down + // clients without hurting performance. + if (headers.has("x-weave-backoff")) { + let backoff = headers.get("x-weave-backoff"); + this._log.debug(`Got X-Weave-Backoff: ${backoff}`); + Observers.notify("weave:service:backoff:interval", parseInt(backoff, 10)); + } + + if (success && headers.has("x-weave-quota-remaining")) { + Observers.notify( + "weave:service:quota:remaining", + parseInt(headers.get("x-weave-quota-remaining"), 10) + ); + } + }, + + get() { + return this._doRequest("GET"); + }, + + put(data) { + return this._doRequest("PUT", data); + }, + + post(data) { + return this._doRequest("POST", data); + }, + + delete() { + return this._doRequest("DELETE"); + }, +}; diff --git a/services/sync/modules/service.sys.mjs b/services/sync/modules/service.sys.mjs new file mode 100644 index 0000000000..97ba0d32cd --- /dev/null +++ b/services/sync/modules/service.sys.mjs @@ -0,0 +1,1643 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const CRYPTO_COLLECTION = "crypto"; +const KEYS_WBO = "keys"; + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { + CLIENT_NOT_CONFIGURED, + CREDENTIALS_CHANGED, + HMAC_EVENT_INTERVAL, + LOGIN_FAILED, + LOGIN_FAILED_INVALID_PASSPHRASE, + LOGIN_FAILED_NETWORK_ERROR, + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_FAILED_NO_USERNAME, + LOGIN_FAILED_SERVER_ERROR, + LOGIN_SUCCEEDED, + MASTER_PASSWORD_LOCKED, + METARECORD_DOWNLOAD_FAIL, + NO_SYNC_NODE_FOUND, + PREFS_BRANCH, + STATUS_DISABLED, + STATUS_OK, + STORAGE_VERSION, + VERSION_OUT_OF_DATE, + WEAVE_VERSION, + kFirefoxShuttingDown, + kFirstSyncChoiceNotMade, + kSyncBackoffNotMet, + kSyncMasterPasswordLocked, + kSyncNetworkOffline, + kSyncNotConfigured, + kSyncWeaveDisabled, +} from "resource://services-sync/constants.sys.mjs"; + +import { EngineManager } from "resource://services-sync/engines.sys.mjs"; +import { ClientEngine } from "resource://services-sync/engines/clients.sys.mjs"; +import { Weave } from "resource://services-sync/main.sys.mjs"; +import { + ErrorHandler, + SyncScheduler, +} from "resource://services-sync/policies.sys.mjs"; +import { + CollectionKeyManager, + CryptoWrapper, + RecordManager, + WBORecord, +} from "resource://services-sync/record.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { EngineSynchronizer } from "resource://services-sync/stages/enginesync.sys.mjs"; +import { DeclinedEngines } from "resource://services-sync/stages/declined.sys.mjs"; +import { Status } from "resource://services-sync/status.sys.mjs"; + +ChromeUtils.importESModule("resource://services-sync/telemetry.sys.mjs"); +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs"; + +const fxAccounts = getFxAccountsSingleton(); + +function getEngineModules() { + let result = { + Addons: { module: "addons.sys.mjs", symbol: "AddonsEngine" }, + Password: { module: "passwords.sys.mjs", symbol: "PasswordEngine" }, + Prefs: { module: "prefs.sys.mjs", symbol: "PrefsEngine" }, + }; + if (AppConstants.MOZ_APP_NAME != "thunderbird") { + result.Bookmarks = { + module: "bookmarks.sys.mjs", + symbol: "BookmarksEngine", + }; + result.Form = { module: "forms.sys.mjs", symbol: "FormEngine" }; + result.History = { module: "history.sys.mjs", symbol: "HistoryEngine" }; + result.Tab = { module: "tabs.sys.mjs", symbol: "TabEngine" }; + } + if (Svc.PrefBranch.getBoolPref("engine.addresses.available", false)) { + result.Addresses = { + module: "resource://autofill/FormAutofillSync.sys.mjs", + symbol: "AddressesEngine", + }; + } + if (Svc.PrefBranch.getBoolPref("engine.creditcards.available", false)) { + result.CreditCards = { + module: "resource://autofill/FormAutofillSync.sys.mjs", + symbol: "CreditCardsEngine", + }; + } + result["Extension-Storage"] = { + module: "extension-storage.sys.mjs", + controllingPref: "webextensions.storage.sync.kinto", + whenTrue: "ExtensionStorageEngineKinto", + whenFalse: "ExtensionStorageEngineBridge", + }; + return result; +} + +const lazy = {}; + +// A unique identifier for this browser session. Used for logging so +// we can easily see whether 2 logs are in the same browser session or +// after the browser restarted. +ChromeUtils.defineLazyGetter(lazy, "browserSessionID", Utils.makeGUID); + +function Sync11Service() { + this._notify = Utils.notify("weave:service:"); + Utils.defineLazyIDProperty(this, "syncID", "services.sync.client.syncID"); +} +Sync11Service.prototype = { + _lock: Utils.lock, + _locked: false, + _loggedIn: false, + + infoURL: null, + storageURL: null, + metaURL: null, + cryptoKeyURL: null, + // The cluster URL comes via the identity object, which in the FxA + // world is ebbedded in the token returned from the token server. + _clusterURL: null, + + get clusterURL() { + return this._clusterURL || ""; + }, + set clusterURL(value) { + if (value != null && typeof value != "string") { + throw new Error("cluster must be a string, got " + typeof value); + } + this._clusterURL = value; + this._updateCachedURLs(); + }, + + get isLoggedIn() { + return this._loggedIn; + }, + + get locked() { + return this._locked; + }, + lock: function lock() { + if (this._locked) { + return false; + } + this._locked = true; + return true; + }, + unlock: function unlock() { + this._locked = false; + }, + + // A specialized variant of Utils.catch. + // This provides a more informative error message when we're already syncing: + // see Bug 616568. + _catch(func) { + function lockExceptions(ex) { + if (Utils.isLockException(ex)) { + // This only happens if we're syncing already. + this._log.info("Cannot start sync: already syncing?"); + } + } + + return Utils.catch.call(this, func, lockExceptions); + }, + + get userBaseURL() { + // The user URL is the cluster URL. + return this.clusterURL; + }, + + _updateCachedURLs: function _updateCachedURLs() { + // Nothing to cache yet if we don't have the building blocks + if (!this.clusterURL) { + // Also reset all other URLs used by Sync to ensure we aren't accidentally + // using one cached earlier - if there's no cluster URL any cached ones + // are invalid. + this.infoURL = undefined; + this.storageURL = undefined; + this.metaURL = undefined; + this.cryptoKeysURL = undefined; + return; + } + + this._log.debug( + "Caching URLs under storage user base: " + this.userBaseURL + ); + + // Generate and cache various URLs under the storage API for this user + this.infoURL = this.userBaseURL + "info/collections"; + this.storageURL = this.userBaseURL + "storage/"; + this.metaURL = this.storageURL + "meta/global"; + this.cryptoKeysURL = this.storageURL + CRYPTO_COLLECTION + "/" + KEYS_WBO; + }, + + _checkCrypto: function _checkCrypto() { + let ok = false; + + try { + let iv = Weave.Crypto.generateRandomIV(); + if (iv.length == 24) { + ok = true; + } + } catch (e) { + this._log.debug("Crypto check failed: " + e); + } + + return ok; + }, + + /** + * Here is a disgusting yet reasonable way of handling HMAC errors deep in + * the guts of Sync. The astute reader will note that this is a hacky way of + * implementing something like continuable conditions. + * + * A handler function is glued to each engine. If the engine discovers an + * HMAC failure, we fetch keys from the server and update our keys, just as + * we would on startup. + * + * If our key collection changed, we signal to the engine (via our return + * value) that it should retry decryption. + * + * If our key collection did not change, it means that we already had the + * correct keys... and thus a different client has the wrong ones. Reupload + * the bundle that we fetched, which will bump the modified time on the + * server and (we hope) prompt a broken client to fix itself. + * + * We keep track of the time at which we last applied this reasoning, because + * thrashing doesn't solve anything. We keep a reasonable interval between + * these remedial actions. + */ + lastHMACEvent: 0, + + /* + * Returns whether to try again. + */ + async handleHMACEvent() { + let now = Date.now(); + + // Leave a sizable delay between HMAC recovery attempts. This gives us + // time for another client to fix themselves if we touch the record. + if (now - this.lastHMACEvent < HMAC_EVENT_INTERVAL) { + return false; + } + + this._log.info( + "Bad HMAC event detected. Attempting recovery " + + "or signaling to other clients." + ); + + // Set the last handled time so that we don't act again. + this.lastHMACEvent = now; + + // Fetch keys. + let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); + try { + let cryptoResp = ( + await cryptoKeys.fetch(this.resource(this.cryptoKeysURL)) + ).response; + + // Save out the ciphertext for when we reupload. If there's a bug in + // CollectionKeyManager, this will prevent us from uploading junk. + let cipherText = cryptoKeys.ciphertext; + + if (!cryptoResp.success) { + this._log.warn("Failed to download keys."); + return false; + } + + let keysChanged = await this.handleFetchedKeys( + this.identity.syncKeyBundle, + cryptoKeys, + true + ); + if (keysChanged) { + // Did they change? If so, carry on. + this._log.info("Suggesting retry."); + return true; // Try again. + } + + // If not, reupload them and continue the current sync. + cryptoKeys.ciphertext = cipherText; + cryptoKeys.cleartext = null; + + let uploadResp = await this._uploadCryptoKeys( + cryptoKeys, + cryptoResp.obj.modified + ); + if (uploadResp.success) { + this._log.info("Successfully re-uploaded keys. Continuing sync."); + } else { + this._log.warn( + "Got error response re-uploading keys. " + + "Continuing sync; let's try again later." + ); + } + + return false; // Don't try again: same keys. + } catch (ex) { + this._log.warn( + "Got exception fetching and handling crypto keys. " + + "Will try again later.", + ex + ); + return false; + } + }, + + async handleFetchedKeys(syncKey, cryptoKeys, skipReset) { + // Don't want to wipe if we're just starting up! + let wasBlank = this.collectionKeys.isClear; + let keysChanged = await this.collectionKeys.updateContents( + syncKey, + cryptoKeys + ); + + if (keysChanged && !wasBlank) { + this._log.debug("Keys changed: " + JSON.stringify(keysChanged)); + + if (!skipReset) { + this._log.info("Resetting client to reflect key change."); + + if (keysChanged.length) { + // Collection keys only. Reset individual engines. + await this.resetClient(keysChanged); + } else { + // Default key changed: wipe it all. + await this.resetClient(); + } + + this._log.info("Downloaded new keys, client reset. Proceeding."); + } + return true; + } + return false; + }, + + /** + * Prepare to initialize the rest of Weave after waiting a little bit + */ + async onStartup() { + this.status = Status; + this.identity = Status._authManager; + this.collectionKeys = new CollectionKeyManager(); + + this.scheduler = new SyncScheduler(this); + this.errorHandler = new ErrorHandler(this); + + this._log = Log.repository.getLogger("Sync.Service"); + this._log.manageLevelFromPref("services.sync.log.logger.service.main"); + + this._log.info("Loading Weave " + WEAVE_VERSION); + + this.recordManager = new RecordManager(this); + + this.enabled = true; + + await this._registerEngines(); + + let ua = Cc["@mozilla.org/network/protocol;1?name=http"].getService( + Ci.nsIHttpProtocolHandler + ).userAgent; + this._log.info(ua); + + if (!this._checkCrypto()) { + this.enabled = false; + this._log.info( + "Could not load the Weave crypto component. Disabling " + + "Weave, since it will not work correctly." + ); + } + + Svc.Obs.add("weave:service:setup-complete", this); + Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon + Svc.Obs.add("fxaccounts:device_disconnected", this); + Services.prefs.addObserver(PREFS_BRANCH + "engine.", this); + + if (!this.enabled) { + this._log.info("Firefox Sync disabled."); + } + + this._updateCachedURLs(); + + let status = this._checkSetup(); + if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) { + this._startTracking(); + } + + // Send an event now that Weave service is ready. We don't do this + // synchronously so that observers can import this module before + // registering an observer. + CommonUtils.nextTick(() => { + this.status.ready = true; + + // UI code uses the flag on the XPCOM service so it doesn't have + // to load a bunch of modules. + let xps = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + xps.ready = true; + + Svc.Obs.notify("weave:service:ready"); + }); + }, + + _checkSetup: function _checkSetup() { + if (!this.enabled) { + return (this.status.service = STATUS_DISABLED); + } + return this.status.checkSetup(); + }, + + /** + * Register the built-in engines for certain applications + */ + async _registerEngines() { + this.engineManager = new EngineManager(this); + + let engineModules = getEngineModules(); + + let engines = []; + // We allow a pref, which has no default value, to limit the engines + // which are registered. We expect only tests will use this. + if ( + Svc.PrefBranch.getPrefType("registerEngines") != + Ci.nsIPrefBranch.PREF_INVALID + ) { + engines = Svc.PrefBranch.getStringPref("registerEngines").split(","); + this._log.info("Registering custom set of engines", engines); + } else { + // default is all engines. + engines = Object.keys(engineModules); + } + + let declined = []; + let pref = Svc.PrefBranch.getStringPref("declinedEngines", null); + if (pref) { + declined = pref.split(","); + } + + let clientsEngine = new ClientEngine(this); + // Ideally clientsEngine should not exist + // (or be a promise that calls initialize() before returning the engine) + await clientsEngine.initialize(); + this.clientsEngine = clientsEngine; + + for (let name of engines) { + if (!(name in engineModules)) { + this._log.info("Do not know about engine: " + name); + continue; + } + let modInfo = engineModules[name]; + if (!modInfo.module.includes(":")) { + modInfo.module = "resource://services-sync/engines/" + modInfo.module; + } + try { + let ns = ChromeUtils.importESModule(modInfo.module); + if (modInfo.symbol) { + let symbol = modInfo.symbol; + if (!(symbol in ns)) { + this._log.warn( + "Could not find exported engine instance: " + symbol + ); + continue; + } + await this.engineManager.register(ns[symbol]); + } else { + let { whenTrue, whenFalse, controllingPref } = modInfo; + if (!(whenTrue in ns) || !(whenFalse in ns)) { + this._log.warn("Could not find all exported engine instances", { + whenTrue, + whenFalse, + }); + continue; + } + await this.engineManager.registerAlternatives( + name.toLowerCase(), + controllingPref, + ns[whenTrue], + ns[whenFalse] + ); + } + } catch (ex) { + this._log.warn("Could not register engine " + name, ex); + } + } + + this.engineManager.setDeclined(declined); + }, + + /** + * This method updates the local engines state from an existing meta/global + * when Sync is disabled. + * Running this code if sync is enabled would end up in very weird results + * (but we're nice and we check before doing anything!). + */ + async updateLocalEnginesState() { + await this.promiseInitialized; + + // Sanity check, this method is not meant to be run if Sync is enabled! + if (Svc.PrefBranch.getStringPref("username", "")) { + throw new Error("Sync is enabled!"); + } + + // For historical reasons the behaviour of setCluster() is bizarre, + // so just check what we care about - the meta URL. + if (!this.metaURL) { + await this.identity.setCluster(); + if (!this.metaURL) { + this._log.warn("Could not find a cluster."); + return; + } + } + // Clear the cache so we always fetch the latest meta/global. + this.recordManager.clearCache(); + let meta = await this.recordManager.get(this.metaURL); + if (!meta) { + this._log.info("Meta record is null, aborting engine state update."); + return; + } + const declinedEngines = meta.payload.declined; + const allEngines = this.engineManager.getAll().map(e => e.name); + // We don't want our observer of the enabled prefs to treat the change as + // a user-change, otherwise we will do the wrong thing with declined etc. + this._ignorePrefObserver = true; + try { + for (const engine of allEngines) { + Svc.PrefBranch.setBoolPref( + `engine.${engine}`, + !declinedEngines.includes(engine) + ); + } + } finally { + this._ignorePrefObserver = false; + } + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + observe(subject, topic, data) { + switch (topic) { + // Ideally this observer should be in the SyncScheduler, but it would require + // some work to know about the sync specific engines. We should move this there once it does. + case "sync:collection_changed": + // We check if we're running TPS here to avoid TPS failing because it + // couldn't get to get the sync lock, due to us currently syncing the + // clients engine. + if ( + data.includes("clients") && + !Svc.PrefBranch.getBoolPref("testing.tps", false) + ) { + // Sync in the background (it's fine not to wait on the returned promise + // because sync() has a lock). + // [] = clients collection only + this.sync({ why: "collection_changed", engines: [] }).catch(e => { + this._log.error(e); + }); + } + break; + case "fxaccounts:device_disconnected": + data = JSON.parse(data); + if (!data.isLocalDevice) { + // Refresh the known stale clients list in the background. + this.clientsEngine.updateKnownStaleClients().catch(e => { + this._log.error(e); + }); + } + break; + case "weave:service:setup-complete": + let status = this._checkSetup(); + if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) { + this._startTracking(); + } + break; + case "nsPref:changed": + if (this._ignorePrefObserver) { + return; + } + const engine = data.slice((PREFS_BRANCH + "engine.").length); + if (engine.includes(".")) { + // A sub-preference of the engine was changed. For example + // `services.sync.engine.bookmarks.validation.percentageChance`. + return; + } + this._handleEngineStatusChanged(engine); + break; + } + }, + + _handleEngineStatusChanged(engine) { + this._log.trace("Status for " + engine + " engine changed."); + if (Svc.PrefBranch.getBoolPref("engineStatusChanged." + engine, false)) { + // The enabled status being changed back to what it was before. + Svc.PrefBranch.clearUserPref("engineStatusChanged." + engine); + } else { + // Remember that the engine status changed locally until the next sync. + Svc.PrefBranch.setBoolPref("engineStatusChanged." + engine, true); + } + }, + + _startTracking() { + const engines = [this.clientsEngine, ...this.engineManager.getAll()]; + for (let engine of engines) { + try { + engine.startTracking(); + } catch (e) { + this._log.error(`Could not start ${engine.name} engine tracker`, e); + } + } + // This is for TPS. We should try to do better. + Svc.Obs.notify("weave:service:tracking-started"); + }, + + async _stopTracking() { + const engines = [this.clientsEngine, ...this.engineManager.getAll()]; + for (let engine of engines) { + try { + await engine.stopTracking(); + } catch (e) { + this._log.error(`Could not stop ${engine.name} engine tracker`, e); + } + } + Svc.Obs.notify("weave:service:tracking-stopped"); + }, + + /** + * Obtain a Resource instance with authentication credentials. + */ + resource: function resource(url) { + let res = new Resource(url); + res.authenticator = this.identity.getResourceAuthenticator(); + + return res; + }, + + /** + * Perform the info fetch as part of a login or key fetch, or + * inside engine sync. + */ + async _fetchInfo(url) { + let infoURL = url || this.infoURL; + + this._log.trace("In _fetchInfo: " + infoURL); + let info; + try { + info = await this.resource(infoURL).get(); + } catch (ex) { + this.errorHandler.checkServerError(ex); + throw ex; + } + + // Always check for errors. + this.errorHandler.checkServerError(info); + if (!info.success) { + this._log.error("Aborting sync: failed to get collections."); + throw info; + } + return info; + }, + + async verifyAndFetchSymmetricKeys(infoResponse) { + this._log.debug( + "Fetching and verifying -- or generating -- symmetric keys." + ); + + let syncKeyBundle = this.identity.syncKeyBundle; + if (!syncKeyBundle) { + this.status.login = LOGIN_FAILED_NO_PASSPHRASE; + this.status.sync = CREDENTIALS_CHANGED; + return false; + } + + try { + if (!infoResponse) { + infoResponse = await this._fetchInfo(); // Will throw an exception on failure. + } + + // This only applies when the server is already at version 4. + if (infoResponse.status != 200) { + this._log.warn( + "info/collections returned non-200 response. Failing key fetch." + ); + this.status.login = LOGIN_FAILED_SERVER_ERROR; + this.errorHandler.checkServerError(infoResponse); + return false; + } + + let infoCollections = infoResponse.obj; + + this._log.info( + "Testing info/collections: " + JSON.stringify(infoCollections) + ); + + if (this.collectionKeys.updateNeeded(infoCollections)) { + this._log.info("collection keys reports that a key update is needed."); + + // Don't always set to CREDENTIALS_CHANGED -- we will probably take care of this. + + // Fetch storage/crypto/keys. + let cryptoKeys; + + if (infoCollections && CRYPTO_COLLECTION in infoCollections) { + try { + cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); + let cryptoResp = ( + await cryptoKeys.fetch(this.resource(this.cryptoKeysURL)) + ).response; + + if (cryptoResp.success) { + await this.handleFetchedKeys(syncKeyBundle, cryptoKeys); + return true; + } else if (cryptoResp.status == 404) { + // On failure, ask to generate new keys and upload them. + // Fall through to the behavior below. + this._log.warn( + "Got 404 for crypto/keys, but 'crypto' in info/collections. Regenerating." + ); + cryptoKeys = null; + } else { + // Some other problem. + this.status.login = LOGIN_FAILED_SERVER_ERROR; + this.errorHandler.checkServerError(cryptoResp); + this._log.warn( + "Got status " + cryptoResp.status + " fetching crypto keys." + ); + return false; + } + } catch (ex) { + this._log.warn("Got exception fetching cryptoKeys.", ex); + // TODO: Um, what exceptions might we get here? Should we re-throw any? + + // One kind of exception: HMAC failure. + if (Utils.isHMACMismatch(ex)) { + this.status.login = LOGIN_FAILED_INVALID_PASSPHRASE; + this.status.sync = CREDENTIALS_CHANGED; + } else { + // In the absence of further disambiguation or more precise + // failure constants, just report failure. + this.status.login = LOGIN_FAILED; + } + return false; + } + } else { + this._log.info( + "... 'crypto' is not a reported collection. Generating new keys." + ); + } + + if (!cryptoKeys) { + this._log.info("No keys! Generating new ones."); + + // Better make some and upload them, and wipe the server to ensure + // consistency. This is all achieved via _freshStart. + // If _freshStart fails to clear the server or upload keys, it will + // throw. + await this._freshStart(); + return true; + } + + // Last-ditch case. + return false; + } + // No update needed: we're good! + return true; + } catch (ex) { + // This means no keys are present, or there's a network error. + this._log.debug("Failed to fetch and verify keys", ex); + this.errorHandler.checkServerError(ex); + return false; + } + }, + + getMaxRecordPayloadSize() { + let config = this.serverConfiguration; + if (!config || !config.max_record_payload_bytes) { + this._log.warn( + "No config or incomplete config in getMaxRecordPayloadSize." + + " Are we running tests?" + ); + return 256 * 1024; + } + let payloadMax = config.max_record_payload_bytes; + if (config.max_post_bytes && payloadMax <= config.max_post_bytes) { + return config.max_post_bytes - 4096; + } + return payloadMax; + }, + + getMemcacheMaxRecordPayloadSize() { + // Collections stored in memcached ("tabs", "clients" or "meta") have a + // different max size than ones stored in the normal storage server db. + // In practice, the real limit here is 1M (bug 1300451 comment 40), but + // there's overhead involved that is hard to calculate on the client, so we + // use 512k to be safe (at the recommendation of the server team). Note + // that if the server reports a lower limit (via info/configuration), we + // respect that limit instead. See also bug 1403052. + return Math.min(512 * 1024, this.getMaxRecordPayloadSize()); + }, + + async verifyLogin(allow40XRecovery = true) { + // Attaching auth credentials to a request requires access to + // passwords, which means that Resource.get can throw MP-related + // exceptions! + // So we ask the identity to verify the login state after unlocking the + // master password (ie, this call is expected to prompt for MP unlock + // if necessary) while we still have control. + this.status.login = await this.identity.unlockAndVerifyAuthState(); + this._log.debug( + "Fetching unlocked auth state returned " + this.status.login + ); + if (this.status.login != STATUS_OK) { + return false; + } + + try { + // Make sure we have a cluster to verify against. + // This is a little weird, if we don't get a node we pretend + // to succeed, since that probably means we just don't have storage. + if (this.clusterURL == "" && !(await this.identity.setCluster())) { + this.status.sync = NO_SYNC_NODE_FOUND; + return true; + } + + // Fetch collection info on every startup. + let test = await this.resource(this.infoURL).get(); + + switch (test.status) { + case 200: + // The user is authenticated. + + // We have no way of verifying the passphrase right now, + // so wait until remoteSetup to do so. + // Just make the most trivial checks. + if (!this.identity.syncKeyBundle) { + this._log.warn("No passphrase in verifyLogin."); + this.status.login = LOGIN_FAILED_NO_PASSPHRASE; + return false; + } + + // Go ahead and do remote setup, so that we can determine + // conclusively that our passphrase is correct. + if (await this._remoteSetup(test)) { + // Username/password verified. + this.status.login = LOGIN_SUCCEEDED; + return true; + } + + this._log.warn("Remote setup failed."); + // Remote setup must have failed. + return false; + + case 401: + this._log.warn("401: login failed."); + // Fall through to the 404 case. + + case 404: + // Check that we're verifying with the correct cluster + if (allow40XRecovery && (await this.identity.setCluster())) { + return await this.verifyLogin(false); + } + + // We must have the right cluster, but the server doesn't expect us. + // For FxA this almost certainly means "transient error fetching token". + this.status.login = LOGIN_FAILED_NETWORK_ERROR; + return false; + + default: + // Server didn't respond with something that we expected + this.status.login = LOGIN_FAILED_SERVER_ERROR; + this.errorHandler.checkServerError(test); + return false; + } + } catch (ex) { + // Must have failed on some network issue + this._log.debug("verifyLogin failed", ex); + this.status.login = LOGIN_FAILED_NETWORK_ERROR; + this.errorHandler.checkServerError(ex); + return false; + } + }, + + async generateNewSymmetricKeys() { + this._log.info("Generating new keys WBO..."); + let wbo = await this.collectionKeys.generateNewKeysWBO(); + this._log.info("Encrypting new key bundle."); + await wbo.encrypt(this.identity.syncKeyBundle); + + let uploadRes = await this._uploadCryptoKeys(wbo, 0); + if (uploadRes.status != 200) { + this._log.warn( + "Got status " + + uploadRes.status + + " uploading new keys. What to do? Throw!" + ); + this.errorHandler.checkServerError(uploadRes); + throw new Error("Unable to upload symmetric keys."); + } + this._log.info("Got status " + uploadRes.status + " uploading keys."); + let serverModified = uploadRes.obj; // Modified timestamp according to server. + this._log.debug("Server reports crypto modified: " + serverModified); + + // Now verify that info/collections shows them! + this._log.debug("Verifying server collection records."); + let info = await this._fetchInfo(); + this._log.debug("info/collections is: " + info.data); + + if (info.status != 200) { + this._log.warn("Non-200 info/collections response. Aborting."); + throw new Error("Unable to upload symmetric keys."); + } + + info = info.obj; + if (!(CRYPTO_COLLECTION in info)) { + this._log.error( + "Consistency failure: info/collections excludes " + + "crypto after successful upload." + ); + throw new Error("Symmetric key upload failed."); + } + + // Can't check against local modified: clock drift. + if (info[CRYPTO_COLLECTION] < serverModified) { + this._log.error( + "Consistency failure: info/collections crypto entry " + + "is stale after successful upload." + ); + throw new Error("Symmetric key upload failed."); + } + + // Doesn't matter if the timestamp is ahead. + + // Download and install them. + let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); + let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))) + .response; + if (cryptoResp.status != 200) { + this._log.warn("Failed to download keys."); + throw new Error("Symmetric key download failed."); + } + let keysChanged = await this.handleFetchedKeys( + this.identity.syncKeyBundle, + cryptoKeys, + true + ); + if (keysChanged) { + this._log.info("Downloaded keys differed, as expected."); + } + }, + + // configures/enabled/turns-on sync. There must be an FxA user signed in. + async configure() { + // We don't, and must not, throw if sync is already configured, because we + // might end up being called as part of a "reconnect" flow. We also want to + // avoid checking the FxA user is the same as the pref because the email + // address for the FxA account can change - we'd need to use the uid. + let user = await fxAccounts.getSignedInUser(); + if (!user) { + throw new Error("No FxA user is signed in"); + } + this._log.info("Configuring sync with current FxA user"); + Svc.PrefBranch.setStringPref("username", user.email); + Svc.Obs.notify("weave:connected"); + }, + + // resets/turns-off sync. + async startOver() { + this._log.trace("Invoking Service.startOver."); + await this._stopTracking(); + this.status.resetSync(); + + // Deletion doesn't make sense if we aren't set up yet! + if (this.clusterURL != "") { + // Clear client-specific data from the server, including disabled engines. + const engines = [this.clientsEngine, ...this.engineManager.getAll()]; + for (let engine of engines) { + try { + await engine.removeClientData(); + } catch (ex) { + this._log.warn(`Deleting client data for ${engine.name} failed`, ex); + } + } + this._log.debug("Finished deleting client data."); + } else { + this._log.debug("Skipping client data removal: no cluster URL."); + } + + this.identity.resetCredentials(); + this.status.login = LOGIN_FAILED_NO_USERNAME; + this.logout(); + Svc.Obs.notify("weave:service:start-over"); + + // Reset all engines and clear keys. + await this.resetClient(); + this.collectionKeys.clear(); + this.status.resetBackoff(); + + // Reset Weave prefs. + this._ignorePrefObserver = true; + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + this._ignorePrefObserver = false; + this.clusterURL = null; + + Svc.PrefBranch.setStringPref("lastversion", WEAVE_VERSION); + + try { + this.identity.finalize(); + this.status.__authManager = null; + this.identity = Status._authManager; + Svc.Obs.notify("weave:service:start-over:finish"); + } catch (err) { + this._log.error( + "startOver failed to re-initialize the identity manager", + err + ); + // Still send the observer notification so the current state is + // reflected in the UI. + Svc.Obs.notify("weave:service:start-over:finish"); + } + }, + + async login() { + async function onNotify() { + this._loggedIn = false; + if (this.scheduler.offline) { + this.status.login = LOGIN_FAILED_NETWORK_ERROR; + throw new Error("Application is offline, login should not be called"); + } + + this._log.info("User logged in successfully - verifying login."); + if (!(await this.verifyLogin())) { + // verifyLogin sets the failure states here. + throw new Error(`Login failed: ${this.status.login}`); + } + + this._updateCachedURLs(); + + this._loggedIn = true; + + return true; + } + + let notifier = this._notify("login", "", onNotify.bind(this)); + return this._catch(this._lock("service.js: login", notifier))(); + }, + + logout: function logout() { + // If we failed during login, we aren't going to have this._loggedIn set, + // but we still want to ask the identity to logout, so it doesn't try and + // reuse any old credentials next time we sync. + this._log.info("Logging out"); + this.identity.logout(); + this._loggedIn = false; + + Svc.Obs.notify("weave:service:logout:finish"); + }, + + // Note: returns false if we failed for a reason other than the server not yet + // supporting the api. + async _fetchServerConfiguration() { + // This is similar to _fetchInfo, but with different error handling. + + let infoURL = this.userBaseURL + "info/configuration"; + this._log.debug("Fetching server configuration", infoURL); + let configResponse; + try { + configResponse = await this.resource(infoURL).get(); + } catch (ex) { + // This is probably a network or similar error. + this._log.warn("Failed to fetch info/configuration", ex); + this.errorHandler.checkServerError(ex); + return false; + } + + if (configResponse.status == 404) { + // This server doesn't support the URL yet - that's OK. + this._log.debug( + "info/configuration returned 404 - using default upload semantics" + ); + } else if (configResponse.status != 200) { + this._log.warn( + `info/configuration returned ${configResponse.status} - using default configuration` + ); + this.errorHandler.checkServerError(configResponse); + return false; + } else { + this.serverConfiguration = configResponse.obj; + } + this._log.trace( + "info/configuration for this server", + this.serverConfiguration + ); + return true; + }, + + // Stuff we need to do after login, before we can really do + // anything (e.g. key setup). + async _remoteSetup(infoResponse, fetchConfig = true) { + if (fetchConfig && !(await this._fetchServerConfiguration())) { + return false; + } + + this._log.debug("Fetching global metadata record"); + let meta = await this.recordManager.get(this.metaURL); + + // Checking modified time of the meta record. + if ( + infoResponse && + infoResponse.obj.meta != this.metaModified && + (!meta || !meta.isNew) + ) { + // Delete the cached meta record... + this._log.debug( + "Clearing cached meta record. metaModified is " + + JSON.stringify(this.metaModified) + + ", setting to " + + JSON.stringify(infoResponse.obj.meta) + ); + + this.recordManager.del(this.metaURL); + + // ... fetch the current record from the server, and COPY THE FLAGS. + let newMeta = await this.recordManager.get(this.metaURL); + + // If we got a 401, we do not want to create a new meta/global - we + // should be able to get the existing meta after we get a new node. + if (this.recordManager.response.status == 401) { + this._log.debug( + "Fetching meta/global record on the server returned 401." + ); + this.errorHandler.checkServerError(this.recordManager.response); + return false; + } + + if (this.recordManager.response.status == 404) { + this._log.debug("No meta/global record on the server. Creating one."); + try { + await this._uploadNewMetaGlobal(); + } catch (uploadRes) { + this._log.warn( + "Unable to upload new meta/global. Failing remote setup." + ); + this.errorHandler.checkServerError(uploadRes); + return false; + } + } else if (!newMeta) { + this._log.warn("Unable to get meta/global. Failing remote setup."); + this.errorHandler.checkServerError(this.recordManager.response); + return false; + } else { + // If newMeta, then it stands to reason that meta != null. + newMeta.isNew = meta.isNew; + newMeta.changed = meta.changed; + } + + // Switch in the new meta object and record the new time. + meta = newMeta; + this.metaModified = infoResponse.obj.meta; + } + + let remoteVersion = + meta && meta.payload.storageVersion ? meta.payload.storageVersion : ""; + + this._log.debug( + [ + "Weave Version:", + WEAVE_VERSION, + "Local Storage:", + STORAGE_VERSION, + "Remote Storage:", + remoteVersion, + ].join(" ") + ); + + // Check for cases that require a fresh start. When comparing remoteVersion, + // we need to convert it to a number as older clients used it as a string. + if ( + !meta || + !meta.payload.storageVersion || + !meta.payload.syncID || + STORAGE_VERSION > parseFloat(remoteVersion) + ) { + this._log.info( + "One of: no meta, no meta storageVersion, or no meta syncID. Fresh start needed." + ); + + // abort the server wipe if the GET status was anything other than 404 or 200 + let status = this.recordManager.response.status; + if (status != 200 && status != 404) { + this.status.sync = METARECORD_DOWNLOAD_FAIL; + this.errorHandler.checkServerError(this.recordManager.response); + this._log.warn( + "Unknown error while downloading metadata record. Aborting sync." + ); + return false; + } + + if (!meta) { + this._log.info("No metadata record, server wipe needed"); + } + if (meta && !meta.payload.syncID) { + this._log.warn("No sync id, server wipe needed"); + } + + this._log.info("Wiping server data"); + await this._freshStart(); + + if (status == 404) { + this._log.info( + "Metadata record not found, server was wiped to ensure " + + "consistency." + ); + } else { + // 200 + this._log.info("Wiped server; incompatible metadata: " + remoteVersion); + } + return true; + } else if (remoteVersion > STORAGE_VERSION) { + this.status.sync = VERSION_OUT_OF_DATE; + this._log.warn("Upgrade required to access newer storage version."); + return false; + } else if (meta.payload.syncID != this.syncID) { + this._log.info( + "Sync IDs differ. Local is " + + this.syncID + + ", remote is " + + meta.payload.syncID + ); + await this.resetClient(); + this.collectionKeys.clear(); + this.syncID = meta.payload.syncID; + this._log.debug("Clear cached values and take syncId: " + this.syncID); + + if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) { + this._log.warn("Failed to fetch symmetric keys. Failing remote setup."); + return false; + } + + // bug 545725 - re-verify creds and fail sanely + if (!(await this.verifyLogin())) { + this.status.sync = CREDENTIALS_CHANGED; + this._log.info( + "Credentials have changed, aborting sync and forcing re-login." + ); + return false; + } + + return true; + } + if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) { + this._log.warn("Failed to fetch symmetric keys. Failing remote setup."); + return false; + } + + return true; + }, + + /** + * Return whether we should attempt login at the start of a sync. + * + * Note that this function has strong ties to _checkSync: callers + * of this function should typically use _checkSync to verify that + * any necessary login took place. + */ + _shouldLogin: function _shouldLogin() { + return ( + this.enabled && + !this.scheduler.offline && + !this.isLoggedIn && + Async.isAppReady() + ); + }, + + /** + * Determine if a sync should run. + * + * @param ignore [optional] + * array of reasons to ignore when checking + * + * @return Reason for not syncing; not-truthy if sync should run + */ + _checkSync: function _checkSync(ignore) { + let reason = ""; + // Ideally we'd call _checkSetup() here but that has too many side-effects. + if (Status.service == CLIENT_NOT_CONFIGURED) { + reason = kSyncNotConfigured; + } else if (Status.service == STATUS_DISABLED || !this.enabled) { + reason = kSyncWeaveDisabled; + } else if (this.scheduler.offline) { + reason = kSyncNetworkOffline; + } else if (this.status.minimumNextSync > Date.now()) { + reason = kSyncBackoffNotMet; + } else if ( + this.status.login == MASTER_PASSWORD_LOCKED && + Utils.mpLocked() + ) { + reason = kSyncMasterPasswordLocked; + } else if (Svc.PrefBranch.getStringPref("firstSync", null) == "notReady") { + reason = kFirstSyncChoiceNotMade; + } else if (!Async.isAppReady()) { + reason = kFirefoxShuttingDown; + } + + if (ignore && ignore.includes(reason)) { + return ""; + } + + return reason; + }, + + async sync({ engines, why } = {}) { + let dateStr = Utils.formatTimestamp(new Date()); + this._log.debug("User-Agent: " + Utils.userAgent); + await this.promiseInitialized; + this._log.info( + `Starting sync at ${dateStr} in browser session ${lazy.browserSessionID}` + ); + return this._catch(async function () { + // Make sure we're logged in. + if (this._shouldLogin()) { + this._log.debug("In sync: should login."); + if (!(await this.login())) { + this._log.debug("Not syncing: login returned false."); + return; + } + } else { + this._log.trace("In sync: no need to login."); + } + await this._lockedSync(engines, why); + })(); + }, + + /** + * Sync up engines with the server. + */ + async _lockedSync(engineNamesToSync, why) { + return this._lock( + "service.js: sync", + this._notify("sync", JSON.stringify({ why }), async function onNotify() { + let histogram = + Services.telemetry.getHistogramById("WEAVE_START_COUNT"); + histogram.add(1); + + let synchronizer = new EngineSynchronizer(this); + await synchronizer.sync(engineNamesToSync, why); // Might throw! + + histogram = Services.telemetry.getHistogramById( + "WEAVE_COMPLETE_SUCCESS_COUNT" + ); + histogram.add(1); + + // We successfully synchronized. + // Check if the identity wants to pre-fetch a migration sentinel from + // the server. + // If we have no clusterURL, we are probably doing a node reassignment + // so don't attempt to get it in that case. + if (this.clusterURL) { + this.identity.prefetchMigrationSentinel(this); + } + + // Now let's update our declined engines + await this._maybeUpdateDeclined(); + }) + )(); + }, + + /** + * Update the "declined" information in meta/global if necessary. + */ + async _maybeUpdateDeclined() { + // if Sync failed due to no node we will not have a meta URL, so can't + // update anything. + if (!this.metaURL) { + return; + } + let meta = await this.recordManager.get(this.metaURL); + if (!meta) { + this._log.warn("No meta/global; can't update declined state."); + return; + } + + let declinedEngines = new DeclinedEngines(this); + let didChange = declinedEngines.updateDeclined(meta, this.engineManager); + if (!didChange) { + this._log.info( + "No change to declined engines. Not reuploading meta/global." + ); + return; + } + + await this.uploadMetaGlobal(meta); + }, + + /** + * Upload a fresh meta/global record + * @throws the response object if the upload request was not a success + */ + async _uploadNewMetaGlobal() { + let meta = new WBORecord("meta", "global"); + meta.payload.syncID = this.syncID; + meta.payload.storageVersion = STORAGE_VERSION; + meta.payload.declined = this.engineManager.getDeclined(); + meta.modified = 0; + meta.isNew = true; + + await this.uploadMetaGlobal(meta); + }, + + /** + * Upload meta/global, throwing the response on failure + * @param {WBORecord} meta meta/global record + * @throws the response object if the request was not a success + */ + async uploadMetaGlobal(meta) { + this._log.debug("Uploading meta/global", meta); + let res = this.resource(this.metaURL); + res.setHeader("X-If-Unmodified-Since", meta.modified); + let response = await res.put(meta); + if (!response.success) { + throw response; + } + // From https://docs.services.mozilla.com/storage/apis-1.5.html: + // "Successful responses will return the new last-modified time for the collection." + meta.modified = response.obj; + this.recordManager.set(this.metaURL, meta); + }, + + /** + * Upload crypto/keys + * @param {WBORecord} cryptoKeys crypto/keys record + * @param {Number} lastModified known last modified timestamp (in decimal seconds), + * will be used to set the X-If-Unmodified-Since header + */ + async _uploadCryptoKeys(cryptoKeys, lastModified) { + this._log.debug(`Uploading crypto/keys (lastModified: ${lastModified})`); + let res = this.resource(this.cryptoKeysURL); + res.setHeader("X-If-Unmodified-Since", lastModified); + return res.put(cryptoKeys); + }, + + async _freshStart() { + this._log.info("Fresh start. Resetting client."); + await this.resetClient(); + this.collectionKeys.clear(); + + // Wipe the server. + await this.wipeServer(); + + // Upload a new meta/global record. + // _uploadNewMetaGlobal throws on failure -- including race conditions. + // If we got into a race condition, we'll abort the sync this way, too. + // That's fine. We'll just wait till the next sync. The client that we're + // racing is probably busy uploading stuff right now anyway. + await this._uploadNewMetaGlobal(); + + // Wipe everything we know about except meta because we just uploaded it + // TODO: there's a bug here. We should be calling resetClient, no? + + // Generate, upload, and download new keys. Do this last so we don't wipe + // them... + await this.generateNewSymmetricKeys(); + }, + + /** + * Wipe user data from the server. + * + * @param collections [optional] + * Array of collections to wipe. If not given, all collections are + * wiped by issuing a DELETE request for `storageURL`. + * + * @return the server's timestamp of the (last) DELETE. + */ + async wipeServer(collections) { + let response; + let histogram = Services.telemetry.getHistogramById( + "WEAVE_WIPE_SERVER_SUCCEEDED" + ); + if (!collections) { + // Strip the trailing slash. + let res = this.resource(this.storageURL.slice(0, -1)); + res.setHeader("X-Confirm-Delete", "1"); + try { + response = await res.delete(); + } catch (ex) { + this._log.debug("Failed to wipe server", ex); + histogram.add(false); + throw ex; + } + if (response.status != 200 && response.status != 404) { + this._log.debug( + "Aborting wipeServer. Server responded with " + + response.status + + " response for " + + this.storageURL + ); + histogram.add(false); + throw response; + } + histogram.add(true); + return response.headers["x-weave-timestamp"]; + } + + let timestamp; + for (let name of collections) { + let url = this.storageURL + name; + try { + response = await this.resource(url).delete(); + } catch (ex) { + this._log.debug("Failed to wipe '" + name + "' collection", ex); + histogram.add(false); + throw ex; + } + + if (response.status != 200 && response.status != 404) { + this._log.debug( + "Aborting wipeServer. Server responded with " + + response.status + + " response for " + + url + ); + histogram.add(false); + throw response; + } + + if ("x-weave-timestamp" in response.headers) { + timestamp = response.headers["x-weave-timestamp"]; + } + } + histogram.add(true); + return timestamp; + }, + + /** + * Wipe all local user data. + * + * @param engines [optional] + * Array of engine names to wipe. If not given, all engines are used. + */ + async wipeClient(engines) { + // If we don't have any engines, reset the service and wipe all engines + if (!engines) { + // Clear out any service data + await this.resetService(); + + engines = [this.clientsEngine, ...this.engineManager.getAll()]; + } else { + // Convert the array of names into engines + engines = this.engineManager.get(engines); + } + + // Fully wipe each engine if it's able to decrypt data + for (let engine of engines) { + if (await engine.canDecrypt()) { + await engine.wipeClient(); + } + } + }, + + /** + * Wipe all remote user data by wiping the server then telling each remote + * client to wipe itself. + * + * @param engines + * Array of engine names to wipe. + */ + async wipeRemote(engines) { + try { + // Make sure stuff gets uploaded. + await this.resetClient(engines); + + // Clear out any server data. + await this.wipeServer(engines); + + // Only wipe the engines provided. + let extra = { reason: "wipe-remote" }; + for (const e of engines) { + await this.clientsEngine.sendCommand("wipeEngine", [e], null, extra); + } + + // Make sure the changed clients get updated. + await this.clientsEngine.sync(); + } catch (ex) { + this.errorHandler.checkServerError(ex); + throw ex; + } + }, + + /** + * Reset local service information like logs, sync times, caches. + */ + async resetService() { + return this._catch(async function reset() { + this._log.info("Service reset."); + + // Pretend we've never synced to the server and drop cached data + this.syncID = ""; + this.recordManager.clearCache(); + })(); + }, + + /** + * Reset the client by getting rid of any local server data and client data. + * + * @param engines [optional] + * Array of engine names to reset. If not given, all engines are used. + */ + async resetClient(engines) { + return this._catch(async function doResetClient() { + // If we don't have any engines, reset everything including the service + if (!engines) { + // Clear out any service data + await this.resetService(); + + engines = [this.clientsEngine, ...this.engineManager.getAll()]; + } else { + // Convert the array of names into engines + engines = this.engineManager.get(engines); + } + + // Have each engine drop any temporary meta data + for (let engine of engines) { + await engine.resetClient(); + } + })(); + }, + + recordTelemetryEvent(object, method, value, extra = undefined) { + Svc.Obs.notify("weave:telemetry:event", { object, method, value, extra }); + }, +}; + +export var Service = new Sync11Service(); +Service.promiseInitialized = new Promise(resolve => { + Service.onStartup().then(resolve); +}); diff --git a/services/sync/modules/stages/declined.sys.mjs b/services/sync/modules/stages/declined.sys.mjs new file mode 100644 index 0000000000..2c74aab117 --- /dev/null +++ b/services/sync/modules/stages/declined.sys.mjs @@ -0,0 +1,78 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains code for maintaining the set of declined engines, + * in conjunction with EngineManager. + */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Observers } from "resource://services-common/observers.sys.mjs"; + +export var DeclinedEngines = function (service) { + this._log = Log.repository.getLogger("Sync.Declined"); + this._log.manageLevelFromPref("services.sync.log.logger.declined"); + + this.service = service; +}; + +DeclinedEngines.prototype = { + updateDeclined(meta, engineManager = this.service.engineManager) { + let enabled = new Set(engineManager.getEnabled().map(e => e.name)); + let known = new Set(engineManager.getAll().map(e => e.name)); + let remoteDeclined = new Set(meta.payload.declined || []); + let localDeclined = new Set(engineManager.getDeclined()); + + this._log.debug( + "Handling remote declined: " + JSON.stringify([...remoteDeclined]) + ); + this._log.debug( + "Handling local declined: " + JSON.stringify([...localDeclined]) + ); + + // Any engines that are locally enabled should be removed from the remote + // declined list. + // + // Any engines that are locally declined should be added to the remote + // declined list. + let newDeclined = CommonUtils.union( + localDeclined, + CommonUtils.difference(remoteDeclined, enabled) + ); + + // If our declined set has changed, put it into the meta object and mark + // it as changed. + let declinedChanged = !CommonUtils.setEqual(newDeclined, remoteDeclined); + this._log.debug("Declined changed? " + declinedChanged); + if (declinedChanged) { + meta.changed = true; + meta.payload.declined = [...newDeclined]; + } + + // Update the engine manager regardless. + engineManager.setDeclined(newDeclined); + + // Any engines that are locally known, locally disabled, and not remotely + // or locally declined, are candidates for enablement. + let undecided = CommonUtils.difference( + CommonUtils.difference(known, enabled), + newDeclined + ); + if (undecided.size) { + let subject = { + declined: newDeclined, + enabled, + known, + undecided, + }; + CommonUtils.nextTick(() => { + Observers.notify("weave:engines:notdeclined", subject); + }); + } + + return declinedChanged; + }, +}; diff --git a/services/sync/modules/stages/enginesync.sys.mjs b/services/sync/modules/stages/enginesync.sys.mjs new file mode 100644 index 0000000000..190fd1f5fa --- /dev/null +++ b/services/sync/modules/stages/enginesync.sys.mjs @@ -0,0 +1,412 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains code for synchronizing engines. + */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { + ABORT_SYNC_COMMAND, + LOGIN_FAILED_NETWORK_ERROR, + NO_SYNC_NODE_FOUND, + STATUS_OK, + SYNC_FAILED_PARTIAL, + SYNC_SUCCEEDED, + WEAVE_VERSION, + kSyncNetworkOffline, +} from "resource://services-sync/constants.sys.mjs"; + +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + Doctor: "resource://services-sync/doctor.sys.mjs", +}); + +/** + * Perform synchronization of engines. + * + * This was originally split out of service.js. The API needs lots of love. + */ +export function EngineSynchronizer(service) { + this._log = Log.repository.getLogger("Sync.Synchronizer"); + this._log.manageLevelFromPref("services.sync.log.logger.synchronizer"); + + this.service = service; +} + +EngineSynchronizer.prototype = { + async sync(engineNamesToSync, why) { + let fastSync = why && why == "sleep"; + let startTime = Date.now(); + + this.service.status.resetSync(); + + // Make sure we should sync or record why we shouldn't. + let reason = this.service._checkSync(); + if (reason) { + if (reason == kSyncNetworkOffline) { + this.service.status.sync = LOGIN_FAILED_NETWORK_ERROR; + } + + // this is a purposeful abort rather than a failure, so don't set + // any status bits + reason = "Can't sync: " + reason; + throw new Error(reason); + } + + // If we don't have a node, get one. If that fails, retry in 10 minutes. + if ( + !this.service.clusterURL && + !(await this.service.identity.setCluster()) + ) { + this.service.status.sync = NO_SYNC_NODE_FOUND; + this._log.info("No cluster URL found. Cannot sync."); + return; + } + + // Ping the server with a special info request once a day. + let infoURL = this.service.infoURL; + let now = Math.floor(Date.now() / 1000); + let lastPing = Svc.PrefBranch.getIntPref("lastPing", 0); + if (now - lastPing > 86400) { + // 60 * 60 * 24 + infoURL += "?v=" + WEAVE_VERSION; + Svc.PrefBranch.setIntPref("lastPing", now); + } + + let engineManager = this.service.engineManager; + + // Figure out what the last modified time is for each collection + let info = await this.service._fetchInfo(infoURL); + + // Convert the response to an object and read out the modified times + for (let engine of [this.service.clientsEngine].concat( + engineManager.getAll() + )) { + engine.lastModified = info.obj[engine.name] || 0; + } + + if (!(await this.service._remoteSetup(info, !fastSync))) { + throw new Error("Aborting sync, remote setup failed"); + } + + if (!fastSync) { + // Make sure we have an up-to-date list of clients before sending commands + this._log.debug("Refreshing client list."); + if (!(await this._syncEngine(this.service.clientsEngine))) { + // Clients is an engine like any other; it can fail with a 401, + // and we can elect to abort the sync. + this._log.warn("Client engine sync failed. Aborting."); + return; + } + } + + // We only honor the "hint" of what engines to Sync if this isn't + // a first sync. + let allowEnginesHint = false; + // Wipe data in the desired direction if necessary + switch (Svc.PrefBranch.getStringPref("firstSync", null)) { + case "resetClient": + await this.service.resetClient(engineManager.enabledEngineNames); + break; + case "wipeClient": + await this.service.wipeClient(engineManager.enabledEngineNames); + break; + case "wipeRemote": + await this.service.wipeRemote(engineManager.enabledEngineNames); + break; + default: + allowEnginesHint = true; + break; + } + + if (!fastSync && this.service.clientsEngine.localCommands) { + try { + if (!(await this.service.clientsEngine.processIncomingCommands())) { + this.service.status.sync = ABORT_SYNC_COMMAND; + throw new Error("Processed command aborted sync."); + } + + // Repeat remoteSetup in-case the commands forced us to reset + if (!(await this.service._remoteSetup(info))) { + throw new Error("Remote setup failed after processing commands."); + } + } finally { + // Always immediately attempt to push back the local client (now + // without commands). + // Note that we don't abort here; if there's a 401 because we've + // been reassigned, we'll handle it around another engine. + await this._syncEngine(this.service.clientsEngine); + } + } + + // Update engines because it might change what we sync. + try { + await this._updateEnabledEngines(); + } catch (ex) { + this._log.debug("Updating enabled engines failed", ex); + this.service.errorHandler.checkServerError(ex); + throw ex; + } + + await this.service.engineManager.switchAlternatives(); + + // If the engines to sync has been specified, we sync in the order specified. + let enginesToSync; + if (allowEnginesHint && engineNamesToSync) { + this._log.info("Syncing specified engines", engineNamesToSync); + enginesToSync = engineManager + .get(engineNamesToSync) + .filter(e => e.enabled); + } else { + this._log.info("Syncing all enabled engines."); + enginesToSync = engineManager.getEnabled(); + } + try { + // We don't bother validating engines that failed to sync. + let enginesToValidate = []; + for (let engine of enginesToSync) { + if (engine.shouldSkipSync(why)) { + this._log.info(`Engine ${engine.name} asked to be skipped`); + continue; + } + // If there's any problems with syncing the engine, report the failure + if ( + !(await this._syncEngine(engine)) || + this.service.status.enforceBackoff + ) { + this._log.info("Aborting sync for failure in " + engine.name); + break; + } + enginesToValidate.push(engine); + } + + // If _syncEngine fails for a 401, we might not have a cluster URL here. + // If that's the case, break out of this immediately, rather than + // throwing an exception when trying to fetch metaURL. + if (!this.service.clusterURL) { + this._log.debug( + "Aborting sync, no cluster URL: not uploading new meta/global." + ); + return; + } + + // Upload meta/global if any engines changed anything. + let meta = await this.service.recordManager.get(this.service.metaURL); + if (meta.isNew || meta.changed) { + this._log.info("meta/global changed locally: reuploading."); + try { + await this.service.uploadMetaGlobal(meta); + delete meta.isNew; + delete meta.changed; + } catch (error) { + this._log.error( + "Unable to upload meta/global. Leaving marked as new." + ); + } + } + + if (!fastSync) { + await lazy.Doctor.consult(enginesToValidate); + } + + // If there were no sync engine failures + if (this.service.status.service != SYNC_FAILED_PARTIAL) { + this.service.status.sync = SYNC_SUCCEEDED; + } + + // Even if there were engine failures, bump lastSync even on partial since + // it's reflected in the UI (bug 1439777). + if ( + this.service.status.service == SYNC_FAILED_PARTIAL || + this.service.status.service == STATUS_OK + ) { + Svc.PrefBranch.setStringPref("lastSync", new Date().toString()); + } + } finally { + Svc.PrefBranch.clearUserPref("firstSync"); + + let syncTime = ((Date.now() - startTime) / 1000).toFixed(2); + let dateStr = Utils.formatTimestamp(new Date()); + this._log.info( + "Sync completed at " + dateStr + " after " + syncTime + " secs." + ); + } + }, + + // Returns true if sync should proceed. + // false / no return value means sync should be aborted. + async _syncEngine(engine) { + try { + await engine.sync(); + } catch (e) { + if (e.status == 401) { + // Maybe a 401, cluster update perhaps needed? + // We rely on ErrorHandler observing the sync failure notification to + // schedule another sync and clear node assignment values. + // Here we simply want to muffle the exception and return an + // appropriate value. + return false; + } + // Note that policies.js has already logged info about the exception... + if (Async.isShutdownException(e)) { + // Failure due to a shutdown exception should prevent other engines + // trying to start and immediately failing. + this._log.info( + `${engine.name} was interrupted by shutdown; no other engines will sync` + ); + return false; + } + } + + return true; + }, + + async _updateEnabledFromMeta( + meta, + numClients, + engineManager = this.service.engineManager + ) { + this._log.info("Updating enabled engines: " + numClients + " clients."); + + if (meta.isNew || !meta.payload.engines) { + this._log.debug( + "meta/global isn't new, or is missing engines. Not updating enabled state." + ); + return; + } + + // If we're the only client, and no engines are marked as enabled, + // thumb our noses at the server data: it can't be right. + // Belt-and-suspenders approach to Bug 615926. + let hasEnabledEngines = false; + for (let e in meta.payload.engines) { + if (e != "clients") { + hasEnabledEngines = true; + break; + } + } + + if (numClients <= 1 && !hasEnabledEngines) { + this._log.info( + "One client and no enabled engines: not touching local engine status." + ); + return; + } + + this.service._ignorePrefObserver = true; + + let enabled = engineManager.enabledEngineNames; + + let toDecline = new Set(); + let toUndecline = new Set(); + + for (let engineName in meta.payload.engines) { + if (engineName == "clients") { + // Clients is special. + continue; + } + let index = enabled.indexOf(engineName); + if (index != -1) { + // The engine is enabled locally. Nothing to do. + enabled.splice(index, 1); + continue; + } + let engine = engineManager.get(engineName); + if (!engine) { + // The engine doesn't exist locally. Nothing to do. + continue; + } + + let attemptedEnable = false; + // If the engine was enabled remotely, enable it locally. + if ( + !Svc.PrefBranch.getBoolPref( + "engineStatusChanged." + engine.prefName, + false + ) + ) { + this._log.trace( + "Engine " + engineName + " was enabled. Marking as non-declined." + ); + toUndecline.add(engineName); + this._log.trace(engineName + " engine was enabled remotely."); + engine.enabled = true; + // Note that setting engine.enabled to true might not have worked for + // the password engine if a master-password is enabled. However, it's + // still OK that we added it to undeclined - the user *tried* to enable + // it remotely - so it still winds up as not being flagged as declined + // even though it's disabled remotely. + attemptedEnable = true; + } + + // If either the engine was disabled locally or enabling the engine + // failed (see above re master-password) then wipe server data and + // disable it everywhere. + if (!engine.enabled) { + this._log.trace("Wiping data for " + engineName + " engine."); + await engine.wipeServer(); + delete meta.payload.engines[engineName]; + meta.changed = true; // the new enabled state must propagate + // We also here mark the engine as declined, because the pref + // was explicitly changed to false - unless we tried, and failed, + // to enable it - in which case we leave the declined state alone. + if (!attemptedEnable) { + // This will be reflected in meta/global in the next stage. + this._log.trace( + "Engine " + + engineName + + " was disabled locally. Marking as declined." + ); + toDecline.add(engineName); + } + } + } + + // Any remaining engines were either enabled locally or disabled remotely. + for (let engineName of enabled) { + let engine = engineManager.get(engineName); + if ( + Svc.PrefBranch.getBoolPref( + "engineStatusChanged." + engine.prefName, + false + ) + ) { + this._log.trace("The " + engineName + " engine was enabled locally."); + toUndecline.add(engineName); + } else { + this._log.trace("The " + engineName + " engine was disabled remotely."); + + // Don't automatically mark it as declined! + try { + engine.enabled = false; + } catch (e) { + this._log.trace("Failed to disable engine " + engineName); + } + } + } + + engineManager.decline(toDecline); + engineManager.undecline(toUndecline); + + for (const pref of Svc.PrefBranch.getChildList("engineStatusChanged.")) { + Svc.PrefBranch.clearUserPref(pref); + } + this.service._ignorePrefObserver = false; + }, + + async _updateEnabledEngines() { + let meta = await this.service.recordManager.get(this.service.metaURL); + let numClients = this.service.scheduler.numClients; + let engineManager = this.service.engineManager; + + await this._updateEnabledFromMeta(meta, numClients, engineManager); + }, +}; +Object.freeze(EngineSynchronizer.prototype); diff --git a/services/sync/modules/status.sys.mjs b/services/sync/modules/status.sys.mjs new file mode 100644 index 0000000000..429dbda7b6 --- /dev/null +++ b/services/sync/modules/status.sys.mjs @@ -0,0 +1,135 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + CLIENT_NOT_CONFIGURED, + ENGINE_SUCCEEDED, + LOGIN_FAILED, + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_FAILED_NO_USERNAME, + LOGIN_SUCCEEDED, + STATUS_OK, + SYNC_FAILED, + SYNC_FAILED_PARTIAL, + SYNC_SUCCEEDED, +} from "resource://services-sync/constants.sys.mjs"; + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { SyncAuthManager } from "resource://services-sync/sync_auth.sys.mjs"; + +export var Status = { + _log: Log.repository.getLogger("Sync.Status"), + __authManager: null, + ready: false, + + get _authManager() { + if (this.__authManager) { + return this.__authManager; + } + this.__authManager = new SyncAuthManager(); + return this.__authManager; + }, + + get service() { + return this._service; + }, + + set service(code) { + this._log.debug( + "Status.service: " + (this._service || undefined) + " => " + code + ); + this._service = code; + }, + + get login() { + return this._login; + }, + + set login(code) { + this._log.debug("Status.login: " + this._login + " => " + code); + this._login = code; + + if ( + code == LOGIN_FAILED_NO_USERNAME || + code == LOGIN_FAILED_NO_PASSPHRASE + ) { + this.service = CLIENT_NOT_CONFIGURED; + } else if (code != LOGIN_SUCCEEDED) { + this.service = LOGIN_FAILED; + } else { + this.service = STATUS_OK; + } + }, + + get sync() { + return this._sync; + }, + + set sync(code) { + this._log.debug("Status.sync: " + this._sync + " => " + code); + this._sync = code; + this.service = code == SYNC_SUCCEEDED ? STATUS_OK : SYNC_FAILED; + }, + + get engines() { + return this._engines; + }, + + set engines([name, code]) { + this._log.debug("Status for engine " + name + ": " + code); + this._engines[name] = code; + + if (code != ENGINE_SUCCEEDED) { + this.service = SYNC_FAILED_PARTIAL; + } + }, + + // Implement toString because adding a logger introduces a cyclic object + // value, so we can't trivially debug-print Status as JSON. + toString: function toString() { + return ( + "<Status" + + ": login: " + + Status.login + + ", service: " + + Status.service + + ", sync: " + + Status.sync + + ">" + ); + }, + + checkSetup: function checkSetup() { + if (!this._authManager.username) { + Status.login = LOGIN_FAILED_NO_USERNAME; + Status.service = CLIENT_NOT_CONFIGURED; + } else if (Status.login == STATUS_OK) { + Status.service = STATUS_OK; + } + return Status.service; + }, + + resetBackoff: function resetBackoff() { + this.enforceBackoff = false; + this.backoffInterval = 0; + this.minimumNextSync = 0; + }, + + resetSync: function resetSync() { + // Logger setup. + this._log.manageLevelFromPref("services.sync.log.logger.status"); + + this._log.info("Resetting Status."); + this.service = STATUS_OK; + this._login = LOGIN_SUCCEEDED; + this._sync = SYNC_SUCCEEDED; + this._engines = {}; + this.partial = false; + }, +}; + +// Initialize various status values. +Status.resetBackoff(); +Status.resetSync(); diff --git a/services/sync/modules/sync_auth.sys.mjs b/services/sync/modules/sync_auth.sys.mjs new file mode 100644 index 0000000000..6b8da4061c --- /dev/null +++ b/services/sync/modules/sync_auth.sys.mjs @@ -0,0 +1,655 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { TokenServerClient } from "resource://services-common/tokenserverclient.sys.mjs"; +import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { + LOGIN_FAILED_LOGIN_REJECTED, + LOGIN_FAILED_NETWORK_ERROR, + LOGIN_FAILED_NO_USERNAME, + LOGIN_SUCCEEDED, + MASTER_PASSWORD_LOCKED, + STATUS_OK, +} from "resource://services-sync/constants.sys.mjs"; + +const lazy = {}; + +// Lazy imports to prevent unnecessary load on startup. +ChromeUtils.defineESModuleGetters(lazy, { + BulkKeyBundle: "resource://services-sync/keys.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +ChromeUtils.defineLazyGetter(lazy, "log", function () { + let log = Log.repository.getLogger("Sync.SyncAuthManager"); + log.manageLevelFromPref("services.sync.log.logger.identity"); + return log; +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "IGNORE_CACHED_AUTH_CREDENTIALS", + "services.sync.debug.ignoreCachedAuthCredentials" +); + +// FxAccountsCommon.js doesn't use a "namespace", so create one here. +import * as fxAccountsCommon from "resource://gre/modules/FxAccountsCommon.sys.mjs"; + +const SCOPE_OLD_SYNC = fxAccountsCommon.SCOPE_OLD_SYNC; + +const OBSERVER_TOPICS = [ + fxAccountsCommon.ONLOGIN_NOTIFICATION, + fxAccountsCommon.ONVERIFIED_NOTIFICATION, + fxAccountsCommon.ONLOGOUT_NOTIFICATION, + fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION, + "weave:connected", +]; + +/* + General authentication error for abstracting authentication + errors from multiple sources (e.g., from FxAccounts, TokenServer). + details is additional details about the error - it might be a string, or + some other error object (which should do the right thing when toString() is + called on it) +*/ +export function AuthenticationError(details, source) { + this.details = details; + this.source = source; +} + +AuthenticationError.prototype = { + toString() { + return "AuthenticationError(" + this.details + ")"; + }, +}; + +// The `SyncAuthManager` coordinates access authorization to the Sync server. +// Its job is essentially to get us from having a signed-in Firefox Accounts user, +// to knowing the user's sync storage node and having the necessary short-lived +// credentials in order to access it. +// + +export function SyncAuthManager() { + // NOTE: _fxaService and _tokenServerClient are replaced with mocks by + // the test suite. + this._fxaService = lazy.fxAccounts; + this._tokenServerClient = new TokenServerClient(); + this._tokenServerClient.observerPrefix = "weave:service"; + this._log = lazy.log; + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_username", + "services.sync.username" + ); + + this.asyncObserver = Async.asyncObserver(this, lazy.log); + for (let topic of OBSERVER_TOPICS) { + Services.obs.addObserver(this.asyncObserver, topic); + } +} + +SyncAuthManager.prototype = { + _fxaService: null, + _tokenServerClient: null, + // https://docs.services.mozilla.com/token/apis.html + _token: null, + // protection against the user changing underneath us - the uid + // of the current user. + _userUid: null, + + hashedUID() { + const id = this._fxaService.telemetry.getSanitizedUID(); + if (!id) { + throw new Error("hashedUID: Don't seem to have previously seen a token"); + } + return id; + }, + + // Return a hashed version of a deviceID, suitable for telemetry. + hashedDeviceID(deviceID) { + const id = this._fxaService.telemetry.sanitizeDeviceId(deviceID); + if (!id) { + throw new Error("hashedUID: Don't seem to have previously seen a token"); + } + return id; + }, + + // The "node type" reported to telemetry or null if not specified. + get telemetryNodeType() { + return this._token && this._token.node_type ? this._token.node_type : null; + }, + + finalize() { + // After this is called, we can expect Service.identity != this. + for (let topic of OBSERVER_TOPICS) { + Services.obs.removeObserver(this.asyncObserver, topic); + } + this.resetCredentials(); + this._userUid = null; + }, + + async getSignedInUser() { + let data = await this._fxaService.getSignedInUser(); + if (!data) { + this._userUid = null; + return null; + } + if (this._userUid == null) { + this._userUid = data.uid; + } else if (this._userUid != data.uid) { + throw new Error("The signed in user has changed"); + } + return data; + }, + + logout() { + // This will be called when sync fails (or when the account is being + // unlinked etc). It may have failed because we got a 401 from a sync + // server, so we nuke the token. Next time sync runs and wants an + // authentication header, we will notice the lack of the token and fetch a + // new one. + this._token = null; + }, + + async observe(subject, topic, data) { + this._log.debug("observed " + topic); + if (!this.username) { + this._log.info("Sync is not configured, so ignoring the notification"); + return; + } + switch (topic) { + case "weave:connected": + case fxAccountsCommon.ONLOGIN_NOTIFICATION: { + this._log.info("Sync has been connected to a logged in user"); + this.resetCredentials(); + let accountData = await this.getSignedInUser(); + + if (!accountData.verified) { + // wait for a verified notification before we kick sync off. + this._log.info("The user is not verified"); + break; + } + } + // We've been configured with an already verified user, so fall-through. + // intentional fall-through - the user is verified. + case fxAccountsCommon.ONVERIFIED_NOTIFICATION: { + this._log.info("The user became verified"); + lazy.Weave.Status.login = LOGIN_SUCCEEDED; + + // And actually sync. If we've never synced before, we force a full sync. + // If we have, then we are probably just reauthenticating so it's a normal sync. + // We can use any pref that must be set if we've synced before, and check + // the sync lock state because we might already be doing that first sync. + let isFirstSync = + !lazy.Weave.Service.locked && + !Svc.PrefBranch.getStringPref("client.syncID", null); + if (isFirstSync) { + this._log.info("Doing initial sync actions"); + Svc.PrefBranch.setStringPref("firstSync", "resetClient"); + Services.obs.notifyObservers(null, "weave:service:setup-complete"); + } + // There's no need to wait for sync to complete and it would deadlock + // our AsyncObserver. + if (!Svc.PrefBranch.getBoolPref("testing.tps", false)) { + lazy.Weave.Service.sync({ why: "login" }); + } + break; + } + + case fxAccountsCommon.ONLOGOUT_NOTIFICATION: + lazy.Weave.Service.startOver() + .then(() => { + this._log.trace("startOver completed"); + }) + .catch(err => { + this._log.warn("Failed to reset sync", err); + }); + // startOver will cause this instance to be thrown away, so there's + // nothing else to do. + break; + + case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION: + // throw away token forcing us to fetch a new one later. + this.resetCredentials(); + break; + } + }, + + /** + * Provide override point for testing token expiration. + */ + _now() { + return this._fxaService._internal.now(); + }, + + get _localtimeOffsetMsec() { + return this._fxaService._internal.localtimeOffsetMsec; + }, + + get syncKeyBundle() { + return this._syncKeyBundle; + }, + + get username() { + return this._username; + }, + + /** + * Set the username value. + * + * Changing the username has the side-effect of wiping credentials. + */ + set username(value) { + // setting .username is an old throwback, but it should no longer happen. + throw new Error("don't set the username"); + }, + + /** + * Resets all calculated credentials we hold for the current user. This will + * *not* force the user to reauthenticate, but instead will force us to + * calculate a new key bundle, fetch a new token, etc. + */ + resetCredentials() { + this._syncKeyBundle = null; + this._token = null; + // The cluster URL comes from the token, so resetting it to empty will + // force Sync to not accidentally use a value from an earlier token. + lazy.Weave.Service.clusterURL = null; + }, + + /** + * Pre-fetches any information that might help with migration away from this + * identity. Called after every sync and is really just an optimization that + * allows us to avoid a network request for when we actually need the + * migration info. + */ + prefetchMigrationSentinel(service) { + // nothing to do here until we decide to migrate away from FxA. + }, + + /** + * Verify the current auth state, unlocking the master-password if necessary. + * + * Returns a promise that resolves with the current auth state after + * attempting to unlock. + */ + async unlockAndVerifyAuthState() { + let data = await this.getSignedInUser(); + const fxa = this._fxaService; + if (!data) { + lazy.log.debug("unlockAndVerifyAuthState has no FxA user"); + return LOGIN_FAILED_NO_USERNAME; + } + if (!this.username) { + lazy.log.debug( + "unlockAndVerifyAuthState finds that sync isn't configured" + ); + return LOGIN_FAILED_NO_USERNAME; + } + if (!data.verified) { + // Treat not verified as if the user needs to re-auth, so the browser + // UI reflects the state. + lazy.log.debug("unlockAndVerifyAuthState has an unverified user"); + return LOGIN_FAILED_LOGIN_REJECTED; + } + if (await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC)) { + lazy.log.debug( + "unlockAndVerifyAuthState already has (or can fetch) sync keys" + ); + return STATUS_OK; + } + // so no keys - ensure MP unlocked. + if (!Utils.ensureMPUnlocked()) { + // user declined to unlock, so we don't know if they are stored there. + lazy.log.debug( + "unlockAndVerifyAuthState: user declined to unlock master-password" + ); + return MASTER_PASSWORD_LOCKED; + } + // If we still can't get keys it probably means the user authenticated + // without unlocking the MP or cleared the saved logins, so we've now + // lost them - the user will need to reauth before continuing. + let result; + if (await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC)) { + result = STATUS_OK; + } else { + result = LOGIN_FAILED_LOGIN_REJECTED; + } + lazy.log.debug( + "unlockAndVerifyAuthState re-fetched credentials and is returning", + result + ); + return result; + }, + + /** + * Do we have a non-null, not yet expired token for the user currently + * signed in? + */ + _hasValidToken() { + // If pref is set to ignore cached authentication credentials for debugging, + // then return false to force the fetching of a new token. + if (lazy.IGNORE_CACHED_AUTH_CREDENTIALS) { + return false; + } + if (!this._token) { + return false; + } + if (this._token.expiration < this._now()) { + return false; + } + return true; + }, + + // Get our tokenServerURL - a private helper. Returns a string. + get _tokenServerUrl() { + // We used to support services.sync.tokenServerURI but this was a + // pain-point for people using non-default servers as Sync may auto-reset + // all services.sync prefs. So if that still exists, it wins. + let url = Svc.PrefBranch.getStringPref("tokenServerURI", null); // Svc.PrefBranch "root" is services.sync + if (!url) { + url = Services.prefs.getStringPref("identity.sync.tokenserver.uri"); + } + while (url.endsWith("/")) { + // trailing slashes cause problems... + url = url.slice(0, -1); + } + return url; + }, + + // Refresh the sync token for our user. Returns a promise that resolves + // with a token, or rejects with an error. + async _fetchTokenForUser() { + const fxa = this._fxaService; + // We need keys for things to work. If we don't have them, just + // return null for the token - sync calling unlockAndVerifyAuthState() + // before actually syncing will setup the error states if necessary. + if (!(await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC))) { + this._log.info( + "Unable to fetch keys (master-password locked?), so aborting token fetch" + ); + throw new Error("Can't fetch a token as we can't get keys"); + } + + // Do the token dance, with a retry in case of transient auth failure. + // We need to prove that we know the sync key in order to get a token + // from the tokenserver. + let getToken = async key => { + this._log.info("Getting a sync token from", this._tokenServerUrl); + let token = await this._fetchTokenUsingOAuth(key); + this._log.trace("Successfully got a token"); + return token; + }; + + try { + let token, key; + try { + this._log.info("Getting sync key"); + key = await fxa.keys.getKeyForScope(SCOPE_OLD_SYNC); + if (!key) { + throw new Error("browser does not have the sync key, cannot sync"); + } + token = await getToken(key); + } catch (err) { + // If we get a 401 fetching the token it may be that our auth tokens needed + // to be regenerated; retry exactly once. + if (!err.response || err.response.status !== 401) { + throw err; + } + this._log.warn( + "Token server returned 401, retrying token fetch with fresh credentials" + ); + key = await fxa.keys.getKeyForScope(SCOPE_OLD_SYNC); + token = await getToken(key); + } + // TODO: Make it be only 80% of the duration, so refresh the token + // before it actually expires. This is to avoid sync storage errors + // otherwise, we may briefly enter a "needs reauthentication" state. + // (XXX - the above may no longer be true - someone should check ;) + token.expiration = this._now() + token.duration * 1000 * 0.8; + if (!this._syncKeyBundle) { + this._syncKeyBundle = lazy.BulkKeyBundle.fromJWK(key); + } + lazy.Weave.Status.login = LOGIN_SUCCEEDED; + this._token = token; + return token; + } catch (caughtErr) { + let err = caughtErr; // The error we will rethrow. + + // TODO: unify these errors - we need to handle errors thrown by + // both tokenserverclient and hawkclient. + // A tokenserver error thrown based on a bad response. + if (err.response && err.response.status === 401) { + err = new AuthenticationError(err, "tokenserver"); + // A hawkclient error. + } else if (err.code && err.code === 401) { + err = new AuthenticationError(err, "hawkclient"); + // An FxAccounts.jsm error. + } else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) { + err = new AuthenticationError(err, "fxaccounts"); + } + + // TODO: write tests to make sure that different auth error cases are handled here + // properly: auth error getting oauth token, auth error getting sync token (invalid + // generation or client-state error) + if (err instanceof AuthenticationError) { + this._log.error("Authentication error in _fetchTokenForUser", err); + // set it to the "fatal" LOGIN_FAILED_LOGIN_REJECTED reason. + lazy.Weave.Status.login = LOGIN_FAILED_LOGIN_REJECTED; + } else { + this._log.error("Non-authentication error in _fetchTokenForUser", err); + // for now assume it is just a transient network related problem + // (although sadly, it might also be a regular unhandled exception) + lazy.Weave.Status.login = LOGIN_FAILED_NETWORK_ERROR; + } + throw err; + } + }, + + /** + * Generates an OAuth access_token using the OLD_SYNC scope and exchanges it + * for a TokenServer token. + * + * @returns {Promise} + * @private + */ + async _fetchTokenUsingOAuth(key) { + this._log.debug("Getting a token using OAuth"); + const fxa = this._fxaService; + const ttl = fxAccountsCommon.OAUTH_TOKEN_FOR_SYNC_LIFETIME_SECONDS; + const accessToken = await fxa.getOAuthToken({ scope: SCOPE_OLD_SYNC, ttl }); + const headers = { + "X-KeyId": key.kid, + }; + + return this._tokenServerClient + .getTokenUsingOAuth(this._tokenServerUrl, accessToken, headers) + .catch(async err => { + if (err.response && err.response.status === 401) { + // remove the cached token if we cannot authorize with it. + // we have to do this here because we know which `token` to remove + // from cache. + await fxa.removeCachedOAuthToken({ token: accessToken }); + } + + // continue the error chain, so other handlers can deal with the error. + throw err; + }); + }, + + // Returns a promise that is resolved with a valid token for the current + // user, or rejects if one can't be obtained. + // NOTE: This does all the authentication for Sync - it both sets the + // key bundle (ie, decryption keys) and does the token fetch. These 2 + // concepts could be decoupled, but there doesn't seem any value in that + // currently. + async _ensureValidToken(forceNewToken = false) { + let signedInUser = await this.getSignedInUser(); + if (!signedInUser) { + throw new Error("no user is logged in"); + } + if (!signedInUser.verified) { + throw new Error("user is not verified"); + } + + await this.asyncObserver.promiseObserversComplete(); + + if (!forceNewToken && this._hasValidToken()) { + this._log.trace("_ensureValidToken already has one"); + return this._token; + } + + // We are going to grab a new token - re-use the same promise if we are + // already fetching one. + if (!this._ensureValidTokenPromise) { + this._ensureValidTokenPromise = this.__ensureValidToken().finally(() => { + this._ensureValidTokenPromise = null; + }); + } + return this._ensureValidTokenPromise; + }, + + async __ensureValidToken() { + // reset this._token as a safety net to reduce the possibility of us + // repeatedly attempting to use an invalid token if _fetchTokenForUser throws. + this._token = null; + try { + let token = await this._fetchTokenForUser(); + this._token = token; + // This is a little bit of a hack. The tokenserver tells us a HMACed version + // of the FxA uid which we can use for metrics purposes without revealing the + // user's true uid. It conceptually belongs to FxA but we get it from tokenserver + // for legacy reasons. Hand it back to the FxA client code to deal with. + this._fxaService.telemetry._setHashedUID(token.hashed_fxa_uid); + return token; + } finally { + Services.obs.notifyObservers(null, "weave:service:login:got-hashed-id"); + } + }, + + getResourceAuthenticator() { + return this._getAuthenticationHeader.bind(this); + }, + + /** + * @return a Hawk HTTP Authorization Header, lightly wrapped, for the .uri + * of a RESTRequest or AsyncResponse object. + */ + async _getAuthenticationHeader(httpObject, method) { + // Note that in failure states we return null, causing the request to be + // made without authorization headers, thereby presumably causing a 401, + // which causes Sync to log out. If we throw, this may not happen as + // expected. + try { + await this._ensureValidToken(); + } catch (ex) { + this._log.error("Failed to fetch a token for authentication", ex); + return null; + } + if (!this._token) { + return null; + } + let credentials = { id: this._token.id, key: this._token.key }; + method = method || httpObject.method; + + // Get the local clock offset from the Firefox Accounts server. This should + // be close to the offset from the storage server. + let options = { + now: this._now(), + localtimeOffsetMsec: this._localtimeOffsetMsec, + credentials, + }; + + let headerValue = await CryptoUtils.computeHAWK( + httpObject.uri, + method, + options + ); + return { headers: { authorization: headerValue.field } }; + }, + + /** + * Determine the cluster for the current user and update state. + * Returns true if a new cluster URL was found and it is different from + * the existing cluster URL, false otherwise. + */ + async setCluster() { + // Make sure we didn't get some unexpected response for the cluster. + let cluster = await this._findCluster(); + this._log.debug("Cluster value = " + cluster); + if (cluster == null) { + return false; + } + + // Convert from the funky "String object with additional properties" that + // resource.js returns to a plain-old string. + cluster = cluster.toString(); + // Don't update stuff if we already have the right cluster + if (cluster == lazy.Weave.Service.clusterURL) { + return false; + } + + this._log.debug("Setting cluster to " + cluster); + lazy.Weave.Service.clusterURL = cluster; + + return true; + }, + + async _findCluster() { + try { + // Ensure we are ready to authenticate and have a valid token. + // We need to handle node reassignment here. If we are being asked + // for a clusterURL while the service already has a clusterURL, then + // it's likely a 401 was received using the existing token - in which + // case we just discard the existing token and fetch a new one. + let forceNewToken = false; + if (lazy.Weave.Service.clusterURL) { + this._log.debug( + "_findCluster has a pre-existing clusterURL, so fetching a new token token" + ); + forceNewToken = true; + } + let token = await this._ensureValidToken(forceNewToken); + let endpoint = token.endpoint; + // For Sync 1.5 storage endpoints, we use the base endpoint verbatim. + // However, it should end in "/" because we will extend it with + // well known path components. So we add a "/" if it's missing. + if (!endpoint.endsWith("/")) { + endpoint += "/"; + } + this._log.debug("_findCluster returning " + endpoint); + return endpoint; + } catch (err) { + this._log.info("Failed to fetch the cluster URL", err); + // service.js's verifyLogin() method will attempt to fetch a cluster + // URL when it sees a 401. If it gets null, it treats it as a "real" + // auth error and sets Status.login to LOGIN_FAILED_LOGIN_REJECTED, which + // in turn causes a notification bar to appear informing the user they + // need to re-authenticate. + // On the other hand, if fetching the cluster URL fails with an exception, + // verifyLogin() assumes it is a transient error, and thus doesn't show + // the notification bar under the assumption the issue will resolve + // itself. + // Thus: + // * On a real 401, we must return null. + // * On any other problem we must let an exception bubble up. + if (err instanceof AuthenticationError) { + return null; + } + throw err; + } + }, +}; diff --git a/services/sync/modules/telemetry.sys.mjs b/services/sync/modules/telemetry.sys.mjs new file mode 100644 index 0000000000..c08f405b0e --- /dev/null +++ b/services/sync/modules/telemetry.sys.mjs @@ -0,0 +1,1279 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Support for Sync-and-FxA-related telemetry, which is submitted in a special-purpose +// telemetry ping called the "sync ping", documented here: +// +// ../../../toolkit/components/telemetry/docs/data/sync-ping.rst +// +// The sync ping contains identifiers that are linked to the user's Firefox Account +// and are separate from the main telemetry client_id, so this file is also responsible +// for ensuring that we can delete those pings upon user request, by plumbing its +// identifiers into the "deletion-request" ping. + +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", + AuthenticationError: "resource://services-sync/sync_auth.sys.mjs", + FxAccounts: "resource://gre/modules/FxAccounts.sys.mjs", + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", + Observers: "resource://services-common/observers.sys.mjs", + Resource: "resource://services-sync/resource.sys.mjs", + Status: "resource://services-sync/status.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + TelemetryController: "resource://gre/modules/TelemetryController.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetryUtils: "resource://gre/modules/TelemetryUtils.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +import * as constants from "resource://services-sync/constants.sys.mjs"; + +ChromeUtils.defineLazyGetter( + lazy, + "WeaveService", + () => Cc["@mozilla.org/weave/service;1"].getService().wrappedJSObject +); +const log = Log.repository.getLogger("Sync.Telemetry"); + +const TOPICS = [ + // For tracking change to account/device identifiers. + "fxaccounts:new_device_id", + "fxaccounts:onlogout", + "weave:service:ready", + "weave:service:login:got-hashed-id", + + // For whole-of-sync metrics. + "weave:service:sync:start", + "weave:service:sync:finish", + "weave:service:sync:error", + + // For individual engine metrics. + "weave:engine:sync:start", + "weave:engine:sync:finish", + "weave:engine:sync:error", + "weave:engine:sync:applied", + "weave:engine:sync:step", + "weave:engine:sync:uploaded", + "weave:engine:validate:finish", + "weave:engine:validate:error", + + // For ad-hoc telemetry events. + "weave:telemetry:event", + "weave:telemetry:histogram", + "fxa:telemetry:event", + + "weave:telemetry:migration", +]; + +const PING_FORMAT_VERSION = 1; + +const EMPTY_UID = "0".repeat(32); + +// The set of engines we record telemetry for - any other engines are ignored. +const ENGINES = new Set([ + "addons", + "bookmarks", + "clients", + "forms", + "history", + "passwords", + "prefs", + "tabs", + "extension-storage", + "addresses", + "creditcards", +]); + +function tryGetMonotonicTimestamp() { + try { + return Services.telemetry.msSinceProcessStart(); + } catch (e) { + log.warn("Unable to get a monotonic timestamp!"); + return -1; + } +} + +function timeDeltaFrom(monotonicStartTime) { + let now = tryGetMonotonicTimestamp(); + if (monotonicStartTime !== -1 && now !== -1) { + return Math.round(now - monotonicStartTime); + } + return -1; +} + +const NS_ERROR_MODULE_BASE_OFFSET = 0x45; +const NS_ERROR_MODULE_NETWORK = 6; + +// A reimplementation of NS_ERROR_GET_MODULE, which surprisingly doesn't seem +// to exist anywhere in .js code in a way that can be reused. +// This is taken from DownloadCore.sys.mjs. +function NS_ERROR_GET_MODULE(code) { + return ((code & 0x7fff0000) >> 16) - NS_ERROR_MODULE_BASE_OFFSET; +} + +// Converts extra integer fields to strings, rounds floats to three +// decimal places (nanosecond precision for timings), and removes profile +// directory paths and URLs from potential error messages. +function normalizeExtraTelemetryFields(extra) { + let result = {}; + for (let key in extra) { + let value = extra[key]; + let type = typeof value; + if (type == "string") { + result[key] = ErrorSanitizer.cleanErrorMessage(value); + } else if (type == "number") { + result[key] = Number.isInteger(value) + ? value.toString(10) + : value.toFixed(3); + } else if (type != "undefined") { + throw new TypeError( + `Invalid type ${type} for extra telemetry field ${key}` + ); + } + } + return lazy.ObjectUtils.isEmpty(result) ? undefined : result; +} + +// Keps track of the counts of individual records fate during a sync cycle +// The main reason this is a class is to keep track of reasons individual records +// failure reasons without huge memory overhead. +export class SyncedRecordsTelemetry { + // applied => number of items that should be applied. + // failed => number of items that failed in this sync. + // newFailed => number of items that failed for the first time in this sync. + // reconciled => number of items that were reconciled. + // failedReasons => {name, count} of reasons a record failed + incomingCounts = { + applied: 0, + failed: 0, + newFailed: 0, + reconciled: 0, + failedReasons: null, + }; + outgoingCounts = { failed: 0, sent: 0, failedReasons: null }; + + addIncomingFailedReason(reason) { + if (!this.incomingCounts.failedReasons) { + this.incomingCounts.failedReasons = []; + } + let transformedReason = SyncTelemetry.transformError(reason); + // Some errors like http/nss errors don't have an error object + // those will be caught by the higher level telemetry + if (!transformedReason.error) { + return; + } + + let index = this.incomingCounts.failedReasons.findIndex( + reasons => reasons.name === transformedReason.error + ); + + if (index >= 0) { + this.incomingCounts.failedReasons[index].count += 1; + } else { + this.incomingCounts.failedReasons.push({ + name: transformedReason.error, + count: 1, + }); + } + } + + addOutgoingFailedReason(reason) { + if (!this.outgoingCounts.failedReasons) { + this.outgoingCounts.failedReasons = []; + } + let transformedReason = SyncTelemetry.transformError(reason); + // Some errors like http/nss errors don't have an error object + // those will be caught by the higher level telemetry + if (!transformedReason.error) { + return; + } + let index = this.outgoingCounts.failedReasons.findIndex( + reasons => reasons.name === transformedReason.error + ); + if (index >= 0) { + this.outgoingCounts.failedReasons[index].count += 1; + } else { + this.outgoingCounts.failedReasons.push({ + name: transformedReason.error, + count: 1, + }); + } + } +} + +// The `ErrorSanitizer` has 2 main jobs: +// * Remove PII from errors, such as the profile dir or URLs the user might +// have visited. +// * Normalize errors so different locales or operating systems etc don't +// generate different messages for the same underlying error. +// * [TODO] Normalize errors so environmental factors don't influence message. +// For example, timestamps or GUIDs should be replaced with something static. +export class ErrorSanitizer { + // Things we normalize - this isn't exhaustive, but covers the common error messages we see. + // Eg: + // > Win error 112 during operation write on file [profileDir]\weave\addonsreconciler.json (Espacio en disco insuficiente. ) + // > Win error 112 during operation write on file [profileDir]\weave\addonsreconciler.json (Diskte yeterli yer yok. ) + // > <snip many other translations of the error> + // > Unix error 28 during operation write on file [profileDir]/weave/addonsreconciler.json (No space left on device) + // These tend to crowd out other errors we might care about (eg, top 5 errors for some engines are + // variations of the "no space left on device") + + // Note that only errors that have same-but-different errors on Windows and Unix are here - we + // still sanitize ones that aren't in these maps to remove the translations etc - eg, + // `ERROR_SHARING_VIOLATION` doesn't really have a unix equivalent, so doesn't appear here, but + // we still strip the translations to avoid the variants. + static E_PERMISSION_DENIED = "OS error [Permission denied]"; + static E_NO_FILE_OR_DIR = "OS error [File/Path not found]"; + + static DOMErrorSubstitutions = { + NotFoundError: this.E_NO_FILE_OR_DIR, + NotAllowedError: this.E_PERMISSION_DENIED, + }; + + static #cleanOSErrorMessage(message, error = undefined) { + if (DOMException.isInstance(error)) { + const sub = this.DOMErrorSubstitutions[error.name]; + message = message.replaceAll("\\", "/"); + if (sub) { + return `${sub} ${message}`; + } + } + + return message; + } + + // A regex we can use to replace the profile dir in error messages. We use a + // regexp so we can simply replace all case-insensitive occurences. + // This escaping function is from: + // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions + static reProfileDir = new RegExp( + PathUtils.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), + "gi" + ); + + /** + * Clean an error message, removing PII and normalizing OS-specific messages. + * + * @param {string} message The error message + * @param {Error?} error The error class instance, if any. + */ + static cleanErrorMessage(message, error = undefined) { + // There's a chance the profiledir is in the error string which is PII we + // want to avoid including in the ping. + message = message.replace(this.reProfileDir, "[profileDir]"); + // MSG_INVALID_URL from /dom/bindings/Errors.msg -- no way to access this + // directly from JS. + if (message.endsWith("is not a valid URL.")) { + message = "<URL> is not a valid URL."; + } + // Try to filter things that look somewhat like a URL (in that they contain a + // colon in the middle of non-whitespace), in case anything else is including + // these in error messages. Note that JSON.stringified stuff comes through + // here, so we explicitly ignore double-quotes as well. + message = message.replace(/[^\s"]+:[^\s"]+/g, "<URL>"); + + // Anywhere that's normalized the guid in errors we can easily filter + // to make it easier to aggregate these types of errors + message = message.replace(/<guid: ([^>]+)>/g, "<GUID>"); + + return this.#cleanOSErrorMessage(message, error); + } +} + +// This function validates the payload of a telemetry "event" - this can be +// removed once there are APIs available for the telemetry modules to collect +// these events (bug 1329530) - but for now we simulate that planned API as +// best we can. +function validateTelemetryEvent(eventDetails) { + let { object, method, value, extra } = eventDetails; + // Do do basic validation of the params - everything except "extra" must + // be a string. method and object are required. + if ( + typeof method != "string" || + typeof object != "string" || + (value && typeof value != "string") || + (extra && typeof extra != "object") + ) { + log.warn("Invalid event parameters - wrong types", eventDetails); + return false; + } + // length checks. + if ( + method.length > 20 || + object.length > 20 || + (value && value.length > 80) + ) { + log.warn("Invalid event parameters - wrong lengths", eventDetails); + return false; + } + + // extra can be falsey, or an object with string names and values. + if (extra) { + if (Object.keys(extra).length > 10) { + log.warn("Invalid event parameters - too many extra keys", eventDetails); + return false; + } + for (let [ename, evalue] of Object.entries(extra)) { + if ( + typeof ename != "string" || + ename.length > 15 || + typeof evalue != "string" || + evalue.length > 85 + ) { + log.warn( + `Invalid event parameters: extra item "${ename} is invalid`, + eventDetails + ); + return false; + } + } + } + return true; +} + +class EngineRecord { + constructor(name) { + // startTime is in ms from process start, but is monotonic (unlike Date.now()) + // so we need to keep both it and when. + this.startTime = tryGetMonotonicTimestamp(); + this.name = name; + + // This allows cases like bookmarks-buffered to have a separate name from + // the bookmarks engine. + let engineImpl = lazy.Weave.Service.engineManager.get(name); + if (engineImpl && engineImpl.overrideTelemetryName) { + this.overrideTelemetryName = engineImpl.overrideTelemetryName; + } + } + + toJSON() { + let result = { name: this.overrideTelemetryName || this.name }; + let properties = [ + "took", + "status", + "failureReason", + "incoming", + "outgoing", + "validation", + "steps", + ]; + for (let property of properties) { + result[property] = this[property]; + } + return result; + } + + finished(error) { + let took = timeDeltaFrom(this.startTime); + if (took > 0) { + this.took = took; + } + if (error) { + this.failureReason = SyncTelemetry.transformError(error); + } + } + + recordApplied(counts) { + if (this.incoming) { + log.error( + `Incoming records applied multiple times for engine ${this.name}!` + ); + return; + } + if (this.name === "clients" && !counts.failed) { + // ignore successful application of client records + // since otherwise they show up every time and are meaningless. + return; + } + + let incomingData = {}; + + if (counts.failedReasons) { + // sort the failed reasons in desc by count, then take top 10 + counts.failedReasons = counts.failedReasons + .sort((a, b) => b.count - a.count) + .slice(0, 10); + } + // Counts has extra stuff used for logging, but we only care about a few + let properties = ["applied", "failed", "failedReasons"]; + // Only record non-zero properties and only record incoming at all if + // there's at least one property we care about. + for (let property of properties) { + if (counts[property]) { + incomingData[property] = counts[property]; + this.incoming = incomingData; + } + } + } + + recordStep(stepResult) { + let step = { + name: stepResult.name, + }; + if (stepResult.took > 0) { + step.took = Math.round(stepResult.took); + } + if (stepResult.counts) { + let counts = stepResult.counts.filter(({ count }) => count > 0); + if (counts.length) { + step.counts = counts; + } + } + if (this.steps) { + this.steps.push(step); + } else { + this.steps = [step]; + } + } + + recordValidation(validationResult) { + if (this.validation) { + log.error(`Multiple validations occurred for engine ${this.name}!`); + return; + } + let { problems, version, took, checked } = validationResult; + let validation = { + version: version || 0, + checked: checked || 0, + }; + if (took > 0) { + validation.took = Math.round(took); + } + let summarized = problems.filter(({ count }) => count > 0); + if (summarized.length) { + validation.problems = summarized; + } + this.validation = validation; + } + + recordValidationError(e) { + if (this.validation) { + log.error(`Multiple validations occurred for engine ${this.name}!`); + return; + } + + this.validation = { + failureReason: SyncTelemetry.transformError(e), + }; + } + + recordUploaded(counts) { + if (counts.sent || counts.failed) { + if (!this.outgoing) { + this.outgoing = []; + } + if (counts.failedReasons) { + // sort the failed reasons in desc by count, then take top 10 + counts.failedReasons = counts.failedReasons + .sort((a, b) => b.count - a.count) + .slice(0, 10); + } + this.outgoing.push({ + sent: counts.sent || undefined, + failed: counts.failed || undefined, + failedReasons: counts.failedReasons || undefined, + }); + } + } +} + +// The record of a single "sync" - typically many of these are submitted in +// a single ping (ie, as a 'syncs' array) +export class SyncRecord { + constructor(allowedEngines, why) { + this.allowedEngines = allowedEngines; + // Our failure reason. This property only exists in the generated ping if an + // error actually occurred. + this.failureReason = undefined; + this.syncNodeType = null; + this.when = Date.now(); + this.startTime = tryGetMonotonicTimestamp(); + this.took = 0; // will be set later. + this.why = why; + + // All engines that have finished (ie, does not include the "current" one) + // We omit this from the ping if it's empty. + this.engines = []; + // The engine that has started but not yet stopped. + this.currentEngine = null; + } + + toJSON() { + let result = { + when: this.when, + took: this.took, + failureReason: this.failureReason, + status: this.status, + }; + if (this.why) { + result.why = this.why; + } + let engines = []; + for (let engine of this.engines) { + engines.push(engine.toJSON()); + } + if (engines.length) { + result.engines = engines; + } + return result; + } + + finished(error) { + this.took = timeDeltaFrom(this.startTime); + if (this.currentEngine != null) { + log.error( + "Finished called for the sync before the current engine finished" + ); + this.currentEngine.finished(null); + this.onEngineStop(this.currentEngine.name); + } + if (error) { + this.failureReason = SyncTelemetry.transformError(error); + } + + this.syncNodeType = lazy.Weave.Service.identity.telemetryNodeType; + + // Check for engine statuses. -- We do this now, and not in engine.finished + // to make sure any statuses that get set "late" are recorded + for (let engine of this.engines) { + let status = lazy.Status.engines[engine.name]; + if (status && status !== constants.ENGINE_SUCCEEDED) { + engine.status = status; + } + } + + let statusObject = {}; + + let serviceStatus = lazy.Status.service; + if (serviceStatus && serviceStatus !== constants.STATUS_OK) { + statusObject.service = serviceStatus; + this.status = statusObject; + } + let syncStatus = lazy.Status.sync; + if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) { + statusObject.sync = syncStatus; + this.status = statusObject; + } + } + + onEngineStart(engineName) { + if (this._shouldIgnoreEngine(engineName, false)) { + return; + } + + if (this.currentEngine) { + log.error( + `Being told that engine ${engineName} has started, but current engine ${this.currentEngine.name} hasn't stopped` + ); + // Just discard the current engine rather than making up data for it. + } + this.currentEngine = new EngineRecord(engineName); + } + + onEngineStop(engineName, error) { + // We only care if it's the current engine if we have a current engine. + if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) { + return; + } + if (!this.currentEngine) { + // It's possible for us to get an error before the start message of an engine + // (somehow), in which case we still want to record that error. + if (!error) { + return; + } + log.error( + `Error triggered on ${engineName} when no current engine exists: ${error}` + ); + this.currentEngine = new EngineRecord(engineName); + } + this.currentEngine.finished(error); + this.engines.push(this.currentEngine); + this.currentEngine = null; + } + + onEngineApplied(engineName, counts) { + if (this._shouldIgnoreEngine(engineName)) { + return; + } + this.currentEngine.recordApplied(counts); + } + + onEngineStep(engineName, step) { + if (this._shouldIgnoreEngine(engineName)) { + return; + } + this.currentEngine.recordStep(step); + } + + onEngineValidated(engineName, validationData) { + if (this._shouldIgnoreEngine(engineName, false)) { + return; + } + let engine = this.engines.find(e => e.name === engineName); + if ( + !engine && + this.currentEngine && + engineName === this.currentEngine.name + ) { + engine = this.currentEngine; + } + if (engine) { + engine.recordValidation(validationData); + } else { + log.warn( + `Validation event triggered for engine ${engineName}, which hasn't been synced!` + ); + } + } + + onEngineValidateError(engineName, error) { + if (this._shouldIgnoreEngine(engineName, false)) { + return; + } + let engine = this.engines.find(e => e.name === engineName); + if ( + !engine && + this.currentEngine && + engineName === this.currentEngine.name + ) { + engine = this.currentEngine; + } + if (engine) { + engine.recordValidationError(error); + } else { + log.warn( + `Validation failure event triggered for engine ${engineName}, which hasn't been synced!` + ); + } + } + + onEngineUploaded(engineName, counts) { + if (this._shouldIgnoreEngine(engineName)) { + return; + } + this.currentEngine.recordUploaded(counts); + } + + _shouldIgnoreEngine(engineName, shouldBeCurrent = true) { + if (!this.allowedEngines.has(engineName)) { + log.info( + `Notification for engine ${engineName}, but we aren't recording telemetry for it` + ); + return true; + } + if (shouldBeCurrent) { + if (!this.currentEngine || engineName != this.currentEngine.name) { + log.info(`Notification for engine ${engineName} but it isn't current`); + return true; + } + } + return false; + } +} + +// The entire "sync ping" - it includes all the syncs, events etc recorded in +// the ping. +class SyncTelemetryImpl { + constructor(allowedEngines) { + log.manageLevelFromPref("services.sync.log.logger.telemetry"); + // This is accessible so we can enable custom engines during tests. + this.allowedEngines = allowedEngines; + this.current = null; + this.setupObservers(); + + this.payloads = []; + this.discarded = 0; + this.events = []; + this.histograms = {}; + this.migrations = []; + this.maxEventsCount = lazy.Svc.PrefBranch.getIntPref( + "telemetry.maxEventsCount", + 1000 + ); + this.maxPayloadCount = lazy.Svc.PrefBranch.getIntPref( + "telemetry.maxPayloadCount" + ); + this.submissionInterval = + lazy.Svc.PrefBranch.getIntPref("telemetry.submissionInterval") * 1000; + this.lastSubmissionTime = Services.telemetry.msSinceProcessStart(); + this.lastUID = EMPTY_UID; + this.lastSyncNodeType = null; + this.currentSyncNodeType = null; + // Note that the sessionStartDate is somewhat arbitrary - the telemetry + // modules themselves just use `new Date()`. This means that our startDate + // isn't going to be the same as the sessionStartDate in the main pings, + // but that's OK for now - if it's a problem we'd need to change the + // telemetry modules to expose what it thinks the sessionStartDate is. + let sessionStartDate = new Date(); + this.sessionStartDate = lazy.TelemetryUtils.toLocalTimeISOString( + lazy.TelemetryUtils.truncateToHours(sessionStartDate) + ); + lazy.TelemetryController.registerSyncPingShutdown(() => this.shutdown()); + } + + sanitizeFxaDeviceId(deviceId) { + return lazy.fxAccounts.telemetry.sanitizeDeviceId(deviceId); + } + + prepareFxaDevices(devices) { + // For non-sync users, the data per device is limited -- just an id and a + // type (and not even the id yet). For sync users, if we can correctly map + // the fxaDevice to a sync device, then we can get os and version info, + // which would be quite unfortunate to lose. + let extraInfoMap = new Map(); + if (this.syncIsEnabled()) { + for (let client of this.getClientsEngineRecords()) { + if (client.fxaDeviceId) { + extraInfoMap.set(client.fxaDeviceId, { + os: client.os, + version: client.version, + syncID: this.sanitizeFxaDeviceId(client.id), + }); + } + } + } + // Finally, sanitize and convert to the proper format. + return devices.map(d => { + let { os, version, syncID } = extraInfoMap.get(d.id) || { + os: undefined, + version: undefined, + syncID: undefined, + }; + return { + id: this.sanitizeFxaDeviceId(d.id) || EMPTY_UID, + type: d.type, + os, + version, + syncID, + }; + }); + } + + syncIsEnabled() { + return lazy.WeaveService.enabled && lazy.WeaveService.ready; + } + + // Separate for testing. + getClientsEngineRecords() { + if (!this.syncIsEnabled()) { + throw new Error("Bug: syncIsEnabled() must be true, check it first"); + } + return lazy.Weave.Service.clientsEngine.remoteClients; + } + + updateFxaDevices(devices) { + if (!devices) { + return {}; + } + let me = devices.find(d => d.isCurrentDevice); + let id = me ? this.sanitizeFxaDeviceId(me.id) : undefined; + let cleanDevices = this.prepareFxaDevices(devices); + return { deviceID: id, devices: cleanDevices }; + } + + getFxaDevices() { + return lazy.fxAccounts.device.recentDeviceList; + } + + getPingJSON(reason) { + let { devices, deviceID } = this.updateFxaDevices(this.getFxaDevices()); + return { + os: lazy.TelemetryEnvironment.currentEnvironment.system.os, + why: reason, + devices, + discarded: this.discarded || undefined, + version: PING_FORMAT_VERSION, + syncs: this.payloads.slice(), + uid: this.lastUID, + syncNodeType: this.lastSyncNodeType || undefined, + deviceID, + sessionStartDate: this.sessionStartDate, + events: !this.events.length ? undefined : this.events, + migrations: !this.migrations.length ? undefined : this.migrations, + histograms: !Object.keys(this.histograms).length + ? undefined + : this.histograms, + }; + } + + _addMigrationRecord(type, info) { + log.debug("Saw telemetry migration info", type, info); + // Updates to this need to be documented in `sync-ping.rst` + switch (type) { + case "webext-storage": + this.migrations.push({ + type: "webext-storage", + entries: +info.entries, + entriesSuccessful: +info.entries_successful, + extensions: +info.extensions, + extensionsSuccessful: +info.extensions_successful, + openFailure: !!info.open_failure, + }); + break; + default: + throw new Error("Bug: Unknown migration record type " + type); + } + } + + finish(reason) { + // Note that we might be in the middle of a sync right now, and so we don't + // want to touch this.current. + let result = this.getPingJSON(reason); + this.payloads = []; + this.discarded = 0; + this.events = []; + this.migrations = []; + this.histograms = {}; + this.submit(result); + } + + setupObservers() { + for (let topic of TOPICS) { + lazy.Observers.add(topic, this, this); + } + } + + shutdown() { + this.finish("shutdown"); + for (let topic of TOPICS) { + lazy.Observers.remove(topic, this, this); + } + } + + submit(record) { + if (!this.isProductionSyncUser()) { + return false; + } + // We still call submit() with possibly illegal payloads so that tests can + // know that the ping was built. We don't end up submitting them, however. + let numEvents = record.events ? record.events.length : 0; + let numMigrations = record.migrations ? record.migrations.length : 0; + if (record.syncs.length || numEvents || numMigrations) { + log.trace( + `submitting ${record.syncs.length} sync record(s) and ` + + `${numEvents} event(s) to telemetry` + ); + lazy.TelemetryController.submitExternalPing("sync", record, { + usePingSender: true, + }).catch(err => { + log.error("failed to submit ping", err); + }); + return true; + } + return false; + } + + isProductionSyncUser() { + // If FxA isn't production then we treat sync as not being production. + // Further, there's the deprecated "services.sync.tokenServerURI" pref we + // need to consider - fxa doesn't consider that as if that's the only + // pref set, they *are* running a production fxa, just not production sync. + if ( + !lazy.FxAccounts.config.isProductionConfig() || + Services.prefs.prefHasUserValue("services.sync.tokenServerURI") + ) { + log.trace(`Not sending telemetry ping for self-hosted Sync user`); + return false; + } + return true; + } + + onSyncStarted(data) { + const why = data && JSON.parse(data).why; + if (this.current) { + log.warn( + "Observed weave:service:sync:start, but we're already recording a sync!" + ); + // Just discard the old record, consistent with our handling of engines, above. + this.current = null; + } + this.current = new SyncRecord(this.allowedEngines, why); + } + + // We need to ensure that the telemetry `deletion-request` ping always contains the user's + // current sync device ID, because if the user opts out of telemetry then the deletion ping + // will be immediately triggered for sending, and we won't have a chance to fill it in later. + // This keeps the `deletion-ping` up-to-date when the user's account state changes. + onAccountInitOrChange() { + // We don't submit sync pings for self-hosters, so don't need to collect their device ids either. + if (!this.isProductionSyncUser()) { + return; + } + // Awkwardly async, but no need to await. If the user's account state changes while + // this promise is in flight, it will reject and we won't record any data in the ping. + // (And a new notification will trigger us to try again with the new state). + lazy.fxAccounts.device + .getLocalId() + .then(deviceId => { + let sanitizedDeviceId = + lazy.fxAccounts.telemetry.sanitizeDeviceId(deviceId); + // In the past we did not persist the FxA metrics identifiers to disk, + // so this might be missing until we can fetch it from the server for the + // first time. There will be a fresh notification tirggered when it's available. + if (sanitizedDeviceId) { + // Sanitized device ids are 64 characters long, but telemetry limits scalar strings to 50. + // The first 32 chars are sufficient to uniquely identify the device, so just send those. + // It's hard to change the sync ping itself to only send 32 chars, to b/w compat reasons. + sanitizedDeviceId = sanitizedDeviceId.substr(0, 32); + Services.telemetry.scalarSet( + "deletion.request.sync_device_id", + sanitizedDeviceId + ); + } + }) + .catch(err => { + log.warn( + `Failed to set sync identifiers in the deletion-request ping: ${err}` + ); + }); + } + + // This keeps the `deletion-request` ping up-to-date when the user signs out, + // clearing the now-nonexistent sync device id. + onAccountLogout() { + Services.telemetry.scalarSet("deletion.request.sync_device_id", ""); + } + + _checkCurrent(topic) { + if (!this.current) { + // This is only `info` because it happens when we do a tabs "quick-write" + log.info( + `Observed notification ${topic} but no current sync is being recorded.` + ); + return false; + } + return true; + } + + _shouldSubmitForDataChange() { + let newID = lazy.fxAccounts.telemetry.getSanitizedUID() || EMPTY_UID; + let oldID = this.lastUID; + if ( + newID != EMPTY_UID && + oldID != EMPTY_UID && + // Both are "real" uids, so we care if they've changed. + newID != oldID + ) { + log.trace( + `shouldSubmitForDataChange - uid from '${oldID}' -> '${newID}'` + ); + return true; + } + // We've gone from knowing one of the ids to not knowing it (which we + // ignore) or we've gone from not knowing it to knowing it (which is fine), + // Now check the node type because a change there also means we should + // submit. + if ( + this.lastSyncNodeType && + this.currentSyncNodeType != this.lastSyncNodeType + ) { + log.trace( + `shouldSubmitForDataChange - nodeType from '${this.lastSyncNodeType}' -> '${this.currentSyncNodeType}'` + ); + return true; + } + log.trace("shouldSubmitForDataChange - no need to submit"); + return false; + } + + maybeSubmitForDataChange() { + if (this._shouldSubmitForDataChange()) { + log.info( + "Early submission of sync telemetry due to changed IDs/NodeType" + ); + this.finish("idchange"); // this actually submits. + this.lastSubmissionTime = Services.telemetry.msSinceProcessStart(); + } + + // Only update the last UIDs if we actually know them. + let current_uid = lazy.fxAccounts.telemetry.getSanitizedUID(); + if (current_uid) { + this.lastUID = current_uid; + } + if (this.currentSyncNodeType) { + this.lastSyncNodeType = this.currentSyncNodeType; + } + } + + maybeSubmitForInterval() { + // We want to submit the ping every `this.submissionInterval` but only when + // there's no current sync in progress, otherwise we may end up submitting + // the sync and the events caused by it in different pings. + if ( + this.current == null && + Services.telemetry.msSinceProcessStart() - this.lastSubmissionTime > + this.submissionInterval + ) { + this.finish("schedule"); + this.lastSubmissionTime = Services.telemetry.msSinceProcessStart(); + } + } + + onSyncFinished(error) { + if (!this.current) { + log.warn("onSyncFinished but we aren't recording"); + return; + } + this.current.finished(error); + this.currentSyncNodeType = this.current.syncNodeType; + let current = this.current; + this.current = null; + this.takeTelemetryRecord(current); + } + + takeTelemetryRecord(record) { + // We check for "data change" before appending the current sync to payloads, + // as it is the current sync which has the data with the new data, and thus + // must go in the *next* submission. + this.maybeSubmitForDataChange(); + if (this.payloads.length < this.maxPayloadCount) { + this.payloads.push(record.toJSON()); + } else { + ++this.discarded; + } + // If we are submitting due to timing, it's desirable that the most recent + // sync is included, so we check after appending the record. + this.maybeSubmitForInterval(); + } + + _addHistogram(hist) { + let histogram = Services.telemetry.getHistogramById(hist); + let s = histogram.snapshot(); + this.histograms[hist] = s; + } + + _recordEvent(eventDetails) { + this.maybeSubmitForDataChange(); + + if (this.events.length >= this.maxEventsCount) { + log.warn("discarding event - already queued our maximum", eventDetails); + return; + } + + let { object, method, value, extra } = eventDetails; + if (extra) { + extra = normalizeExtraTelemetryFields(extra); + eventDetails = { object, method, value, extra }; + } + + if (!validateTelemetryEvent(eventDetails)) { + // we've already logged what the problem is... + return; + } + log.debug("recording event", eventDetails); + + if (extra && lazy.Resource.serverTime && !extra.serverTime) { + extra.serverTime = String(lazy.Resource.serverTime); + } + let category = "sync"; + let ts = Math.floor(tryGetMonotonicTimestamp()); + + // An event record is a simple array with at least 4 items. + let event = [ts, category, method, object]; + // It may have up to 6 elements if |extra| is defined + if (value) { + event.push(value); + if (extra) { + event.push(extra); + } + } else if (extra) { + event.push(null); // a null for the empty value. + event.push(extra); + } + this.events.push(event); + this.maybeSubmitForInterval(); + } + + observe(subject, topic, data) { + log.trace(`observed ${topic} ${data}`); + + switch (topic) { + case "weave:service:ready": + case "weave:service:login:got-hashed-id": + case "fxaccounts:new_device_id": + this.onAccountInitOrChange(); + break; + + case "fxaccounts:onlogout": + this.onAccountLogout(); + break; + + /* sync itself state changes */ + case "weave:service:sync:start": + this.onSyncStarted(data); + break; + + case "weave:service:sync:finish": + if (this._checkCurrent(topic)) { + this.onSyncFinished(null); + } + break; + + case "weave:service:sync:error": + // argument needs to be truthy (this should always be the case) + this.onSyncFinished(subject || "Unknown"); + break; + + /* engine sync state changes */ + case "weave:engine:sync:start": + if (this._checkCurrent(topic)) { + this.current.onEngineStart(data); + } + break; + case "weave:engine:sync:finish": + if (this._checkCurrent(topic)) { + this.current.onEngineStop(data, null); + } + break; + + case "weave:engine:sync:error": + if (this._checkCurrent(topic)) { + // argument needs to be truthy (this should always be the case) + this.current.onEngineStop(data, subject || "Unknown"); + } + break; + + /* engine counts */ + case "weave:engine:sync:applied": + if (this._checkCurrent(topic)) { + this.current.onEngineApplied(data, subject); + } + break; + + case "weave:engine:sync:step": + if (this._checkCurrent(topic)) { + this.current.onEngineStep(data, subject); + } + break; + + case "weave:engine:sync:uploaded": + if (this._checkCurrent(topic)) { + this.current.onEngineUploaded(data, subject); + } + break; + + case "weave:engine:validate:finish": + if (this._checkCurrent(topic)) { + this.current.onEngineValidated(data, subject); + } + break; + + case "weave:engine:validate:error": + if (this._checkCurrent(topic)) { + this.current.onEngineValidateError(data, subject || "Unknown"); + } + break; + + case "weave:telemetry:event": + case "fxa:telemetry:event": + this._recordEvent(subject); + break; + + case "weave:telemetry:histogram": + this._addHistogram(data); + break; + + case "weave:telemetry:migration": + this._addMigrationRecord(data, subject); + break; + + default: + log.warn(`unexpected observer topic ${topic}`); + break; + } + } + + // Transform an exception into a standard description. Exposed here for when + // this module isn't directly responsible for knowing the transform should + // happen (for example, when including an error in the |extra| field of + // event telemetry) + transformError(error) { + // Certain parts of sync will use this pattern as a way to communicate to + // processIncoming to abort the processing. However, there's no guarantee + // this can only happen then. + if (typeof error == "object" && error.code && error.cause) { + error = error.cause; + } + if (lazy.Async.isShutdownException(error)) { + return { name: "shutdownerror" }; + } + + if (typeof error === "string") { + if (error.startsWith("error.")) { + // This is hacky, but I can't imagine that it's not also accurate. + return { name: "othererror", error }; + } + error = ErrorSanitizer.cleanErrorMessage(error); + return { name: "unexpectederror", error }; + } + + if (error instanceof lazy.AuthenticationError) { + return { name: "autherror", from: error.source }; + } + + if (DOMException.isInstance(error)) { + return { + name: "unexpectederror", + error: ErrorSanitizer.cleanErrorMessage(error.message, error), + }; + } + + let httpCode = + error.status || (error.response && error.response.status) || error.code; + + if (httpCode) { + return { name: "httperror", code: httpCode }; + } + + if (error.failureCode) { + return { name: "othererror", error: error.failureCode }; + } + + if (error.result) { + // many "nsresult" errors are actually network errors - if they are + // associated with the "network" module we assume that's true. + // We also assume NS_ERROR_ABORT is such an error - for almost everything + // we care about, it acually is (eg, if the connection fails early enough + // or if we have a captive portal etc) - we don't lose anything by this + // assumption, it's just that the error will no longer be in the "nserror" + // category, so our analysis can still find them. + if ( + error.result == Cr.NS_ERROR_ABORT || + NS_ERROR_GET_MODULE(error.result) == NS_ERROR_MODULE_NETWORK + ) { + return { name: "httperror", code: error.result }; + } + return { name: "nserror", code: error.result }; + } + // It's probably an Error object, but it also could be some + // other object that may or may not override toString to do + // something useful. + let msg = String(error); + if (msg.startsWith("[object")) { + // Nothing useful in the default, check for a string "message" property. + if (typeof error.message == "string") { + msg = String(error.message); + } else { + // Hopefully it won't come to this... + msg = JSON.stringify(error); + } + } + return { + name: "unexpectederror", + error: ErrorSanitizer.cleanErrorMessage(msg), + }; + } +} + +export var SyncTelemetry = new SyncTelemetryImpl(ENGINES); diff --git a/services/sync/modules/util.sys.mjs b/services/sync/modules/util.sys.mjs new file mode 100644 index 0000000000..e68ae0e19f --- /dev/null +++ b/services/sync/modules/util.sys.mjs @@ -0,0 +1,780 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Observers } from "resource://services-common/observers.sys.mjs"; + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs"; + +import { + DEVICE_TYPE_DESKTOP, + MAXIMUM_BACKOFF_INTERVAL, + PREFS_BRANCH, + SYNC_KEY_DECODED_LENGTH, + SYNC_KEY_ENCODED_LENGTH, + WEAVE_VERSION, +} from "resource://services-sync/constants.sys.mjs"; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; +import * as FxAccountsCommon from "resource://gre/modules/FxAccountsCommon.sys.mjs"; + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "cryptoSDR", + "@mozilla.org/login-manager/crypto/SDR;1", + "nsILoginManagerCrypto" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "localDeviceType", + "services.sync.client.type", + DEVICE_TYPE_DESKTOP +); + +/* + * Custom exception types. + */ +class LockException extends Error { + constructor(message) { + super(message); + this.name = "LockException"; + } +} + +class HMACMismatch extends Error { + constructor(message) { + super(message); + this.name = "HMACMismatch"; + } +} + +/* + * Utility functions + */ +export var Utils = { + // Aliases from CryptoUtils. + generateRandomBytesLegacy: CryptoUtils.generateRandomBytesLegacy, + computeHTTPMACSHA1: CryptoUtils.computeHTTPMACSHA1, + digestUTF8: CryptoUtils.digestUTF8, + digestBytes: CryptoUtils.digestBytes, + sha256: CryptoUtils.sha256, + hkdfExpand: CryptoUtils.hkdfExpand, + pbkdf2Generate: CryptoUtils.pbkdf2Generate, + getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header, + + /** + * The string to use as the base User-Agent in Sync requests. + * This string will look something like + * + * Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop + */ + _userAgent: null, + get userAgent() { + if (!this._userAgent) { + let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService( + Ci.nsIHttpProtocolHandler + ); + /* eslint-disable no-multi-spaces */ + this._userAgent = + Services.appinfo.name + + "/" + + Services.appinfo.version + // Product. + " (" + + hph.oscpu + + ")" + // (oscpu) + " FxSync/" + + WEAVE_VERSION + + "." + // Sync. + Services.appinfo.appBuildID + + "."; // Build. + /* eslint-enable no-multi-spaces */ + } + return this._userAgent + lazy.localDeviceType; + }, + + /** + * Wrap a [promise-returning] function to catch all exceptions and log them. + * + * @usage MyObj._catch = Utils.catch; + * MyObj.foo = function() { this._catch(func)(); } + * + * Optionally pass a function which will be called if an + * exception occurs. + */ + catch(func, exceptionCallback) { + let thisArg = this; + return async function WrappedCatch() { + try { + return await func.call(thisArg); + } catch (ex) { + thisArg._log.debug( + "Exception calling " + (func.name || "anonymous function"), + ex + ); + if (exceptionCallback) { + return exceptionCallback.call(thisArg, ex); + } + return null; + } + }; + }, + + throwLockException(label) { + throw new LockException(`Could not acquire lock. Label: "${label}".`); + }, + + /** + * Wrap a [promise-returning] function to call lock before calling the function + * then unlock when it finishes executing or if it threw an error. + * + * @usage MyObj._lock = Utils.lock; + * MyObj.foo = async function() { await this._lock(func)(); } + */ + lock(label, func) { + let thisArg = this; + return async function WrappedLock() { + if (!thisArg.lock()) { + Utils.throwLockException(label); + } + + try { + return await func.call(thisArg); + } finally { + thisArg.unlock(); + } + }; + }, + + isLockException: function isLockException(ex) { + return ex instanceof LockException; + }, + + /** + * Wrap [promise-returning] functions to notify when it starts and + * finishes executing or if it threw an error. + * + * The message is a combination of a provided prefix, the local name, and + * the event. Possible events are: "start", "finish", "error". The subject + * is the function's return value on "finish" or the caught exception on + * "error". The data argument is the predefined data value. + * + * Example: + * + * @usage function MyObj(name) { + * this.name = name; + * this._notify = Utils.notify("obj:"); + * } + * MyObj.prototype = { + * foo: function() this._notify("func", "data-arg", async function () { + * //... + * }(), + * }; + */ + notify(prefix) { + return function NotifyMaker(name, data, func) { + let thisArg = this; + let notify = function (state, subject) { + let mesg = prefix + name + ":" + state; + thisArg._log.trace("Event: " + mesg); + Observers.notify(mesg, subject, data); + }; + + return async function WrappedNotify() { + notify("start", null); + try { + let ret = await func.call(thisArg); + notify("finish", ret); + return ret; + } catch (ex) { + notify("error", ex); + throw ex; + } + }; + }; + }, + + /** + * GUIDs are 9 random bytes encoded with base64url (RFC 4648). + * That makes them 12 characters long with 72 bits of entropy. + */ + makeGUID: function makeGUID() { + return CommonUtils.encodeBase64URL(Utils.generateRandomBytesLegacy(9)); + }, + + _base64url_regex: /^[-abcdefghijklmnopqrstuvwxyz0123456789_]{12}$/i, + checkGUID: function checkGUID(guid) { + return !!guid && this._base64url_regex.test(guid); + }, + + /** + * Add a simple getter/setter to an object that defers access of a property + * to an inner property. + * + * @param obj + * Object to add properties to defer in its prototype + * @param defer + * Property of obj to defer to + * @param prop + * Property name to defer (or an array of property names) + */ + deferGetSet: function Utils_deferGetSet(obj, defer, prop) { + if (Array.isArray(prop)) { + return prop.map(prop => Utils.deferGetSet(obj, defer, prop)); + } + + let prot = obj.prototype; + + // Create a getter if it doesn't exist yet + if (!prot.__lookupGetter__(prop)) { + prot.__defineGetter__(prop, function () { + return this[defer][prop]; + }); + } + + // Create a setter if it doesn't exist yet + if (!prot.__lookupSetter__(prop)) { + prot.__defineSetter__(prop, function (val) { + this[defer][prop] = val; + }); + } + }, + + deepEquals: function eq(a, b) { + // If they're triple equals, then it must be equals! + if (a === b) { + return true; + } + + // If they weren't equal, they must be objects to be different + if (typeof a != "object" || typeof b != "object") { + return false; + } + + // But null objects won't have properties to compare + if (a === null || b === null) { + return false; + } + + // Make sure all of a's keys have a matching value in b + for (let k in a) { + if (!eq(a[k], b[k])) { + return false; + } + } + + // Do the same for b's keys but skip those that we already checked + for (let k in b) { + if (!(k in a) && !eq(a[k], b[k])) { + return false; + } + } + + return true; + }, + + // Generator and discriminator for HMAC exceptions. + // Split these out in case we want to make them richer in future, and to + // avoid inevitable confusion if the message changes. + throwHMACMismatch: function throwHMACMismatch(shouldBe, is) { + throw new HMACMismatch( + `Record SHA256 HMAC mismatch: should be ${shouldBe}, is ${is}` + ); + }, + + isHMACMismatch: function isHMACMismatch(ex) { + return ex instanceof HMACMismatch; + }, + + /** + * Turn RFC 4648 base32 into our own user-friendly version. + * ABCDEFGHIJKLMNOPQRSTUVWXYZ234567 + * becomes + * abcdefghijk8mn9pqrstuvwxyz234567 + */ + base32ToFriendly: function base32ToFriendly(input) { + return input.toLowerCase().replace(/l/g, "8").replace(/o/g, "9"); + }, + + base32FromFriendly: function base32FromFriendly(input) { + return input.toUpperCase().replace(/8/g, "L").replace(/9/g, "O"); + }, + + /** + * Key manipulation. + */ + + // Return an octet string in friendly base32 *with no trailing =*. + encodeKeyBase32: function encodeKeyBase32(keyData) { + return Utils.base32ToFriendly(CommonUtils.encodeBase32(keyData)).slice( + 0, + SYNC_KEY_ENCODED_LENGTH + ); + }, + + decodeKeyBase32: function decodeKeyBase32(encoded) { + return CommonUtils.decodeBase32( + Utils.base32FromFriendly(Utils.normalizePassphrase(encoded)) + ).slice(0, SYNC_KEY_DECODED_LENGTH); + }, + + jsonFilePath(...args) { + let [fileName] = args.splice(-1); + + return PathUtils.join( + PathUtils.profileDir, + "weave", + ...args, + `${fileName}.json` + ); + }, + + /** + * Load a JSON file from disk in the profile directory. + * + * @param filePath + * JSON file path load from profile. Loaded file will be + * extension. + * @param that + * Object to use for logging. + * + * @return Promise<> + * Promise resolved when the write has been performed. + */ + async jsonLoad(filePath, that) { + let path; + if (Array.isArray(filePath)) { + path = Utils.jsonFilePath(...filePath); + } else { + path = Utils.jsonFilePath(filePath); + } + + if (that._log && that._log.trace) { + that._log.trace("Loading json from disk: " + path); + } + + try { + return await IOUtils.readJSON(path); + } catch (e) { + if (!DOMException.isInstance(e) || e.name !== "NotFoundError") { + if (that._log) { + that._log.debug("Failed to load json", e); + } + } + return null; + } + }, + + /** + * Save a json-able object to disk in the profile directory. + * + * @param filePath + * JSON file path save to <filePath>.json + * @param that + * Object to use for logging. + * @param obj + * Function to provide json-able object to save. If this isn't a + * function, it'll be used as the object to make a json string.* + * Function called when the write has been performed. Optional. + * + * @return Promise<> + * Promise resolved when the write has been performed. + */ + async jsonSave(filePath, that, obj) { + let path = PathUtils.join( + PathUtils.profileDir, + "weave", + ...(filePath + ".json").split("/") + ); + let dir = PathUtils.parent(path); + + await IOUtils.makeDirectory(dir, { createAncestors: true }); + + if (that._log) { + that._log.trace("Saving json to disk: " + path); + } + + let json = typeof obj == "function" ? obj.call(that) : obj; + + return IOUtils.writeJSON(path, json); + }, + + /** + * Helper utility function to fit an array of records so that when serialized, + * they will be within payloadSizeMaxBytes. Returns a new array without the + * items. + * + * Note: This shouldn't be used for extremely large record sizes as + * it uses JSON.stringify, which could lead to a heavy performance hit. + * See Bug 1815151 for more details. + * + */ + tryFitItems(records, payloadSizeMaxBytes) { + // Copy this so that callers don't have to do it in advance. + records = records.slice(); + let encoder = Utils.utf8Encoder; + const computeSerializedSize = () => + encoder.encode(JSON.stringify(records)).byteLength; + // Figure out how many records we can pack into a payload. + // We use byteLength here because the data is not encrypted in ascii yet. + let size = computeSerializedSize(); + // See bug 535326 comment 8 for an explanation of the estimation + const maxSerializedSize = (payloadSizeMaxBytes / 4) * 3 - 1500; + if (maxSerializedSize < 0) { + // This is probably due to a test, but it causes very bad behavior if a + // test causes this accidentally. We could throw, but there's an obvious/ + // natural way to handle it, so we do that instead (otherwise we'd have a + // weird lower bound of ~1125b on the max record payload size). + return []; + } + if (size > maxSerializedSize) { + // Estimate a little more than the direct fraction to maximize packing + let cutoff = Math.ceil((records.length * maxSerializedSize) / size); + records = records.slice(0, cutoff + 1); + + // Keep dropping off the last entry until the data fits. + while (computeSerializedSize() > maxSerializedSize) { + records.pop(); + } + } + return records; + }, + + /** + * Move a json file in the profile directory. Will fail if a file exists at the + * destination. + * + * @returns a promise that resolves to undefined on success, or rejects on failure + * + * @param aFrom + * Current path to the JSON file saved on disk, relative to profileDir/weave + * .json will be appended to the file name. + * @param aTo + * New path to the JSON file saved on disk, relative to profileDir/weave + * .json will be appended to the file name. + * @param that + * Object to use for logging + */ + jsonMove(aFrom, aTo, that) { + let pathFrom = PathUtils.join( + PathUtils.profileDir, + "weave", + ...(aFrom + ".json").split("/") + ); + let pathTo = PathUtils.join( + PathUtils.profileDir, + "weave", + ...(aTo + ".json").split("/") + ); + if (that._log) { + that._log.trace("Moving " + pathFrom + " to " + pathTo); + } + return IOUtils.move(pathFrom, pathTo, { noOverwrite: true }); + }, + + /** + * Removes a json file in the profile directory. + * + * @returns a promise that resolves to undefined on success, or rejects on failure + * + * @param filePath + * Current path to the JSON file saved on disk, relative to profileDir/weave + * .json will be appended to the file name. + * @param that + * Object to use for logging + */ + jsonRemove(filePath, that) { + let path = PathUtils.join( + PathUtils.profileDir, + "weave", + ...(filePath + ".json").split("/") + ); + if (that._log) { + that._log.trace("Deleting " + path); + } + return IOUtils.remove(path, { ignoreAbsent: true }); + }, + + /** + * The following are the methods supported for UI use: + * + * * isPassphrase: + * determines whether a string is either a normalized or presentable + * passphrase. + * * normalizePassphrase: + * take a presentable passphrase and reduce it to a normalized + * representation for storage. normalizePassphrase can safely be called + * on normalized input. + */ + + isPassphrase(s) { + if (s) { + return /^[abcdefghijkmnpqrstuvwxyz23456789]{26}$/.test( + Utils.normalizePassphrase(s) + ); + } + return false; + }, + + normalizePassphrase: function normalizePassphrase(pp) { + // Short var name... have you seen the lines below?! + // Allow leading and trailing whitespace. + pp = pp.trim().toLowerCase(); + + // 20-char sync key. + if (pp.length == 23 && [5, 11, 17].every(i => pp[i] == "-")) { + return ( + pp.slice(0, 5) + pp.slice(6, 11) + pp.slice(12, 17) + pp.slice(18, 23) + ); + } + + // "Modern" 26-char key. + if (pp.length == 31 && [1, 7, 13, 19, 25].every(i => pp[i] == "-")) { + return ( + pp.slice(0, 1) + + pp.slice(2, 7) + + pp.slice(8, 13) + + pp.slice(14, 19) + + pp.slice(20, 25) + + pp.slice(26, 31) + ); + } + + // Something else -- just return. + return pp; + }, + + /** + * Create an array like the first but without elements of the second. Reuse + * arrays if possible. + */ + arraySub: function arraySub(minuend, subtrahend) { + if (!minuend.length || !subtrahend.length) { + return minuend; + } + let setSubtrahend = new Set(subtrahend); + return minuend.filter(i => !setSubtrahend.has(i)); + }, + + /** + * Build the union of two arrays. Reuse arrays if possible. + */ + arrayUnion: function arrayUnion(foo, bar) { + if (!foo.length) { + return bar; + } + if (!bar.length) { + return foo; + } + return foo.concat(Utils.arraySub(bar, foo)); + }, + + /** + * Add all the items in `items` to the provided Set in-place. + * + * @return The provided set. + */ + setAddAll(set, items) { + for (let item of items) { + set.add(item); + } + return set; + }, + + /** + * Delete every items in `items` to the provided Set in-place. + * + * @return The provided set. + */ + setDeleteAll(set, items) { + for (let item of items) { + set.delete(item); + } + return set; + }, + + /** + * Take the first `size` items from the Set `items`. + * + * @return A Set of size at most `size` + */ + subsetOfSize(items, size) { + let result = new Set(); + let count = 0; + for (let item of items) { + if (count++ == size) { + return result; + } + result.add(item); + } + return result; + }, + + bind2: function Async_bind2(object, method) { + return function innerBind() { + return method.apply(object, arguments); + }; + }, + + /** + * Is there a master password configured and currently locked? + */ + mpLocked() { + return !lazy.cryptoSDR.isLoggedIn; + }, + + // If Master Password is enabled and locked, present a dialog to unlock it. + // Return whether the system is unlocked. + ensureMPUnlocked() { + if (lazy.cryptoSDR.uiBusy) { + return false; + } + try { + lazy.cryptoSDR.encrypt("bacon"); + return true; + } catch (e) {} + return false; + }, + + /** + * Return a value for a backoff interval. Maximum is eight hours, unless + * Status.backoffInterval is higher. + * + */ + calculateBackoff: function calculateBackoff( + attempts, + baseInterval, + statusInterval + ) { + let backoffInterval = + attempts * (Math.floor(Math.random() * baseInterval) + baseInterval); + return Math.max( + Math.min(backoffInterval, MAXIMUM_BACKOFF_INTERVAL), + statusInterval + ); + }, + + /** + * Return a set of hostnames (including the protocol) which may have + * credentials for sync itself stored in the login manager. + * + * In general, these hosts will not have their passwords synced, will be + * reset when we drop sync credentials, etc. + */ + getSyncCredentialsHosts() { + let result = new Set(); + // the FxA host + result.add(FxAccountsCommon.FXA_PWDMGR_HOST); + // We used to include the FxA hosts (hence the Set() result) but we now + // don't give them special treatment (hence the Set() with exactly 1 item) + return result; + }, + + /** + * Helper to implement a more efficient version of fairly common pattern: + * + * Utils.defineLazyIDProperty(this, "syncID", "services.sync.client.syncID") + * + * is equivalent to (but more efficient than) the following: + * + * Foo.prototype = { + * ... + * get syncID() { + * let syncID = Svc.PrefBranch.getStringPref("client.syncID", ""); + * return syncID == "" ? this.syncID = Utils.makeGUID() : syncID; + * }, + * set syncID(value) { + * Svc.PrefBranch.setStringPref("client.syncID", value); + * }, + * ... + * }; + */ + defineLazyIDProperty(object, propName, prefName) { + // An object that exists to be the target of the lazy pref getter. + // We can't use `object` (at least, not using `propName`) since XPCOMUtils + // will stomp on any setter we define. + const storage = {}; + XPCOMUtils.defineLazyPreferenceGetter(storage, "value", prefName, ""); + Object.defineProperty(object, propName, { + configurable: true, + enumerable: true, + get() { + let value = storage.value; + if (!value) { + value = Utils.makeGUID(); + Services.prefs.setStringPref(prefName, value); + } + return value; + }, + set(value) { + Services.prefs.setStringPref(prefName, value); + }, + }); + }, + + getDeviceType() { + return lazy.localDeviceType; + }, + + formatTimestamp(date) { + // Format timestamp as: "%Y-%m-%d %H:%M:%S" + let year = String(date.getFullYear()); + let month = String(date.getMonth() + 1).padStart(2, "0"); + let day = String(date.getDate()).padStart(2, "0"); + let hours = String(date.getHours()).padStart(2, "0"); + let minutes = String(date.getMinutes()).padStart(2, "0"); + let seconds = String(date.getSeconds()).padStart(2, "0"); + + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; + }, + + *walkTree(tree, parent = null) { + if (tree) { + // Skip root node + if (parent) { + yield [tree, parent]; + } + if (tree.children) { + for (let child of tree.children) { + yield* Utils.walkTree(child, tree); + } + } + } + }, +}; + +/** + * A subclass of Set that serializes as an Array when passed to JSON.stringify. + */ +export class SerializableSet extends Set { + toJSON() { + return Array.from(this); + } +} + +ChromeUtils.defineLazyGetter(Utils, "_utf8Converter", function () { + let converter = Cc[ + "@mozilla.org/intl/scriptableunicodeconverter" + ].createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + return converter; +}); + +ChromeUtils.defineLazyGetter(Utils, "utf8Encoder", () => new TextEncoder()); + +/* + * Commonly-used services + */ +export var Svc = {}; + +Svc.PrefBranch = Services.prefs.getBranch(PREFS_BRANCH); +Svc.Obs = Observers; + +Svc.Obs.add("xpcom-shutdown", function () { + for (let name in Svc) { + delete Svc[name]; + } +}); diff --git a/services/sync/moz.build b/services/sync/moz.build new file mode 100644 index 0000000000..a00472056e --- /dev/null +++ b/services/sync/moz.build @@ -0,0 +1,72 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Sync") + +XPCSHELL_TESTS_MANIFESTS += ["tests/unit/xpcshell.toml"] + +EXTRA_COMPONENTS += [ + "SyncComponents.manifest", +] + +EXTRA_JS_MODULES["services-sync"] += [ + "modules/addonsreconciler.sys.mjs", + "modules/addonutils.sys.mjs", + "modules/bridged_engine.sys.mjs", + "modules/collection_validator.sys.mjs", + "modules/constants.sys.mjs", + "modules/doctor.sys.mjs", + "modules/engines.sys.mjs", + "modules/keys.sys.mjs", + "modules/main.sys.mjs", + "modules/policies.sys.mjs", + "modules/record.sys.mjs", + "modules/resource.sys.mjs", + "modules/service.sys.mjs", + "modules/status.sys.mjs", + "modules/sync_auth.sys.mjs", + "modules/SyncDisconnect.sys.mjs", + "modules/SyncedTabs.sys.mjs", + "modules/telemetry.sys.mjs", + "modules/UIState.sys.mjs", + "modules/util.sys.mjs", + "Weave.sys.mjs", +] + +EXTRA_JS_MODULES["services-sync"].engines += [ + "modules/engines/addons.sys.mjs", + "modules/engines/clients.sys.mjs", + "modules/engines/extension-storage.sys.mjs", + "modules/engines/passwords.sys.mjs", + "modules/engines/prefs.sys.mjs", +] + +if not CONFIG["MOZ_THUNDERBIRD"]: + EXTRA_JS_MODULES["services-sync"].engines += [ + "modules/engines/bookmarks.sys.mjs", + "modules/engines/forms.sys.mjs", + "modules/engines/history.sys.mjs", + "modules/engines/tabs.sys.mjs", + ] + +EXTRA_JS_MODULES["services-sync"].stages += [ + "modules/stages/declined.sys.mjs", + "modules/stages/enginesync.sys.mjs", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] + +TESTING_JS_MODULES.services.sync += [ + "modules-testing/fakeservices.sys.mjs", + "modules-testing/fxa_utils.sys.mjs", + "modules-testing/rotaryengine.sys.mjs", + "modules-testing/utils.sys.mjs", +] + +SPHINX_TREES["/services/sync"] = "docs" diff --git a/services/sync/tests/tps/.eslintrc.js b/services/sync/tests/tps/.eslintrc.js new file mode 100644 index 0000000000..182e87933b --- /dev/null +++ b/services/sync/tests/tps/.eslintrc.js @@ -0,0 +1,28 @@ +"use strict"; + +module.exports = { + globals: { + // Injected into tests via tps.jsm + Addons: false, + Addresses: false, + Bookmarks: false, + CreditCards: false, + EnableEngines: false, + EnsureTracking: false, + ExtStorage: false, + Formdata: false, + History: false, + Login: false, + Passwords: false, + Phase: false, + Prefs: false, + STATE_DISABLED: false, + STATE_ENABLED: false, + Sync: false, + SYNC_WIPE_CLIENT: false, + SYNC_WIPE_REMOTE: false, + Tabs: false, + Windows: false, + WipeServer: false, + }, +}; diff --git a/services/sync/tests/tps/addons/api/restartless-xpi@tests.mozilla.org.json b/services/sync/tests/tps/addons/api/restartless-xpi@tests.mozilla.org.json new file mode 100644 index 0000000000..8593bad089 --- /dev/null +++ b/services/sync/tests/tps/addons/api/restartless-xpi@tests.mozilla.org.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Restartless Test XPI", + "type": "extension", + "guid": "restartless-xpi@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:4567/addons/restartless.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/tps/addons/api/test-webext@quality.mozilla.org.json b/services/sync/tests/tps/addons/api/test-webext@quality.mozilla.org.json new file mode 100644 index 0000000000..298ecc2ead --- /dev/null +++ b/services/sync/tests/tps/addons/api/test-webext@quality.mozilla.org.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Tet Webext XPI", + "type": "extension", + "guid": "test-webext@quality.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 3412, + "url": "http://127.0.0.1:4567/addons/webextension.xpi" + } + ] + }, + "last_updated": "2018-04-17T18:24:42Z" + } + ] +} diff --git a/services/sync/tests/tps/addons/restartless.xpi b/services/sync/tests/tps/addons/restartless.xpi Binary files differnew file mode 100644 index 0000000000..973bc00cb5 --- /dev/null +++ b/services/sync/tests/tps/addons/restartless.xpi diff --git a/services/sync/tests/tps/addons/webextension.xpi b/services/sync/tests/tps/addons/webextension.xpi Binary files differnew file mode 100644 index 0000000000..0ed64f79ac --- /dev/null +++ b/services/sync/tests/tps/addons/webextension.xpi diff --git a/services/sync/tests/tps/all_tests.json b/services/sync/tests/tps/all_tests.json new file mode 100644 index 0000000000..ea92e04b8f --- /dev/null +++ b/services/sync/tests/tps/all_tests.json @@ -0,0 +1,34 @@ +{ + "tests": { + "test_bookmark_conflict.js": {}, + "test_sync.js": {}, + "test_prefs.js": {}, + "test_tabs.js": {}, + "test_passwords.js": {}, + "test_history.js": {}, + "test_formdata.js": {}, + "test_bug530717.js": {}, + "test_bug531489.js": {}, + "test_bug538298.js": {}, + "test_bug556509.js": {}, + "test_bug562515.js": {}, + "test_bug535326.js": {}, + "test_bug501528.js": {}, + "test_bug575423.js": {}, + "test_bug546807.js": {}, + "test_history_collision.js": {}, + "test_privbrw_passwords.js": {}, + "test_privbrw_tabs.js": {}, + "test_bookmarks_in_same_named_folder.js": {}, + "test_client_wipe.js": {}, + "test_special_tabs.js": {}, + "test_addon_restartless_xpi.js": { "disabled": "Bug 1498974" }, + "test_addon_webext_xpi.js": { "disabled": "Bug 1498974" }, + "test_addon_reconciling.js": { "disabled": "Bug 1498974" }, + "test_addon_wipe.js": { "disabled": "Bug 1498974" }, + "test_existing_bookmarks.js": {}, + "test_addresses.js": {}, + "test_creditcards.js": {}, + "test_extstorage.js": {} + } +} diff --git a/services/sync/tests/tps/test_addon_reconciling.js b/services/sync/tests/tps/test_addon_reconciling.js new file mode 100644 index 0000000000..9c928734c8 --- /dev/null +++ b/services/sync/tests/tps/test_addon_reconciling.js @@ -0,0 +1,45 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test verifies that record reconciling works as expected. It makes +// similar changes to add-ons in separate profiles and does a sync to verify +// the proper action is taken. +EnableEngines(["addons"]); + +var phases = { + phase01: "profile1", + phase02: "profile2", + phase03: "profile1", + phase04: "profile2", + phase05: "profile1", + phase06: "profile2", +}; + +const id = "restartless-xpi@tests.mozilla.org"; + +// Install the add-on in 2 profiles. +Phase("phase01", [ + [Addons.verifyNot, [id]], + [Addons.install, [id]], + [Addons.verify, [id], STATE_ENABLED], + [Sync], +]); +Phase("phase02", [ + [Addons.verifyNot, [id]], + [Sync], + [Addons.verify, [id], STATE_ENABLED], +]); + +// Now we disable in one and uninstall in the other. +Phase("phase03", [ + [Sync], // Get GUID updates, potentially. + [Addons.setEnabled, [id], STATE_DISABLED], + // We've changed the state, but don't want this profile to sync until phase5, + // so if we ran a validation now we'd be expecting to find errors. + [Addons.skipValidation], +]); +Phase("phase04", [[EnsureTracking], [Addons.uninstall, [id]], [Sync]]); + +// When we sync, the uninstall should take precedence because it was newer. +Phase("phase05", [[Sync]]); +Phase("phase06", [[Sync], [Addons.verifyNot, [id]]]); diff --git a/services/sync/tests/tps/test_addon_restartless_xpi.js b/services/sync/tests/tps/test_addon_restartless_xpi.js new file mode 100644 index 0000000000..2b14245a6e --- /dev/null +++ b/services/sync/tests/tps/test_addon_restartless_xpi.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test verifies that install of restartless extensions syncs to +// other profiles. +EnableEngines(["addons"]); + +var phases = { + phase01: "profile1", + phase02: "profile2", + phase03: "profile1", + phase04: "profile2", + phase05: "profile1", + phase06: "profile2", + phase07: "profile1", + phase08: "profile2", +}; + +const id = "restartless-xpi@tests.mozilla.org"; + +// Verify install is synced +Phase("phase01", [ + [Addons.verifyNot, [id]], + [Addons.install, [id]], + [Addons.verify, [id], STATE_ENABLED], + [Sync], +]); +Phase("phase02", [ + [Addons.verifyNot, [id]], + [Sync], + [Addons.verify, [id], STATE_ENABLED], +]); + +// Now disable and see that is is synced. +Phase("phase03", [ + [EnsureTracking], + [Addons.setEnabled, [id], STATE_DISABLED], + [Addons.verify, [id], STATE_DISABLED], + [Sync], +]); +Phase("phase04", [[Sync], [Addons.verify, [id], STATE_DISABLED]]); + +// Enable and see it is synced. +Phase("phase05", [ + [EnsureTracking], + [Addons.setEnabled, [id], STATE_ENABLED], + [Addons.verify, [id], STATE_ENABLED], + [Sync], +]); +Phase("phase06", [[Sync], [Addons.verify, [id], STATE_ENABLED]]); + +// Uninstall and see it is synced. +Phase("phase07", [ + [EnsureTracking], + [Addons.verify, [id], STATE_ENABLED], + [Addons.uninstall, [id]], + [Addons.verifyNot, [id]], + [Sync], +]); +Phase("phase08", [ + [Addons.verify, [id], STATE_ENABLED], + [Sync], + [Addons.verifyNot, [id]], +]); diff --git a/services/sync/tests/tps/test_addon_webext_xpi.js b/services/sync/tests/tps/test_addon_webext_xpi.js new file mode 100644 index 0000000000..27065ee6af --- /dev/null +++ b/services/sync/tests/tps/test_addon_webext_xpi.js @@ -0,0 +1,65 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test verifies that install of web extensions sync to other profiles. +// It's more or less copied from test_addon_restartless_xpi with a different id. + +EnableEngines(["addons"]); + +var phases = { + phase01: "profile1", + phase02: "profile2", + phase03: "profile1", + phase04: "profile2", + phase05: "profile1", + phase06: "profile2", + phase07: "profile1", + phase08: "profile2", +}; + +const id = "test-webext@quality.mozilla.org"; + +// Verify install is synced +Phase("phase01", [ + [Addons.verifyNot, [id]], + [Addons.install, [id]], + [Addons.verify, [id], STATE_ENABLED], + [Sync], +]); +Phase("phase02", [ + [Addons.verifyNot, [id]], + [Sync], + [Addons.verify, [id], STATE_ENABLED], +]); + +// Now disable and see that is is synced. +Phase("phase03", [ + [EnsureTracking], + [Addons.setEnabled, [id], STATE_DISABLED], + [Addons.verify, [id], STATE_DISABLED], + [Sync], +]); +Phase("phase04", [[Sync], [Addons.verify, [id], STATE_DISABLED]]); + +// Enable and see it is synced. +Phase("phase05", [ + [EnsureTracking], + [Addons.setEnabled, [id], STATE_ENABLED], + [Addons.verify, [id], STATE_ENABLED], + [Sync], +]); +Phase("phase06", [[Sync], [Addons.verify, [id], STATE_ENABLED]]); + +// Uninstall and see it is synced. +Phase("phase07", [ + [EnsureTracking], + [Addons.verify, [id], STATE_ENABLED], + [Addons.uninstall, [id]], + [Addons.verifyNot, [id]], + [Sync], +]); +Phase("phase08", [ + [Addons.verify, [id], STATE_ENABLED], + [Sync], + [Addons.verifyNot, [id]], +]); diff --git a/services/sync/tests/tps/test_addon_wipe.js b/services/sync/tests/tps/test_addon_wipe.js new file mode 100644 index 0000000000..038f01d014 --- /dev/null +++ b/services/sync/tests/tps/test_addon_wipe.js @@ -0,0 +1,31 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test ensures that a client wipe followed by an "initial" sync will +// restore add-ons. This test should expose flaws in the reconciling logic, +// specifically around AddonsReconciler. This test is in response to bug +// 792990. + +EnableEngines(["addons"]); + +var phases = { + phase01: "profile1", + phase02: "profile1", + phase03: "profile1", +}; + +const id1 = "restartless-xpi@tests.mozilla.org"; +const id2 = "test-webext@quality.mozilla.org"; + +Phase("phase01", [[Addons.install, [id1]], [Addons.install, [id2]], [Sync]]); +Phase("phase02", [ + [Addons.verify, [id1], STATE_ENABLED], + [Addons.verify, [id2], STATE_ENABLED], + [Sync, SYNC_WIPE_CLIENT], + [Sync], +]); +Phase("phase03", [ + [Addons.verify, [id1], STATE_ENABLED], + [Addons.verify, [id2], STATE_ENABLED], + [Sync], // Sync to ensure that the addon validator can run without error +]); diff --git a/services/sync/tests/tps/test_addresses.js b/services/sync/tests/tps/test_addresses.js new file mode 100644 index 0000000000..33af349dd5 --- /dev/null +++ b/services/sync/tests/tps/test_addresses.js @@ -0,0 +1,84 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* global Services */ +Services.prefs.setBoolPref("services.sync.engine.addresses", true); + +EnableEngines(["addresses"]); + +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", +}; + +const address1 = [ + { + "given-name": "Timothy", + "additional-name": "John", + "family-name": "Berners-Lee", + organization: "World Wide Web Consortium", + "street-address": "32 Vassar Street\nMIT Room 32-G524", + "address-level2": "Cambridge", + "address-level1": "MA", + "postal-code": "02139", + country: "US", + tel: "+16172535702", + email: "timbl@w3.org", + changes: { + organization: "W3C", + }, + "unknown-1": "an unknown field from another client", + }, +]; + +const address1_after = [ + { + "given-name": "Timothy", + "additional-name": "John", + "family-name": "Berners-Lee", + organization: "W3C", + "street-address": "32 Vassar Street\nMIT Room 32-G524", + "address-level2": "Cambridge", + "address-level1": "MA", + "postal-code": "02139", + country: "US", + tel: "+16172535702", + email: "timbl@w3.org", + "unknown-1": "an unknown field from another client", + }, +]; + +const address2 = [ + { + "given-name": "John", + "additional-name": "R.", + "family-name": "Smith", + organization: "Mozilla", + "street-address": + "Geb\u00E4ude 3, 4. Obergeschoss\nSchlesische Stra\u00DFe 27", + "address-level2": "Berlin", + "address-level1": "BE", + "postal-code": "10997", + country: "DE", + tel: "+4930983333000", + email: "timbl@w3.org", + }, +]; + +Phase("phase1", [[Addresses.add, address1], [Sync]]); + +Phase("phase2", [ + [Sync], + [Addresses.verify, address1], + [Addresses.modify, address1], + [Addresses.add, address2], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [Addresses.verify, address1_after], + [Addresses.verify, address2], + [Sync], +]); diff --git a/services/sync/tests/tps/test_bookmark_conflict.js b/services/sync/tests/tps/test_bookmark_conflict.js new file mode 100644 index 0000000000..2832bded5d --- /dev/null +++ b/services/sync/tests/tps/test_bookmark_conflict.js @@ -0,0 +1,138 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +// the initial list of bookmarks to add to the browser +var bookmarksInitial = { + menu: [ + { folder: "foldera" }, + { folder: "folderb" }, + { folder: "folderc" }, + { folder: "folderd" }, + ], + + "menu/foldera": [{ uri: "http://www.cnn.com", title: "CNN" }], + "menu/folderb": [{ uri: "http://www.apple.com", title: "Apple", tags: [] }], + "menu/folderc": [{ uri: "http://www.yahoo.com", title: "Yahoo" }], + + "menu/folderd": [], +}; + +// a list of bookmarks to delete during a 'delete' action on P2 +var bookmarksToDelete = { + menu: [{ folder: "foldera" }, { folder: "folderb" }], + "menu/folderc": [{ uri: "http://www.yahoo.com", title: "Yahoo" }], +}; + +// the modifications to make on P1, after P2 has synced, but before P1 has gotten +// P2's changes +var bookmarkMods = { + menu: [ + { folder: "foldera" }, + { folder: "folderb" }, + { folder: "folderc" }, + { folder: "folderd" }, + ], + + // we move this child out of its folder (p1), after deleting the folder (p2) + // and expect the child to come back to p2 after sync. + "menu/foldera": [ + { + uri: "http://www.cnn.com", + title: "CNN", + changes: { location: "menu/folderd" }, + }, + ], + + // we rename this child (p1) after deleting the folder (p2), and expect the child + // to be moved into great grandparent (menu) + "menu/folderb": [ + { + uri: "http://www.apple.com", + title: "Apple", + tags: [], + changes: { title: "Mac" }, + }, + ], + + // we move this child (p1) after deleting the child (p2) and expect it to survive + "menu/folderc": [ + { + uri: "http://www.yahoo.com", + title: "Yahoo", + changes: { location: "menu/folderd" }, + }, + ], + + "menu/folderd": [], +}; + +// a list of bookmarks to delete during a 'delete' action +bookmarksToDelete = { + menu: [{ folder: "foldera" }, { folder: "folderb" }], + "menu/folderc": [{ uri: "http://www.yahoo.com", title: "Yahoo" }], +}; + +// expected bookmark state after conflict resolution +var bookmarksExpected = { + menu: [ + { folder: "folderc" }, + { folder: "folderd" }, + { uri: "http://www.apple.com", title: "Mac" }, + ], + + "menu/folderc": [], + + "menu/folderd": [ + { uri: "http://www.cnn.com", title: "CNN" }, + { uri: "http://www.yahoo.com", title: "Yahoo" }, + ], +}; + +// Add bookmarks to profile1 and sync. +Phase("phase1", [ + [Bookmarks.add, bookmarksInitial], + [Bookmarks.verify, bookmarksInitial], + [Sync], + [Bookmarks.verify, bookmarksInitial], +]); + +// Sync to profile2 and verify that the bookmarks are present. Delete +// bookmarks/folders, verify that it's not present, and sync +Phase("phase2", [ + [Sync], + [Bookmarks.verify, bookmarksInitial], + [Bookmarks.delete, bookmarksToDelete], + [Bookmarks.verifyNot, bookmarksToDelete], + [Sync], +]); + +// Using profile1, modify the bookmarks, and sync *after* the modification, +// and then sync again to propagate the reconciliation changes. +Phase("phase3", [ + [Bookmarks.verify, bookmarksInitial], + [Bookmarks.modify, bookmarkMods], + [Sync], + [Bookmarks.verify, bookmarksExpected], + [Bookmarks.verifyNot, bookmarksToDelete], +]); + +// Back in profile2, do a sync and verify that we're in the expected state +Phase("phase4", [ + [Sync], + [Bookmarks.verify, bookmarksExpected], + [Bookmarks.verifyNot, bookmarksToDelete], +]); diff --git a/services/sync/tests/tps/test_bookmarks_in_same_named_folder.js b/services/sync/tests/tps/test_bookmarks_in_same_named_folder.js new file mode 100644 index 0000000000..69dd8ba8a6 --- /dev/null +++ b/services/sync/tests/tps/test_bookmarks_in_same_named_folder.js @@ -0,0 +1,53 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// bug 558077 + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +var bookmarks_initial_1 = { + menu: [ + { folder: "aaa", description: "foo" }, + { uri: "http://www.mozilla.com" }, + ], + "menu/aaa": [ + { uri: "http://www.yahoo.com", title: "testing Yahoo" }, + { uri: "http://www.google.com", title: "testing Google" }, + ], +}; + +var bookmarks_initial_2 = { + menu: [ + { folder: "aaa", description: "bar" }, + { uri: "http://www.mozilla.com" }, + ], + "menu/aaa": [ + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + }, + { uri: "http://www.apple.com", tags: ["apple"] }, + ], +}; + +Phase("phase1", [[Bookmarks.add, bookmarks_initial_1], [Sync]]); + +Phase("phase2", [ + [Sync], + [Bookmarks.verify, bookmarks_initial_1], + [Bookmarks.add, bookmarks_initial_2], + [Sync], +]); + +Phase("phase3", [ + [Sync], + // XXX [Bookmarks.verify, bookmarks_initial_1], + [Bookmarks.verify, bookmarks_initial_2], +]); diff --git a/services/sync/tests/tps/test_bug501528.js b/services/sync/tests/tps/test_bug501528.js new file mode 100644 index 0000000000..f86ba02403 --- /dev/null +++ b/services/sync/tests/tps/test_bug501528.js @@ -0,0 +1,75 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["passwords"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Password lists + */ + +var passwords_initial = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "secret", + usernameField: "uname", + passwordField: "pword", + changes: { + password: "SeCrEt$$$", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +var passwords_after_first_update = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "SeCrEt$$$", + usernameField: "uname", + passwordField: "pword", + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Passwords.add, passwords_initial], [Sync]]); + +Phase("phase2", [[Passwords.add, passwords_initial], [Sync]]); + +Phase("phase3", [ + [Sync], + [Passwords.verify, passwords_initial], + [Passwords.modify, passwords_initial], + [Passwords.verify, passwords_after_first_update], + [Sync], +]); + +Phase("phase4", [[Sync], [Passwords.verify, passwords_after_first_update]]); diff --git a/services/sync/tests/tps/test_bug530717.js b/services/sync/tests/tps/test_bug530717.js new file mode 100644 index 0000000000..37e6711cb5 --- /dev/null +++ b/services/sync/tests/tps/test_bug530717.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["prefs"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +/* + * Preference lists + */ + +var prefs1 = [ + { name: "browser.startup.homepage", value: "http://www.getfirefox.com" }, + { name: "browser.urlbar.maxRichResults", value: 20 }, + { name: "privacy.clearOnShutdown.siteSettings", value: true }, +]; + +var prefs2 = [ + { name: "browser.startup.homepage", value: "http://www.mozilla.com" }, + { name: "browser.urlbar.maxRichResults", value: 18 }, + { name: "privacy.clearOnShutdown.siteSettings", value: false }, +]; + +/* + * Test phases + */ + +// Add prefs to profile1 and sync. +Phase("phase1", [[Prefs.modify, prefs1], [Prefs.verify, prefs1], [Sync]]); + +// Sync profile2 and verify same prefs are present. +Phase("phase2", [[Sync], [Prefs.verify, prefs1]]); + +// Using profile1, change some prefs, then do another sync with wipe-client. +// Verify that the cloud's prefs are restored, and the recent local changes +// discarded. +Phase("phase3", [ + [Prefs.modify, prefs2], + [Prefs.verify, prefs2], + [Sync, SYNC_WIPE_CLIENT], + [Prefs.verify, prefs1], +]); diff --git a/services/sync/tests/tps/test_bug531489.js b/services/sync/tests/tps/test_bug531489.js new file mode 100644 index 0000000000..3cc79c87ec --- /dev/null +++ b/services/sync/tests/tps/test_bug531489.js @@ -0,0 +1,43 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +/* + * Bookmark asset lists: these define bookmarks that are used during the test + */ + +// the initial list of bookmarks to add to the browser +var bookmarks_initial = { + menu: [ + { folder: "foldera" }, + { uri: "http://www.google.com", title: "Google" }, + ], + "menu/foldera": [{ uri: "http://www.google.com", title: "Google" }], + toolbar: [{ uri: "http://www.google.com", title: "Google" }], +}; + +/* + * Test phases + */ + +// Add three bookmarks with the same url to different locations and sync. +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +// Sync to profile2 and verify that all three bookmarks are present +Phase("phase2", [[Sync], [Bookmarks.verify, bookmarks_initial]]); + +// Sync again to profile1 and verify that all three bookmarks are still +// present. +Phase("phase3", [[Sync], [Bookmarks.verify, bookmarks_initial]]); diff --git a/services/sync/tests/tps/test_bug535326.js b/services/sync/tests/tps/test_bug535326.js new file mode 100644 index 0000000000..ede8f45c10 --- /dev/null +++ b/services/sync/tests/tps/test_bug535326.js @@ -0,0 +1,148 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["tabs"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2" }; + +var tabs1 = [ + { + uri: "data:text/html,<html><head><title>Howdy</title></head><body>Howdy</body></html>", + title: "Howdy", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>America</title></head><body>America</body></html>", + title: "America", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>", + title: "Apple", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>This</title></head><body>This</body></html>", + title: "This", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Bug</title></head><body>Bug</body></html>", + title: "Bug", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>IRC</title></head><body>IRC</body></html>", + title: "IRC", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Tinderbox</title></head><body>Tinderbox</body></html>", + title: "Tinderbox", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Fox</title></head><body>Fox</body></html>", + title: "Fox", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Hello</title></head><body>Hello</body></html>", + title: "Hello", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Eagle</title></head><body>Eagle</body></html>", + title: "Eagle", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Train</title></head><body>Train</body></html>", + title: "Train", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Macbook</title></head><body>Macbook</body></html>", + title: "Macbook", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Clock</title></head><body>Clock</body></html>", + title: "Clock", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Google</title></head><body>Google</body></html>", + title: "Google", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Human</title></head><body>Human</body></html>", + title: "Human", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Jetpack</title></head><body>Jetpack</body></html>", + title: "Jetpack", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Selenium</title></head><body>Selenium</body></html>", + title: "Selenium", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>", + title: "Mozilla", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>", + title: "Firefox", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Weave</title></head><body>Weave</body></html>", + title: "Weave", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Android</title></head><body>Android</body></html>", + title: "Android", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Bye</title></head><body>Bye</body></html>", + title: "Bye", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Hi</title></head><body>Hi</body></html>", + title: "Hi", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Final</title></head><body>Final</body></html>", + title: "Final", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Fennec</title></head><body>Fennec</body></html>", + title: "Fennec", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Mobile</title></head><body>Mobile</body></html>", + title: "Mobile", + profile: "profile1", + }, +]; + +Phase("phase1", [[Tabs.add, tabs1], [Sync]]); + +Phase("phase2", [[Sync], [Tabs.verify, tabs1]]); diff --git a/services/sync/tests/tps/test_bug538298.js b/services/sync/tests/tps/test_bug538298.js new file mode 100644 index 0000000000..3f1abf2a96 --- /dev/null +++ b/services/sync/tests/tps/test_bug538298.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Bookmark asset lists: these define bookmarks that are used during the test + */ + +// the initial list of bookmarks to add to the browser +var bookmarks_initial = { + toolbar: [ + { uri: "http://www.google.com", title: "Google" }, + { + uri: "http://www.cnn.com", + title: "CNN", + changes: { + position: "Google", + }, + }, + { uri: "http://www.mozilla.com", title: "Mozilla" }, + { + uri: "http://www.firefox.com", + title: "Firefox", + changes: { + position: "Mozilla", + }, + }, + ], +}; + +var bookmarks_after_move = { + toolbar: [ + { uri: "http://www.cnn.com", title: "CNN" }, + { uri: "http://www.google.com", title: "Google" }, + { uri: "http://www.firefox.com", title: "Firefox" }, + { uri: "http://www.mozilla.com", title: "Mozilla" }, + ], +}; + +/* + * Test phases + */ + +// Add four bookmarks to the toolbar and sync. +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +// Sync to profile2 and verify that all four bookmarks are present. +Phase("phase2", [[Sync], [Bookmarks.verify, bookmarks_initial]]); + +// Change the order of the toolbar bookmarks, and sync. +Phase("phase3", [ + [Sync], + [Bookmarks.verify, bookmarks_initial], + [Bookmarks.modify, bookmarks_initial], + [Bookmarks.verify, bookmarks_after_move], + [Sync], +]); + +// Go back to profile2, sync, and verify that the bookmarks are reordered +// as expected. +Phase("phase4", [[Sync], [Bookmarks.verify, bookmarks_after_move]]); diff --git a/services/sync/tests/tps/test_bug546807.js b/services/sync/tests/tps/test_bug546807.js new file mode 100644 index 0000000000..f02f632b41 --- /dev/null +++ b/services/sync/tests/tps/test_bug546807.js @@ -0,0 +1,38 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["tabs"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2" }; + +/* + * Tabs data + */ + +var tabs1 = [ + { uri: "about:config", profile: "profile1" }, + { uri: "about:credits", profile: "profile1" }, + { + uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>", + title: "Apple", + profile: "profile1", + }, +]; + +var tabs_absent = [ + { uri: "about:config", profile: "profile1" }, + { uri: "about:credits", profile: "profile1" }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Tabs.add, tabs1], [Sync]]); + +Phase("phase2", [[Sync], [Tabs.verifyNot, tabs_absent]]); diff --git a/services/sync/tests/tps/test_bug556509.js b/services/sync/tests/tps/test_bug556509.js new file mode 100644 index 0000000000..3a46c3c23e --- /dev/null +++ b/services/sync/tests/tps/test_bug556509.js @@ -0,0 +1,32 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2" }; + +// the initial list of bookmarks to add to the browser +var bookmarks_initial = { + menu: [{ folder: "testfolder", description: "it's just me, a test folder" }], + "menu/testfolder": [{ uri: "http://www.mozilla.com", title: "Mozilla" }], +}; + +/* + * Test phases + */ + +// Add a bookmark folder which has a description, and sync. +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +// Sync to profile2 and verify that the bookmark folder is created, along +// with its description. +Phase("phase2", [[Sync], [Bookmarks.verify, bookmarks_initial]]); diff --git a/services/sync/tests/tps/test_bug562515.js b/services/sync/tests/tps/test_bug562515.js new file mode 100644 index 0000000000..3a1a6cbee2 --- /dev/null +++ b/services/sync/tests/tps/test_bug562515.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Bookmark lists + */ + +// the initial list of bookmarks to add to the browser +var bookmarks_initial = { + menu: [ + { + uri: "http://www.google.com", + tags: ["google", "computers", "internet", "www"], + }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + keyword: "bz", + }, + { folder: "foldera" }, + { uri: "http://www.mozilla.com" }, + { separator: true }, + { folder: "folderb" }, + ], + "menu/foldera": [ + { uri: "http://www.yahoo.com", title: "testing Yahoo" }, + { + uri: "http://www.cnn.com", + description: "This is a description of the site a at www.cnn.com", + }, + ], + "menu/folderb": [{ uri: "http://www.apple.com", tags: ["apple", "mac"] }], + toolbar: [ + { + uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0", + title: "Visited Today", + }, + ], +}; + +// a list of bookmarks to delete during a 'delete' action +var bookmarks_to_delete = { + menu: [ + { + uri: "http://www.google.com", + tags: ["google", "computers", "internet", "www"], + }, + ], + "menu/foldera": [{ uri: "http://www.yahoo.com", title: "testing Yahoo" }], +}; + +/* + * Test phases + */ + +// add bookmarks to profile1 and sync +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +// sync to profile2 and verify that the bookmarks are present +Phase("phase2", [[Sync], [Bookmarks.verify, bookmarks_initial]]); + +// delete some bookmarks from profile1, then sync with "wipe-client" +// set; finally, verify that the deleted bookmarks were restored. +Phase("phase3", [ + [Bookmarks.delete, bookmarks_to_delete], + [Bookmarks.verifyNot, bookmarks_to_delete], + [Sync, SYNC_WIPE_CLIENT], + [Bookmarks.verify, bookmarks_initial], +]); + +// sync profile2 again, verify no bookmarks have been deleted +Phase("phase4", [[Sync], [Bookmarks.verify, bookmarks_initial]]); diff --git a/services/sync/tests/tps/test_bug575423.js b/services/sync/tests/tps/test_bug575423.js new file mode 100644 index 0000000000..53f46db5b7 --- /dev/null +++ b/services/sync/tests/tps/test_bug575423.js @@ -0,0 +1,67 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["history"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2" }; + +/* + * History data + */ + +// the history data to add to the browser +var history1 = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -1 }, + ], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, +]; + +// Another history data to add to the browser +var history2 = [ + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.google.com/language_tools?hl=en", + title: "Language Tools", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -40 }, + ], + }, +]; + +/* + * Test phases + */ +Phase("phase1", [ + [History.add, history1], + [Sync], + [History.add, history2], + [Sync], +]); + +Phase("phase2", [[Sync], [History.verify, history2]]); diff --git a/services/sync/tests/tps/test_client_wipe.js b/services/sync/tests/tps/test_client_wipe.js new file mode 100644 index 0000000000..e0fb7a97ac --- /dev/null +++ b/services/sync/tests/tps/test_client_wipe.js @@ -0,0 +1,142 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +/* + * Bookmark lists + */ + +// the initial list of bookmarks to add to the browser +var bookmarks_initial = { + toolbar: [ + { uri: "http://www.google.com", title: "Google" }, + { + uri: "http://www.cnn.com", + title: "CNN", + changes: { + position: "Google", + }, + }, + { uri: "http://www.mozilla.com", title: "Mozilla" }, + { + uri: "http://www.firefox.com", + title: "Firefox", + changes: { + position: "Mozilla", + }, + }, + ], +}; + +var bookmarks_after_move = { + toolbar: [ + { uri: "http://www.cnn.com", title: "CNN" }, + { uri: "http://www.google.com", title: "Google" }, + { uri: "http://www.firefox.com", title: "Firefox" }, + { uri: "http://www.mozilla.com", title: "Mozilla" }, + ], +}; + +/* + * Password data + */ + +// Initial password data +var passwords_initial = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "secret", + usernameField: "uname", + passwordField: "pword", + changes: { + password: "SeCrEt$$$", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +// Password after first modify action has been performed +var passwords_after_change = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "SeCrEt$$$", + usernameField: "uname", + passwordField: "pword", + changes: { + username: "james", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +/* + * Prefs to use in the test + */ +var prefs1 = [ + { name: "browser.startup.homepage", value: "http://www.getfirefox.com" }, + { name: "browser.urlbar.maxRichResults", value: 20 }, + { name: "privacy.clearOnShutdown.siteSettings", value: true }, +]; + +var prefs2 = [ + { name: "browser.startup.homepage", value: "http://www.mozilla.com" }, + { name: "browser.urlbar.maxRichResults", value: 18 }, + { name: "privacy.clearOnShutdown.siteSettings", value: false }, +]; + +/* + * Test phases + */ + +// Add prefs,passwords and bookmarks to profile1 and sync. +Phase("phase1", [ + [Passwords.add, passwords_initial], + [Bookmarks.add, bookmarks_initial], + [Prefs.modify, prefs1], + [Prefs.verify, prefs1], + [Sync], +]); + +// Sync profile2 and verify same prefs,passwords and bookmarks are present. +Phase("phase2", [ + [Sync], + [Prefs.verify, prefs1], + [Passwords.verify, passwords_initial], + [Bookmarks.verify, bookmarks_initial], +]); + +// Using profile1, change some prefs,bookmarks and pwds, then do another sync with wipe-client. +// Verify that the cloud's settings are restored, and the recent local changes +// discarded. +Phase("phase3", [ + [Prefs.modify, prefs2], + [Passwords.modify, passwords_initial], + [Bookmarks.modify, bookmarks_initial], + [Prefs.verify, prefs2], + [Passwords.verify, passwords_after_change], + [Bookmarks.verify, bookmarks_after_move], + [Sync, SYNC_WIPE_CLIENT], + [Prefs.verify, prefs1], + [Passwords.verify, passwords_initial], + [Bookmarks.verify, bookmarks_initial], +]); diff --git a/services/sync/tests/tps/test_creditcards.js b/services/sync/tests/tps/test_creditcards.js new file mode 100644 index 0000000000..fea2b8a541 --- /dev/null +++ b/services/sync/tests/tps/test_creditcards.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* global Services */ +Services.prefs.setBoolPref("services.sync.engine.creditcards", true); + +EnableEngines(["creditcards"]); + +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", +}; + +const cc1 = [ + { + "cc-name": "John Doe", + "cc-number": "4716179744040592", + "cc-exp-month": 4, + "cc-exp-year": 2050, + "unknown-1": "an unknown field from another client", + changes: { + "cc-exp-year": 2051, + }, + }, +]; + +const cc1_after = [ + { + "cc-name": "John Doe", + "cc-number": "4716179744040592", + "cc-exp-month": 4, + "cc-exp-year": 2051, + "unknown-1": "an unknown field from another client", + }, +]; + +const cc2 = [ + { + "cc-name": "Timothy Berners-Lee", + "cc-number": "2221000374457678", + "cc-exp-month": 12, + "cc-exp-year": 2050, + }, +]; + +Phase("phase1", [[CreditCards.add, cc1], [Sync]]); + +Phase("phase2", [ + [Sync], + [CreditCards.verify, cc1], + [CreditCards.modify, cc1], + [CreditCards.add, cc2], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [CreditCards.verifyNot, cc1], + [CreditCards.verify, cc1_after], + [CreditCards.verify, cc2], +]); diff --git a/services/sync/tests/tps/test_existing_bookmarks.js b/services/sync/tests/tps/test_existing_bookmarks.js new file mode 100644 index 0000000000..cde41a96e6 --- /dev/null +++ b/services/sync/tests/tps/test_existing_bookmarks.js @@ -0,0 +1,80 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["bookmarks"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile2", + phase4: "profile1", +}; + +/* + * Bookmark lists + */ +var bookmarks_initial = { + menu: [ + { + uri: "http://www.google.com", + title: "Google", + changes: { + title: "google", + }, + }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + }, + { uri: "http://www.mozilla.com" }, + { + uri: "http://www.cnn.com", + description: "This is a description of the site a at www.cnn.com", + changes: { + description: "Global news", + }, + }, + ], +}; + +var bookmarks_after = { + menu: [ + { uri: "http://www.google.com", title: "google" }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + }, + { uri: "http://www.mozilla.com" }, + { uri: "http://www.cnn.com", description: "Global news" }, + ], +}; + +/* + * Test phases + */ + +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +Phase("phase2", [ + [Bookmarks.add, bookmarks_initial], + [Bookmarks.verify, bookmarks_initial], + [Sync], +]); + +Phase("phase3", [ + [Bookmarks.verify, bookmarks_initial], + [Bookmarks.modify, bookmarks_initial], + [Bookmarks.verify, bookmarks_after], + [Sync], +]); + +Phase("phase4", [[Sync], [Bookmarks.verify, bookmarks_after]]); diff --git a/services/sync/tests/tps/test_extstorage.js b/services/sync/tests/tps/test_extstorage.js new file mode 100644 index 0000000000..cf25187bc3 --- /dev/null +++ b/services/sync/tests/tps/test_extstorage.js @@ -0,0 +1,154 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["addons"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", + phase5: "profile1", +}; + +/* + * Test phases + */ + +Phase("phase1", [ + [ + ExtStorage.set, + "ext-1", + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": { + sk_1: "v1", + sk_2: "v2", + }, + }, + ], + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": { + sk_1: "v1", + sk_2: "v2", + }, + }, + ], + [Sync], +]); + +Phase("phase2", [ + [Sync], + [ + ExtStorage.set, + "ext-1", + { + "key-2": "value from profile 2", + }, + ], + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": "value from profile 2", + }, + ], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": "value from profile 2", + }, + ], + [ + ExtStorage.set, + "ext-1", + { + "key-2": "value from profile 1", + }, + ], + // exit without syncing. +]); + +Phase("phase4", [ + [Sync], + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": "value from profile 2", + }, + ], + [ + ExtStorage.set, + "ext-1", + { + "key-2": "second value from profile 2", + }, + ], + [Sync], +]); + +Phase("phase5", [ + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": "value from profile 1", + }, + ], + [Sync], + [ + ExtStorage.verify, + "ext-1", + null, + { + "key-1": { + sub_key_1: "value 1", + sub_key_2: "value 2", + }, + "key-2": "second value from profile 2", + }, + ], +]); diff --git a/services/sync/tests/tps/test_formdata.js b/services/sync/tests/tps/test_formdata.js new file mode 100644 index 0000000000..764b12342f --- /dev/null +++ b/services/sync/tests/tps/test_formdata.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["forms"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Form data asset lists: these define form values that are used in the tests. + */ + +var formdata1 = [ + { fieldname: "testing", value: "success", date: -1 }, + { fieldname: "testing", value: "failure", date: -2 }, + { fieldname: "username", value: "joe" }, +]; + +var formdata2 = [ + { fieldname: "testing", value: "success", date: -1 }, + { fieldname: "username", value: "joe" }, +]; + +var formdata_delete = [{ fieldname: "testing", value: "failure" }]; + +var formdata_new = [{ fieldname: "new-field", value: "new-value" }]; +/* + * Test phases + */ + +Phase("phase1", [ + [Formdata.add, formdata1], + [Formdata.verify, formdata1], + [Sync], +]); + +Phase("phase2", [[Sync], [Formdata.verify, formdata1]]); + +Phase("phase3", [ + [Sync], + [Formdata.delete, formdata_delete], + [Formdata.verifyNot, formdata_delete], + [Formdata.verify, formdata2], + // add new data after the first Sync, ensuring the tracker works. + [Formdata.add, formdata_new], + [Sync], +]); + +Phase("phase4", [ + [Sync], + [Formdata.verify, formdata2], + [Formdata.verify, formdata_new], + [Formdata.verifyNot, formdata_delete], +]); diff --git a/services/sync/tests/tps/test_history.js b/services/sync/tests/tps/test_history.js new file mode 100644 index 0000000000..e1df78c762 --- /dev/null +++ b/services/sync/tests/tps/test_history.js @@ -0,0 +1,129 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["history"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2" }; + +/* + * History asset lists: these define history entries that are used during + * the test + */ + +// the initial list of history items to add to the browser +var history1 = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -1 }, + ], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.google.com/language_tools?hl=en", + title: "Language Tools", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -40 }, + ], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 1, date: -1 }, + { type: 1, date: -20 }, + { type: 2, date: -36 }, + ], + }, +]; + +// a list of items to delete from the history +var history_to_delete = [ + { uri: "http://www.cnn.com/" }, + { begin: -24, end: -1 }, + { host: "www.google.com" }, +]; + +// a list which reflects items that should be in the history after +// the above items are deleted +var history2 = [ + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -36 }, + ], + }, +]; + +// a list which includes history entries that should not be present +// after deletion of the history_to_delete entries +var history_not = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -1 }, + ], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.google.com/language_tools?hl=en", + title: "Language Tools", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -40 }, + ], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: -1 }, + { type: 1, date: -20 }, + ], + }, +]; + +/* + * Test phases + * Note: there is no test phase in which deleted history entries are + * synced to other clients. This functionality is not supported by + * Sync, see bug 446517. + */ + +Phase("phase1", [[History.add, history1], [Sync]]); + +Phase("phase2", [ + [Sync], + [History.verify, history1], + [History.delete, history_to_delete], + [History.verify, history2], + [History.verifyNot, history_not], + [Sync], +]); diff --git a/services/sync/tests/tps/test_history_collision.js b/services/sync/tests/tps/test_history_collision.js new file mode 100644 index 0000000000..ea428d1051 --- /dev/null +++ b/services/sync/tests/tps/test_history_collision.js @@ -0,0 +1,98 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["history"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * History lists + */ + +// the initial list of history to add to the browser +var history1 = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [{ type: 1, date: 0 }], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -36 }, + ], + }, +]; + +// the history to delete +var history_to_delete = [ + { uri: "http://www.cnn.com/", title: "CNN" }, + { begin: -36, end: -1 }, +]; + +var history_not = [ + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, +]; + +var history_after_delete = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [{ type: 1, date: 0 }], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [{ type: 1, date: 0 }], + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[History.add, history1], [Sync]]); + +Phase("phase2", [[Sync], [History.add, history1], [Sync, SYNC_WIPE_REMOTE]]); + +Phase("phase3", [ + [Sync], + [History.verify, history1], + [History.delete, history_to_delete], + [History.verify, history_after_delete], + [History.verifyNot, history_not], + [Sync], +]); + +Phase("phase4", [ + [Sync], + [History.verify, history_after_delete], + [History.verifyNot, history_not], +]); diff --git a/services/sync/tests/tps/test_passwords.js b/services/sync/tests/tps/test_passwords.js new file mode 100644 index 0000000000..2dfd7de8dd --- /dev/null +++ b/services/sync/tests/tps/test_passwords.js @@ -0,0 +1,119 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["passwords"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Password asset lists: these define password entries that are used during + * the test + */ + +// initial password list to be loaded into the browser +var passwords_initial = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "SeCrEt123", + usernameField: "uname", + passwordField: "pword", + changes: { + password: "zippity-do-dah", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// expected state of passwords after the changes in the above list are applied +var passwords_after_first_update = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "zippity-do-dah", + usernameField: "uname", + passwordField: "pword", + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +var passwords_to_delete = [ + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +var passwords_absent = [ + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// expected state of passwords after the delete operation +var passwords_after_second_update = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "zippity-do-dah", + usernameField: "uname", + passwordField: "pword", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Passwords.add, passwords_initial], [Sync]]); + +Phase("phase2", [ + [Sync], + [Passwords.verify, passwords_initial], + [Passwords.modify, passwords_initial], + [Passwords.verify, passwords_after_first_update], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [Passwords.verify, passwords_after_first_update], + [Passwords.delete, passwords_to_delete], + [Passwords.verify, passwords_after_second_update], + [Passwords.verifyNot, passwords_absent], + [Sync], +]); + +Phase("phase4", [ + [Sync], + [Passwords.verify, passwords_after_second_update], + [Passwords.verifyNot, passwords_absent], +]); diff --git a/services/sync/tests/tps/test_prefs.js b/services/sync/tests/tps/test_prefs.js new file mode 100644 index 0000000000..3e26e947dd --- /dev/null +++ b/services/sync/tests/tps/test_prefs.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["prefs"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +var prefs1 = [ + { name: "browser.startup.homepage", value: "http://www.getfirefox.com" }, + { name: "browser.urlbar.maxRichResults", value: 20 }, + { name: "privacy.clearOnShutdown.siteSettings", value: true }, +]; + +var prefs2 = [ + { name: "browser.startup.homepage", value: "http://www.mozilla.com" }, + { name: "browser.urlbar.maxRichResults", value: 18 }, + { name: "privacy.clearOnShutdown.siteSettings", value: false }, +]; + +Phase("phase1", [[Prefs.modify, prefs1], [Prefs.verify, prefs1], [Sync]]); + +Phase("phase2", [ + [Sync], + [Prefs.verify, prefs1], + [Prefs.modify, prefs2], + [Prefs.verify, prefs2], + [Sync], +]); + +Phase("phase3", [[Sync], [Prefs.verify, prefs2]]); diff --git a/services/sync/tests/tps/test_privbrw_passwords.js b/services/sync/tests/tps/test_privbrw_passwords.js new file mode 100644 index 0000000000..dcde6c02b8 --- /dev/null +++ b/services/sync/tests/tps/test_privbrw_passwords.js @@ -0,0 +1,105 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in strict JSON format, as it will get parsed by the Python + * testrunner (no single quotes, extra comma's, etc). + */ +EnableEngines(["passwords"]); + +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Password data + */ + +// Initial password data +var passwords_initial = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "secret", + usernameField: "uname", + passwordField: "pword", + changes: { + password: "SeCrEt$$$", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +// Password after first modify action has been performed +var passwords_after_first_change = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "SeCrEt$$$", + usernameField: "uname", + passwordField: "pword", + changes: { + username: "james", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +// Password after second modify action has been performed +var passwords_after_second_change = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "james", + password: "SeCrEt$$$", + usernameField: "uname", + passwordField: "pword", + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "jack", + password: "secretlogin", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Passwords.add, passwords_initial], [Sync]]); + +Phase("phase2", [ + [Sync], + [Passwords.verify, passwords_initial], + [Passwords.modify, passwords_initial], + [Passwords.verify, passwords_after_first_change], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [Windows.add, { private: true }], + [Passwords.verify, passwords_after_first_change], + [Passwords.modify, passwords_after_first_change], + [Passwords.verify, passwords_after_second_change], + [Sync], +]); + +Phase("phase4", [[Sync], [Passwords.verify, passwords_after_second_change]]); diff --git a/services/sync/tests/tps/test_privbrw_tabs.js b/services/sync/tests/tps/test_privbrw_tabs.js new file mode 100644 index 0000000000..c7c877677f --- /dev/null +++ b/services/sync/tests/tps/test_privbrw_tabs.js @@ -0,0 +1,86 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in strict JSON format, as it will get parsed by the Python + * testrunner (no single quotes, extra comma's, etc). + */ +EnableEngines(["tabs"]); + +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Tabs data + */ + +var tabs1 = [ + { + uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>", + title: "Firefox", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Weave</title></head><body>Weave</body></html>", + title: "Weave", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>", + title: "Apple", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>IRC</title></head><body>IRC</body></html>", + title: "IRC", + profile: "profile1", + }, +]; + +var tabs2 = [ + { + uri: "data:text/html,<html><head><title>Tinderbox</title></head><body>Tinderbox</body></html>", + title: "Tinderbox", + profile: "profile2", + }, + { + uri: "data:text/html,<html><head><title>Fox</title></head><body>Fox</body></html>", + title: "Fox", + profile: "profile2", + }, +]; + +var tabs3 = [ + { + uri: "data:text/html,<html><head><title>Jetpack</title></head><body>Jetpack</body></html>", + title: "Jetpack", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Selenium</title></head><body>Selenium</body></html>", + title: "Selenium", + profile: "profile1", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Tabs.add, tabs1], [Sync]]); + +Phase("phase2", [[Sync], [Tabs.verify, tabs1], [Tabs.add, tabs2], [Sync]]); + +Phase("phase3", [ + [Sync], + [Windows.add, { private: true }], + [Tabs.add, tabs3], + [Sync], +]); + +Phase("phase4", [[Sync], [Tabs.verifyNot, tabs3]]); diff --git a/services/sync/tests/tps/test_special_tabs.js b/services/sync/tests/tps/test_special_tabs.js new file mode 100644 index 0000000000..9e1eae88e3 --- /dev/null +++ b/services/sync/tests/tps/test_special_tabs.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Bug 532173 - Dont sync tabs like about:* , weave firstrun etc + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in strict JSON format, as it will get parsed by the Python + * testrunner (no single quotes, extra comma's, etc). + */ +EnableEngines(["tabs"]); + +var phases = { phase1: "profile1", phase2: "profile2" }; + +var tabs1 = [ + { + uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>", + title: "Firefox", + profile: "profile1", + }, + { uri: "about:robots", title: "About", profile: "profile1" }, + { uri: "about:credits", title: "Credits", profile: "profile1" }, + { + uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>", + title: "Mozilla", + profile: "profile1", + }, + { + uri: "http://www.mozilla.com/en-US/firefox/sync/firstrun.html", + title: "Firstrun", + profile: "profile1", + }, +]; + +var tabs2 = [ + { + uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>", + title: "Firefox", + profile: "profile1", + }, + { + uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>", + title: "Mozilla", + profile: "profile1", + }, +]; + +var tabs3 = [ + { + uri: "http://www.mozilla.com/en-US/firefox/sync/firstrun.html", + title: "Firstrun", + profile: "profile1", + }, + { uri: "about:robots", title: "About", profile: "profile1" }, + { uri: "about:credits", title: "Credits", profile: "profile1" }, +]; + +/* + * Test phases + */ +Phase("phase1", [[Tabs.add, tabs1], [Sync]]); + +Phase("phase2", [[Sync], [Tabs.verify, tabs2], [Tabs.verifyNot, tabs3]]); diff --git a/services/sync/tests/tps/test_sync.js b/services/sync/tests/tps/test_sync.js new file mode 100644 index 0000000000..3d14430c60 --- /dev/null +++ b/services/sync/tests/tps/test_sync.js @@ -0,0 +1,403 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in strict JSON format, as it will get parsed by the Python + * testrunner (no single quotes, extra comma's, etc). + */ + +var phases = { + phase1: "profile1", + phase2: "profile2", + phase3: "profile1", + phase4: "profile2", +}; + +/* + * Bookmark asset lists: these define bookmarks that are used during the test + */ + +// the initial list of bookmarks to be added to the browser +var bookmarks_initial = { + menu: [ + { + uri: "http://www.google.com", + tags: ["google", "computers", "internet", "www"], + changes: { + title: "Google", + tags: ["google", "computers", "misc"], + }, + }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + keyword: "bz", + changes: { + keyword: "bugzilla", + }, + }, + { folder: "foldera" }, + { uri: "http://www.mozilla.com" }, + { separator: true }, + { folder: "folderb" }, + ], + "menu/foldera": [ + { + uri: "http://www.yahoo.com", + title: "testing Yahoo", + changes: { + location: "menu/folderb", + }, + }, + { + uri: "http://www.cnn.com", + description: "This is a description of the site a at www.cnn.com", + changes: { + uri: "http://money.cnn.com", + description: "new description", + }, + }, + ], + "menu/folderb": [ + { + uri: "http://www.apple.com", + tags: ["apple", "mac"], + changes: { + uri: "http://www.apple.com/iphone/", + title: "iPhone", + location: "menu", + position: "Google", + tags: [], + }, + }, + ], + toolbar: [ + { + uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0", + title: "Visited Today", + }, + ], +}; + +// the state of bookmarks after the first 'modify' action has been performed +// on them +var bookmarks_after_first_modify = { + menu: [ + { + uri: "http://www.apple.com/iphone/", + title: "iPhone", + before: "Google", + tags: [], + }, + { + uri: "http://www.google.com", + title: "Google", + tags: ["google", "computers", "misc"], + }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + keyword: "bugzilla", + }, + { folder: "foldera" }, + { uri: "http://www.mozilla.com" }, + { separator: true }, + { + folder: "folderb", + changes: { + location: "menu/foldera", + folder: "Folder B", + description: "folder description", + }, + }, + ], + "menu/foldera": [ + { + uri: "http://money.cnn.com", + title: "http://www.cnn.com", + description: "new description", + }, + ], + "menu/folderb": [{ uri: "http://www.yahoo.com", title: "testing Yahoo" }], + toolbar: [ + { + uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0", + title: "Visited Today", + }, + ], +}; + +// a list of bookmarks to delete during a 'delete' action +var bookmarks_to_delete = { + menu: [ + { + uri: "http://www.google.com", + title: "Google", + tags: ["google", "computers", "misc"], + }, + ], +}; + +// the state of bookmarks after the second 'modify' action has been performed +// on them +var bookmarks_after_second_modify = { + menu: [ + { uri: "http://www.apple.com/iphone/", title: "iPhone" }, + { + uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s", + title: "Bugzilla", + keyword: "bugzilla", + }, + { folder: "foldera" }, + { uri: "http://www.mozilla.com" }, + { separator: true }, + ], + "menu/foldera": [ + { + uri: "http://money.cnn.com", + title: "http://www.cnn.com", + description: "new description", + }, + { folder: "Folder B", description: "folder description" }, + ], + "menu/foldera/Folder B": [ + { uri: "http://www.yahoo.com", title: "testing Yahoo" }, + ], +}; + +// a list of bookmarks which should not be present after the last +// 'delete' and 'modify' actions +var bookmarks_absent = { + menu: [ + { uri: "http://www.google.com", title: "Google" }, + { folder: "folderb" }, + { folder: "Folder B" }, + ], +}; + +/* + * History asset lists: these define history entries that are used during + * the test + */ + +// the initial list of history items to add to the browser +var history_initial = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -1 }, + ], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.google.com/language_tools?hl=en", + title: "Language Tools", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -40 }, + ], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 1, date: -1 }, + { type: 1, date: -20 }, + { type: 2, date: -36 }, + ], + }, +]; + +// a list of history entries to delete during a 'delete' action +var history_to_delete = [ + { uri: "http://www.cnn.com/" }, + { begin: -24, end: -1 }, + { host: "www.google.com" }, +]; + +// the expected history entries after the first 'delete' action +var history_after_delete = [ + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -36 }, + ], + }, +]; + +// history entries expected to not exist after a 'delete' action +var history_absent = [ + { + uri: "http://www.google.com/", + title: "Google", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -1 }, + ], + }, + { + uri: "http://www.cnn.com/", + title: "CNN", + visits: [ + { type: 1, date: -1 }, + { type: 2, date: -36 }, + ], + }, + { + uri: "http://www.google.com/language_tools?hl=en", + title: "Language Tools", + visits: [ + { type: 1, date: 0 }, + { type: 2, date: -40 }, + ], + }, + { + uri: "http://www.mozilla.com/", + title: "Mozilla", + visits: [ + { type: 1, date: -1 }, + { type: 1, date: -20 }, + ], + }, +]; + +/* + * Password asset lists: these define password entries that are used during + * the test + */ + +// the initial list of passwords to add to the browser +var passwords_initial = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "SeCrEt123", + usernameField: "uname", + passwordField: "pword", + changes: { + password: "zippity-do-dah", + }, + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// the expected state of passwords after the first 'modify' action +var passwords_after_first_modify = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "zippity-do-dah", + usernameField: "uname", + passwordField: "pword", + }, + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// a list of passwords to delete during a 'delete' action +var passwords_to_delete = [ + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// a list of passwords expected to be absent after 'delete' and 'modify' +// actions +var passwords_absent = [ + { + hostname: "http://www.example.com", + realm: "login", + username: "joe", + password: "secretlogin", + }, +]; + +// the expected state of passwords after the seconds 'modify' action +var passwords_after_second_modify = [ + { + hostname: "http://www.example.com", + submitURL: "http://login.example.com", + username: "joe", + password: "zippity-do-dah", + usernameField: "uname", + passwordField: "pword", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [ + [Bookmarks.add, bookmarks_initial], + [Passwords.add, passwords_initial], + [History.add, history_initial], + [Sync], +]); + +Phase("phase2", [ + [Sync], + [Bookmarks.verify, bookmarks_initial], + [Passwords.verify, passwords_initial], + [History.verify, history_initial], + [Bookmarks.modify, bookmarks_initial], + [Passwords.modify, passwords_initial], + [History.delete, history_to_delete], + [Bookmarks.verify, bookmarks_after_first_modify], + [Passwords.verify, passwords_after_first_modify], + [History.verify, history_after_delete], + [History.verifyNot, history_absent], + [Sync], +]); + +Phase("phase3", [ + [Sync], + [Bookmarks.verify, bookmarks_after_first_modify], + [Passwords.verify, passwords_after_first_modify], + [History.verify, history_after_delete], + [Bookmarks.modify, bookmarks_after_first_modify], + [Passwords.modify, passwords_after_first_modify], + [Bookmarks.delete, bookmarks_to_delete], + [Passwords.delete, passwords_to_delete], + [Bookmarks.verify, bookmarks_after_second_modify], + [Passwords.verify, passwords_after_second_modify], + [Bookmarks.verifyNot, bookmarks_absent], + [Passwords.verifyNot, passwords_absent], + [Sync], +]); + +Phase("phase4", [ + [Sync], + [Bookmarks.verify, bookmarks_after_second_modify], + [Passwords.verify, passwords_after_second_modify], + [Bookmarks.verifyNot, bookmarks_absent], + [Passwords.verifyNot, passwords_absent], + [History.verifyNot, history_absent], +]); diff --git a/services/sync/tests/tps/test_tabs.js b/services/sync/tests/tps/test_tabs.js new file mode 100644 index 0000000000..58e8d358cd --- /dev/null +++ b/services/sync/tests/tps/test_tabs.js @@ -0,0 +1,42 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +EnableEngines(["tabs"]); + +/* + * The list of phases mapped to their corresponding profiles. The object + * here must be in JSON format as it will get parsed by the Python + * testrunner. It is parsed by the YAML package, so it relatively flexible. + */ +var phases = { phase1: "profile1", phase2: "profile2", phase3: "profile1" }; + +/* + * Tab lists. + */ + +var tabs1 = [ + { uri: "https://www.mozilla.org/en-US/firefox/", profile: "profile1" }, + { + uri: "https://example.com/", + title: "Example Domain", + profile: "profile1", + }, +]; + +var tabs2 = [ + { uri: "https://www.mozilla.org/en-US/contribute/", profile: "profile2" }, + { + uri: "https://example.com/", + profile: "profile2", + }, +]; + +/* + * Test phases + */ + +Phase("phase1", [[Tabs.add, tabs1], [Sync]]); + +Phase("phase2", [[Sync], [Tabs.verify, tabs1], [Tabs.add, tabs2], [Sync]]); + +Phase("phase3", [[Sync], [Tabs.verify, tabs2]]); diff --git a/services/sync/tests/unit/addon1-search.json b/services/sync/tests/unit/addon1-search.json new file mode 100644 index 0000000000..55f8af8857 --- /dev/null +++ b/services/sync/tests/unit/addon1-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Non-Restartless Test Extension", + "type": "extension", + "guid": "addon1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/addon1.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/bootstrap1-search.json b/services/sync/tests/unit/bootstrap1-search.json new file mode 100644 index 0000000000..8cd1cf43ed --- /dev/null +++ b/services/sync/tests/unit/bootstrap1-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Restartless Test Extension", + "type": "extension", + "guid": "bootstrap1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/bootstrap1.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/head_appinfo.js b/services/sync/tests/unit/head_appinfo.js new file mode 100644 index 0000000000..79379ccea8 --- /dev/null +++ b/services/sync/tests/unit/head_appinfo.js @@ -0,0 +1,58 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +// Required to avoid failures. +do_get_profile(); + +// Init FormHistoryStartup and pretend we opened a profile. +var fhs = Cc["@mozilla.org/satchel/form-history-startup;1"].getService( + Ci.nsIObserver +); +fhs.observe(null, "profile-after-change", null); + +// An app is going to have some prefs set which xpcshell tests don't. +Services.prefs.setStringPref( + "identity.sync.tokenserver.uri", + "http://token-server" +); + +// Make sure to provide the right OS so crypto loads the right binaries +function getOS() { + switch (mozinfo.os) { + case "win": + return "WINNT"; + case "mac": + return "Darwin"; + default: + return "Linux"; + } +} + +const { updateAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); +updateAppInfo({ + name: "XPCShell", + ID: "xpcshell@tests.mozilla.org", + version: "1", + platformVersion: "", + OS: getOS(), +}); + +// Register resource aliases. Normally done in SyncComponents.manifest. +function addResourceAlias() { + const resProt = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + for (let s of ["common", "sync", "crypto"]) { + let uri = Services.io.newURI("resource://gre/modules/services-" + s + "/"); + resProt.setSubstitution("services-" + s, uri); + } +} +addResourceAlias(); diff --git a/services/sync/tests/unit/head_errorhandler_common.js b/services/sync/tests/unit/head_errorhandler_common.js new file mode 100644 index 0000000000..fcf44ec43b --- /dev/null +++ b/services/sync/tests/unit/head_errorhandler_common.js @@ -0,0 +1,195 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_helpers.js */ +/* import-globals-from head_http_server.js */ + +// This file expects Service to be defined in the global scope when EHTestsCommon +// is used (from service.js). +/* global Service */ + +var { Changeset, EngineManager, Store, SyncEngine, Tracker, LegacyTracker } = + ChromeUtils.importESModule("resource://services-sync/engines.sys.mjs"); +var { + ABORT_SYNC_COMMAND, + CLIENT_NOT_CONFIGURED, + CREDENTIALS_CHANGED, + DEFAULT_DOWNLOAD_BATCH_SIZE, + DEFAULT_GUID_FETCH_BATCH_SIZE, + DEFAULT_KEYBUNDLE_NAME, + DEVICE_TYPE_DESKTOP, + DEVICE_TYPE_MOBILE, + ENGINE_APPLY_FAIL, + ENGINE_BATCH_INTERRUPTED, + ENGINE_DOWNLOAD_FAIL, + ENGINE_SUCCEEDED, + ENGINE_UNKNOWN_FAIL, + ENGINE_UPLOAD_FAIL, + HMAC_EVENT_INTERVAL, + IDLE_OBSERVER_BACK_DELAY, + LOGIN_FAILED, + LOGIN_FAILED_INVALID_PASSPHRASE, + LOGIN_FAILED_LOGIN_REJECTED, + LOGIN_FAILED_NETWORK_ERROR, + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_FAILED_NO_USERNAME, + LOGIN_FAILED_SERVER_ERROR, + LOGIN_SUCCEEDED, + MASTER_PASSWORD_LOCKED, + MASTER_PASSWORD_LOCKED_RETRY_INTERVAL, + MAXIMUM_BACKOFF_INTERVAL, + MAX_ERROR_COUNT_BEFORE_BACKOFF, + MAX_HISTORY_DOWNLOAD, + MAX_HISTORY_UPLOAD, + METARECORD_DOWNLOAD_FAIL, + MINIMUM_BACKOFF_INTERVAL, + MULTI_DEVICE_THRESHOLD, + NO_SYNC_NODE_FOUND, + NO_SYNC_NODE_INTERVAL, + OVER_QUOTA, + PREFS_BRANCH, + RESPONSE_OVER_QUOTA, + SCORE_INCREMENT_MEDIUM, + SCORE_INCREMENT_SMALL, + SCORE_INCREMENT_XLARGE, + SCORE_UPDATE_DELAY, + SERVER_MAINTENANCE, + SINGLE_USER_THRESHOLD, + SQLITE_MAX_VARIABLE_NUMBER, + STATUS_DISABLED, + STATUS_OK, + STORAGE_VERSION, + SYNC_FAILED, + SYNC_FAILED_PARTIAL, + SYNC_KEY_DECODED_LENGTH, + SYNC_KEY_ENCODED_LENGTH, + SYNC_SUCCEEDED, + URI_LENGTH_MAX, + VERSION_OUT_OF_DATE, + WEAVE_VERSION, + kFirefoxShuttingDown, + kFirstSyncChoiceNotMade, + kSyncBackoffNotMet, + kSyncMasterPasswordLocked, + kSyncNetworkOffline, + kSyncNotConfigured, + kSyncWeaveDisabled, +} = ChromeUtils.importESModule("resource://services-sync/constants.sys.mjs"); +var { BulkKeyBundle, SyncKeyBundle } = ChromeUtils.importESModule( + "resource://services-sync/keys.sys.mjs" +); + +// Common code for test_errorhandler_{1,2}.js -- pulled out to make it less +// monolithic and take less time to execute. +const EHTestsCommon = { + service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "42"); + response.bodyOutputStream.write(body, body.length); + }, + + async sync_httpd_setup() { + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let catapultEngine = Service.engineManager.get("catapult"); + let catapultSyncID = await catapultEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + catapult: { version: catapultEngine.version, syncID: catapultSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let handler_401 = httpd_handler(401, "Unauthorized"); + return httpd_setup({ + // Normal server behaviour. + "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + + // Credentials are wrong or node reallocated. + "/1.1/janedoe/storage/meta/global": handler_401, + "/1.1/janedoe/info/collections": handler_401, + + // Maintenance or overloaded (503 + Retry-After) at info/collections. + "/1.1/broken.info/info/collections": EHTestsCommon.service_unavailable, + + // Maintenance or overloaded (503 + Retry-After) at meta/global. + "/1.1/broken.meta/storage/meta/global": EHTestsCommon.service_unavailable, + "/1.1/broken.meta/info/collections": collectionsHelper.handler, + + // Maintenance or overloaded (503 + Retry-After) at crypto/keys. + "/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()), + "/1.1/broken.keys/info/collections": collectionsHelper.handler, + "/1.1/broken.keys/storage/crypto/keys": EHTestsCommon.service_unavailable, + + // Maintenance or overloaded (503 + Retry-After) at wiping collection. + "/1.1/broken.wipe/info/collections": collectionsHelper.handler, + "/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()), + "/1.1/broken.wipe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/broken.wipe/storage": EHTestsCommon.service_unavailable, + "/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/broken.wipe/storage/catapult": EHTestsCommon.service_unavailable, + }); + }, + + CatapultEngine: (function () { + function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); + } + CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + if (this.exception) { + throw this.exception; + } + }, + }; + Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + + return CatapultEngine; + })(), + + async generateCredentialsChangedFailure() { + // Make sync fail due to changed credentials. We simply re-encrypt + // the keys with a different Sync Key, without changing the local one. + let newSyncKeyBundle = new BulkKeyBundle("crypto"); + await newSyncKeyBundle.generateRandom(); + let keys = Service.collectionKeys.asWBO(); + await keys.encrypt(newSyncKeyBundle); + return keys.upload(Service.resource(Service.cryptoKeysURL)); + }, + + async setUp(server) { + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + return EHTestsCommon.generateAndUploadKeys(); + }, + + async generateAndUploadKeys() { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let response = await serverKeys.upload( + Service.resource(Service.cryptoKeysURL) + ); + return response.success; + }, +}; diff --git a/services/sync/tests/unit/head_helpers.js b/services/sync/tests/unit/head_helpers.js new file mode 100644 index 0000000000..e79e55e57f --- /dev/null +++ b/services/sync/tests/unit/head_helpers.js @@ -0,0 +1,709 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_errorhandler_common.js */ +/* import-globals-from head_http_server.js */ + +// This file expects Service to be defined in the global scope when EHTestsCommon +// is used (from service.js). +/* global Service */ + +var { AddonTestUtils, MockAsyncShutdown } = ChromeUtils.importESModule( + "resource://testing-common/AddonTestUtils.sys.mjs" +); +var { Async } = ChromeUtils.importESModule( + "resource://services-common/async.sys.mjs" +); +var { CommonUtils } = ChromeUtils.importESModule( + "resource://services-common/utils.sys.mjs" +); +var { PlacesTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PlacesTestUtils.sys.mjs" +); +var { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); +var { SerializableSet, Svc, Utils, getChromeWindow } = + ChromeUtils.importESModule("resource://services-sync/util.sys.mjs"); +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +var { PlacesUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesUtils.sys.mjs" +); +var { PlacesSyncUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesSyncUtils.sys.mjs" +); +var { ObjectUtils } = ChromeUtils.importESModule( + "resource://gre/modules/ObjectUtils.sys.mjs" +); +var { + MockFxaStorageManager, + SyncTestingInfrastructure, + configureFxAccountIdentity, + configureIdentity, + encryptPayload, + getLoginTelemetryScalar, + makeFxAccountsInternalMock, + makeIdentityConfig, + promiseNamedTimer, + promiseZeroTimer, + sumHistogram, + syncTestLogging, + waitForZeroTimer, +} = ChromeUtils.importESModule( + "resource://testing-common/services/sync/utils.sys.mjs" +); +ChromeUtils.defineESModuleGetters(this, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", +}); + +add_setup(async function head_setup() { + // Initialize logging. This will sometimes be reset by a pref reset, + // so it's also called as part of SyncTestingInfrastructure(). + syncTestLogging(); + // If a test imports Service, make sure it is initialized first. + if (typeof Service !== "undefined") { + await Service.promiseInitialized; + } +}); + +ChromeUtils.defineLazyGetter(this, "SyncPingSchema", function () { + let { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" + ); + let { NetUtil } = ChromeUtils.importESModule( + "resource://gre/modules/NetUtil.sys.mjs" + ); + let stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + let schema; + try { + let schemaFile = do_get_file("sync_ping_schema.json"); + stream.init(schemaFile, FileUtils.MODE_RDONLY, FileUtils.PERMS_FILE, 0); + + let bytes = NetUtil.readInputStream(stream, stream.available()); + schema = JSON.parse(new TextDecoder().decode(bytes)); + } finally { + stream.close(); + } + + // Allow tests to make whatever engines they want, this shouldn't cause + // validation failure. + schema.definitions.engine.properties.name = { type: "string" }; + return schema; +}); + +ChromeUtils.defineLazyGetter(this, "SyncPingValidator", function () { + const { JsonSchema } = ChromeUtils.importESModule( + "resource://gre/modules/JsonSchema.sys.mjs" + ); + return new JsonSchema.Validator(SyncPingSchema); +}); + +// This is needed for loadAddonTestFunctions(). +var gGlobalScope = this; + +function ExtensionsTestPath(path) { + if (path[0] != "/") { + throw Error("Path must begin with '/': " + path); + } + + return "../../../../toolkit/mozapps/extensions/test/xpcshell" + path; +} + +function webExtensionsTestPath(path) { + if (path[0] != "/") { + throw Error("Path must begin with '/': " + path); + } + + return "../../../../toolkit/components/extensions/test/xpcshell" + path; +} + +/** + * Loads the WebExtension test functions by importing its test file. + */ +function loadWebExtensionTestFunctions() { + /* import-globals-from ../../../../toolkit/components/extensions/test/xpcshell/head_sync.js */ + const path = webExtensionsTestPath("/head_sync.js"); + let file = do_get_file(path); + let uri = Services.io.newFileURI(file); + Services.scriptloader.loadSubScript(uri.spec, gGlobalScope); +} + +/** + * Installs an add-on from an addonInstall + * + * @param install addonInstall instance to install + */ +async function installAddonFromInstall(install) { + await install.install(); + + Assert.notEqual(null, install.addon); + Assert.notEqual(null, install.addon.syncGUID); + + return install.addon; +} + +/** + * Convenience function to install an add-on from the extensions unit tests. + * + * @param file + * Add-on file to install. + * @param reconciler + * addons reconciler, if passed we will wait on the events to be + * processed before resolving + * @return addon object that was installed + */ +async function installAddon(file, reconciler = null) { + let install = await AddonManager.getInstallForFile(file); + Assert.notEqual(null, install); + const addon = await installAddonFromInstall(install); + if (reconciler) { + await reconciler.queueCaller.promiseCallsComplete(); + } + return addon; +} + +/** + * Convenience function to uninstall an add-on. + * + * @param addon + * Addon instance to uninstall + * @param reconciler + * addons reconciler, if passed we will wait on the events to be + * processed before resolving + */ +async function uninstallAddon(addon, reconciler = null) { + const uninstallPromise = new Promise(res => { + let listener = { + onUninstalled(uninstalled) { + if (uninstalled.id == addon.id) { + AddonManager.removeAddonListener(listener); + res(uninstalled); + } + }, + }; + AddonManager.addAddonListener(listener); + }); + addon.uninstall(); + await uninstallPromise; + if (reconciler) { + await reconciler.queueCaller.promiseCallsComplete(); + } +} + +async function generateNewKeys(collectionKeys, collections = null) { + let wbo = await collectionKeys.generateNewKeysWBO(collections); + let modified = new_timestamp(); + collectionKeys.setContents(wbo.cleartext, modified); +} + +// Helpers for testing open tabs. +// These reflect part of the internal structure of TabEngine, +// and stub part of Service.wm. + +function mockShouldSkipWindow(win) { + return win.closed || win.mockIsPrivate; +} + +function mockGetTabState(tab) { + return tab; +} + +function mockGetWindowEnumerator(urls) { + let elements = []; + + const numWindows = 1; + for (let w = 0; w < numWindows; ++w) { + let tabs = []; + let win = { + closed: false, + mockIsPrivate: false, + gBrowser: { + tabs, + }, + }; + elements.push(win); + + let lastAccessed = 2000; + for (let url of urls) { + tabs.push({ + linkedBrowser: { + currentURI: Services.io.newURI(url), + contentTitle: "title", + }, + lastAccessed, + }); + lastAccessed += 1000; + } + } + + // Always include a closed window and a private window. + elements.push({ + closed: true, + mockIsPrivate: false, + gBrowser: { + tabs: [], + }, + }); + + elements.push({ + closed: false, + mockIsPrivate: true, + gBrowser: { + tabs: [], + }, + }); + + return elements.values(); +} + +// Helper function to get the sync telemetry and add the typically used test +// engine names to its list of allowed engines. +function get_sync_test_telemetry() { + let { SyncTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + SyncTelemetry.tryRefreshDevices = function () {}; + let testEngines = ["rotary", "steam", "sterling", "catapult", "nineties"]; + for (let engineName of testEngines) { + SyncTelemetry.allowedEngines.add(engineName); + } + SyncTelemetry.submissionInterval = -1; + return SyncTelemetry; +} + +function assert_valid_ping(record) { + // Our JSON validator does not like `undefined` values, even though they will + // be skipped when we serialize to JSON. + record = JSON.parse(JSON.stringify(record)); + + // This is called as the test harness tears down due to shutdown. This + // will typically have no recorded syncs, and the validator complains about + // it. So ignore such records (but only ignore when *both* shutdown and + // no Syncs - either of them not being true might be an actual problem) + if (record && (record.why != "shutdown" || !!record.syncs.length)) { + const result = SyncPingValidator.validate(record); + if (!result.valid) { + if (result.errors.length) { + // validation failed - using a simple |deepEqual([], errors)| tends to + // truncate the validation errors in the output and doesn't show that + // the ping actually was - so be helpful. + info("telemetry ping validation failed"); + info("the ping data is: " + JSON.stringify(record, undefined, 2)); + info( + "the validation failures: " + + JSON.stringify(result.errors, undefined, 2) + ); + ok( + false, + "Sync telemetry ping validation failed - see output above for details" + ); + } + } + equal(record.version, 1); + record.syncs.forEach(p => { + lessOrEqual(p.when, Date.now()); + }); + } +} + +// Asserts that `ping` is a ping that doesn't contain any failure information +function assert_success_ping(ping) { + ok(!!ping); + assert_valid_ping(ping); + ping.syncs.forEach(record => { + ok(!record.failureReason, JSON.stringify(record.failureReason)); + equal(undefined, record.status); + greater(record.engines.length, 0); + for (let e of record.engines) { + ok(!e.failureReason); + equal(undefined, e.status); + if (e.validation) { + equal(undefined, e.validation.problems); + equal(undefined, e.validation.failureReason); + } + if (e.outgoing) { + for (let o of e.outgoing) { + equal(undefined, o.failed); + notEqual(undefined, o.sent); + } + } + if (e.incoming) { + equal(undefined, e.incoming.failed); + equal(undefined, e.incoming.newFailed); + notEqual(undefined, e.incoming.applied || e.incoming.reconciled); + } + } + }); +} + +// Hooks into telemetry to validate all pings after calling. +function validate_all_future_pings() { + let telem = get_sync_test_telemetry(); + telem.submit = assert_valid_ping; +} + +function wait_for_pings(expectedPings) { + return new Promise(resolve => { + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + let pings = []; + telem.submit = function (record) { + pings.push(record); + if (pings.length == expectedPings) { + telem.submit = oldSubmit; + resolve(pings); + } + }; + }); +} + +async function wait_for_ping(callback, allowErrorPings, getFullPing = false) { + let pingsPromise = wait_for_pings(1); + await callback(); + let [record] = await pingsPromise; + if (allowErrorPings) { + assert_valid_ping(record); + } else { + assert_success_ping(record); + } + if (getFullPing) { + return record; + } + equal(record.syncs.length, 1); + return record.syncs[0]; +} + +// Perform a sync and validate all telemetry caused by the sync. If fnValidate +// is null, we just check the ping records success. If fnValidate is specified, +// then the sync must have recorded just a single sync, and that sync will be +// passed to the function to be checked. +async function sync_and_validate_telem( + fnValidate = null, + wantFullPing = false +) { + let numErrors = 0; + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + try { + telem.submit = function (record) { + // This is called via an observer, so failures here don't cause the test + // to fail :( + try { + // All pings must be valid. + assert_valid_ping(record); + if (fnValidate) { + // for historical reasons most of these callbacks expect a "sync" + // record, not the entire ping. + if (wantFullPing) { + fnValidate(record); + } else { + Assert.equal(record.syncs.length, 1); + fnValidate(record.syncs[0]); + } + } else { + // no validation function means it must be a "success" ping. + assert_success_ping(record); + } + } catch (ex) { + print("Failure in ping validation callback", ex, "\n", ex.stack); + numErrors += 1; + } + }; + await Service.sync(); + Assert.ok(numErrors == 0, "There were telemetry validation errors"); + } finally { + telem.submit = oldSubmit; + } +} + +// Used for the (many) cases where we do a 'partial' sync, where only a single +// engine is actually synced, but we still want to ensure we're generating a +// valid ping. Returns a promise that resolves to the ping, or rejects with the +// thrown error after calling an optional callback. +async function sync_engine_and_validate_telem( + engine, + allowErrorPings, + onError, + wantFullPing = false +) { + let telem = get_sync_test_telemetry(); + let caughtError = null; + // Clear out status, so failures from previous syncs won't show up in the + // telemetry ping. + let { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" + ); + Status._engines = {}; + Status.partial = false; + // Ideally we'd clear these out like we do with engines, (probably via + // Status.resetSync()), but this causes *numerous* tests to fail, so we just + // assume that if no failureReason or engine failures are set, and the + // status properties are the same as they were initially, that it's just + // a leftover. + // This is only an issue since we're triggering the sync of just one engine, + // without doing any other parts of the sync. + let initialServiceStatus = Status._service; + let initialSyncStatus = Status._sync; + + let oldSubmit = telem.submit; + let submitPromise = new Promise((resolve, reject) => { + telem.submit = function (ping) { + telem.submit = oldSubmit; + ping.syncs.forEach(record => { + if (record && record.status) { + // did we see anything to lead us to believe that something bad actually happened + let realProblem = + record.failureReason || + record.engines.some(e => { + if (e.failureReason || e.status) { + return true; + } + if (e.outgoing && e.outgoing.some(o => o.failed > 0)) { + return true; + } + return e.incoming && e.incoming.failed; + }); + if (!realProblem) { + // no, so if the status is the same as it was initially, just assume + // that its leftover and that we can ignore it. + if (record.status.sync && record.status.sync == initialSyncStatus) { + delete record.status.sync; + } + if ( + record.status.service && + record.status.service == initialServiceStatus + ) { + delete record.status.service; + } + if (!record.status.sync && !record.status.service) { + delete record.status; + } + } + } + }); + if (allowErrorPings) { + assert_valid_ping(ping); + } else { + assert_success_ping(ping); + } + equal(ping.syncs.length, 1); + if (caughtError) { + if (onError) { + onError(ping.syncs[0], ping); + } + reject(caughtError); + } else if (wantFullPing) { + resolve(ping); + } else { + resolve(ping.syncs[0]); + } + }; + }); + // neuter the scheduler as it interacts badly with some of the tests - the + // engine being synced usually isn't the registered engine, so we see + // scored incremented and not removed, which schedules unexpected syncs. + let oldObserve = Service.scheduler.observe; + Service.scheduler.observe = () => {}; + try { + Svc.Obs.notify("weave:service:sync:start"); + try { + await engine.sync(); + } catch (e) { + caughtError = e; + } + if (caughtError) { + Svc.Obs.notify("weave:service:sync:error", caughtError); + } else { + Svc.Obs.notify("weave:service:sync:finish"); + } + } finally { + Service.scheduler.observe = oldObserve; + } + return submitPromise; +} + +// Returns a promise that resolves once the specified observer notification +// has fired. +function promiseOneObserver(topic, callback) { + return new Promise((resolve, reject) => { + let observer = function (subject, data) { + Svc.Obs.remove(topic, observer); + resolve({ subject, data }); + }; + Svc.Obs.add(topic, observer); + }); +} + +async function registerRotaryEngine() { + let { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" + ); + await Service.engineManager.clear(); + + await Service.engineManager.register(RotaryEngine); + let engine = Service.engineManager.get("rotary"); + let syncID = await engine.resetLocalSyncID(); + engine.enabled = true; + + return { engine, syncID, tracker: engine._tracker }; +} + +// Set the validation prefs to attempt validation every time to avoid non-determinism. +function enableValidationPrefs(engines = ["bookmarks"]) { + for (let engine of engines) { + Svc.PrefBranch.setIntPref(`engine.${engine}.validation.interval`, 0); + Svc.PrefBranch.setIntPref( + `engine.${engine}.validation.percentageChance`, + 100 + ); + Svc.PrefBranch.setIntPref(`engine.${engine}.validation.maxRecords`, -1); + Svc.PrefBranch.setBoolPref(`engine.${engine}.validation.enabled`, true); + } +} + +async function serverForEnginesWithKeys(users, engines, callback) { + // Generate and store a fake default key bundle to avoid resetting the client + // before the first sync. + let wbo = await Service.collectionKeys.generateNewKeysWBO(); + let modified = new_timestamp(); + Service.collectionKeys.setContents(wbo.cleartext, modified); + + let allEngines = [Service.clientsEngine].concat(engines); + + let globalEngines = {}; + for (let engine of allEngines) { + let syncID = await engine.resetLocalSyncID(); + globalEngines[engine.name] = { version: engine.version, syncID }; + } + + let contents = { + meta: { + global: { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: globalEngines, + }, + }, + crypto: { + keys: encryptPayload(wbo.cleartext), + }, + }; + for (let engine of allEngines) { + contents[engine.name] = {}; + } + + return serverForUsers(users, contents, callback); +} + +async function serverForFoo(engine, callback) { + // The bookmarks engine *always* tracks changes, meaning we might try + // and sync due to the bookmarks we ourselves create! Worse, because we + // do an engine sync only, there's no locking - so we end up with multiple + // syncs running. Neuter that by making the threshold very large. + Service.scheduler.syncThreshold = 10000000; + return serverForEnginesWithKeys({ foo: "password" }, engine, callback); +} + +// Places notifies history observers asynchronously, so `addVisits` might return +// before the tracker receives the notification. This helper registers an +// observer that resolves once the expected notification fires. +async function promiseVisit(expectedType, expectedURI) { + return new Promise(resolve => { + function done(type, uri) { + if (uri == expectedURI.spec && type == expectedType) { + PlacesObservers.removeListener( + ["page-visited", "page-removed"], + observer.handlePlacesEvents + ); + resolve(); + } + } + let observer = { + handlePlacesEvents(events) { + Assert.equal(events.length, 1); + + if (events[0].type === "page-visited") { + done("added", events[0].url); + } else if (events[0].type === "page-removed") { + Assert.ok(events[0].isRemovedFromStore); + done("removed", events[0].url); + } + }, + }; + PlacesObservers.addListener( + ["page-visited", "page-removed"], + observer.handlePlacesEvents + ); + }); +} + +async function addVisit( + suffix, + referrer = null, + transition = PlacesUtils.history.TRANSITION_LINK +) { + let uriString = "http://getfirefox.com/" + suffix; + let uri = CommonUtils.makeURI(uriString); + _("Adding visit for URI " + uriString); + + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition, + referrer, + }); + await visitAddedPromise; + + return uri; +} + +function bookmarkNodesToInfos(nodes) { + return nodes.map(node => { + let info = { + guid: node.guid, + index: node.index, + }; + if (node.children) { + info.children = bookmarkNodesToInfos(node.children); + } + return info; + }); +} + +async function assertBookmarksTreeMatches(rootGuid, expected, message) { + let root = await PlacesUtils.promiseBookmarksTree(rootGuid, { + includeItemIds: true, + }); + let actual = bookmarkNodesToInfos(root.children); + + if (!ObjectUtils.deepEqual(actual, expected)) { + _(`Expected structure for ${rootGuid}`, JSON.stringify(expected)); + _(`Actual structure for ${rootGuid}`, JSON.stringify(actual)); + throw new Assert.constructor.AssertionError({ actual, expected, message }); + } +} + +function add_bookmark_test(task) { + const { BookmarksEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); + + add_task(async function () { + _(`Running bookmarks test ${task.name}`); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + await engine._resetClient(); + try { + await task(engine); + } finally { + await engine.finalize(); + } + }); +} diff --git a/services/sync/tests/unit/head_http_server.js b/services/sync/tests/unit/head_http_server.js new file mode 100644 index 0000000000..84dbb33951 --- /dev/null +++ b/services/sync/tests/unit/head_http_server.js @@ -0,0 +1,1265 @@ +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_helpers.js */ + +var Cm = Components.manager; + +// Shared logging for all HTTP server functions. +var { Log } = ChromeUtils.importESModule("resource://gre/modules/Log.sys.mjs"); +var { CommonUtils } = ChromeUtils.importESModule( + "resource://services-common/utils.sys.mjs" +); +var { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); +var { + MockFxaStorageManager, + SyncTestingInfrastructure, + configureFxAccountIdentity, + configureIdentity, + encryptPayload, + getLoginTelemetryScalar, + makeFxAccountsInternalMock, + makeIdentityConfig, + promiseNamedTimer, + promiseZeroTimer, + sumHistogram, + syncTestLogging, + waitForZeroTimer, +} = ChromeUtils.importESModule( + "resource://testing-common/services/sync/utils.sys.mjs" +); + +const SYNC_HTTP_LOGGER = "Sync.Test.Server"; + +// While the sync code itself uses 1.5, the tests hard-code 1.1, +// so we're sticking with 1.1 here. +const SYNC_API_VERSION = "1.1"; + +// Use the same method that record.js does, which mirrors the server. +// The server returns timestamps with 1/100 sec granularity. Note that this is +// subject to change: see Bug 650435. +function new_timestamp() { + return round_timestamp(Date.now()); +} + +// Rounds a millisecond timestamp `t` to seconds, with centisecond precision. +function round_timestamp(t) { + return Math.round(t / 10) / 100; +} + +function return_timestamp(request, response, timestamp) { + if (!timestamp) { + timestamp = new_timestamp(); + } + let body = "" + timestamp; + response.setHeader("X-Weave-Timestamp", body); + response.setStatusLine(request.httpVersion, 200, "OK"); + writeBytesToOutputStream(response.bodyOutputStream, body); + return timestamp; +} + +function has_hawk_header(req) { + return ( + req.hasHeader("Authorization") && + req.getHeader("Authorization").startsWith("Hawk") + ); +} + +function basic_auth_header(user, password) { + return "Basic " + btoa(user + ":" + CommonUtils.encodeUTF8(password)); +} + +function basic_auth_matches(req, user, password) { + if (!req.hasHeader("Authorization")) { + return false; + } + + let expected = basic_auth_header(user, CommonUtils.encodeUTF8(password)); + return req.getHeader("Authorization") == expected; +} + +function httpd_basic_auth_handler(body, metadata, response) { + if (basic_auth_matches(metadata, "guest", "guest")) { + response.setStatusLine(metadata.httpVersion, 200, "OK, authorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } else { + body = "This path exists and is protected - failed"; + response.setStatusLine(metadata.httpVersion, 401, "Unauthorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } + writeBytesToOutputStream(response.bodyOutputStream, body); +} + +/* + * Represent a WBO on the server + */ +function ServerWBO(id, initialPayload, modified) { + if (!id) { + throw new Error("No ID for ServerWBO!"); + } + this.id = id; + if (!initialPayload) { + return; + } + + if (typeof initialPayload == "object") { + initialPayload = JSON.stringify(initialPayload); + } + this.payload = initialPayload; + this.modified = modified || new_timestamp(); + this.sortindex = 0; +} +ServerWBO.prototype = { + get data() { + return JSON.parse(this.payload); + }, + + get() { + return { id: this.id, modified: this.modified, payload: this.payload }; + }, + + put(input) { + input = JSON.parse(input); + this.payload = input.payload; + this.modified = new_timestamp(); + this.sortindex = input.sortindex || 0; + }, + + delete() { + delete this.payload; + delete this.modified; + delete this.sortindex; + }, + + // This handler sets `newModified` on the response body if the collection + // timestamp has changed. This allows wrapper handlers to extract information + // that otherwise would exist only in the body stream. + handler() { + let self = this; + + return function (request, response) { + var statusCode = 200; + var status = "OK"; + var body; + + switch (request.method) { + case "GET": + if (self.payload) { + body = JSON.stringify(self.get()); + } else { + statusCode = 404; + status = "Not Found"; + body = "Not Found"; + } + break; + + case "PUT": + self.put(readBytesFromInputStream(request.bodyInputStream)); + body = JSON.stringify(self.modified); + response.setHeader("Content-Type", "application/json"); + response.newModified = self.modified; + break; + + case "DELETE": + self.delete(); + let ts = new_timestamp(); + body = JSON.stringify(ts); + response.setHeader("Content-Type", "application/json"); + response.newModified = ts; + break; + } + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + response.setStatusLine(request.httpVersion, statusCode, status); + writeBytesToOutputStream(response.bodyOutputStream, body); + }; + }, + + /** + * Get the cleartext data stored in the payload. + * + * This isn't `get cleartext`, because `x.cleartext.blah = 3;` wouldn't work, + * which seems like a footgun. + */ + getCleartext() { + return JSON.parse(JSON.parse(this.payload).ciphertext); + }, + + /** + * Setter for getCleartext(), but lets you adjust the modified timestamp too. + * Returns this ServerWBO object. + */ + setCleartext(cleartext, modifiedTimestamp = this.modified) { + this.payload = JSON.stringify(encryptPayload(cleartext)); + this.modified = modifiedTimestamp; + return this; + }, +}; + +/** + * Represent a collection on the server. The '_wbos' attribute is a + * mapping of id -> ServerWBO objects. + * + * Note that if you want these records to be accessible individually, + * you need to register their handlers with the server separately, or use a + * containing HTTP server that will do so on your behalf. + * + * @param wbos + * An object mapping WBO IDs to ServerWBOs. + * @param acceptNew + * If true, POSTs to this collection URI will result in new WBOs being + * created and wired in on the fly. + * @param timestamp + * An optional timestamp value to initialize the modified time of the + * collection. This should be in the format returned by new_timestamp(). + * + * @return the new ServerCollection instance. + * + */ +function ServerCollection(wbos, acceptNew, timestamp) { + this._wbos = wbos || {}; + this.acceptNew = acceptNew || false; + + /* + * Track modified timestamp. + * We can't just use the timestamps of contained WBOs: an empty collection + * has a modified time. + */ + this.timestamp = timestamp || new_timestamp(); + this._log = Log.repository.getLogger(SYNC_HTTP_LOGGER); +} +ServerCollection.prototype = { + /** + * Convenience accessor for our WBO keys. + * Excludes deleted items, of course. + * + * @param filter + * A predicate function (applied to the ID and WBO) which dictates + * whether to include the WBO's ID in the output. + * + * @return an array of IDs. + */ + keys: function keys(filter) { + let ids = []; + for (let [id, wbo] of Object.entries(this._wbos)) { + if (wbo.payload && (!filter || filter(id, wbo))) { + ids.push(id); + } + } + return ids; + }, + + /** + * Convenience method to get an array of WBOs. + * Optionally provide a filter function. + * + * @param filter + * A predicate function, applied to the WBO, which dictates whether to + * include the WBO in the output. + * + * @return an array of ServerWBOs. + */ + wbos: function wbos(filter) { + let os = []; + for (let wbo of Object.values(this._wbos)) { + if (wbo.payload) { + os.push(wbo); + } + } + + if (filter) { + return os.filter(filter); + } + return os; + }, + + /** + * Convenience method to get an array of parsed ciphertexts. + * + * @return an array of the payloads of each stored WBO. + */ + payloads() { + return this.wbos().map(wbo => wbo.getCleartext()); + }, + + // Just for syntactic elegance. + wbo: function wbo(id) { + return this._wbos[id]; + }, + + payload: function payload(id) { + return this.wbo(id).payload; + }, + + cleartext(id) { + return this.wbo(id).getCleartext(); + }, + + /** + * Insert the provided WBO under its ID. + * + * @return the provided WBO. + */ + insertWBO: function insertWBO(wbo) { + this.timestamp = Math.max(this.timestamp, wbo.modified); + return (this._wbos[wbo.id] = wbo); + }, + + /** + * Update an existing WBO's cleartext using a callback function that modifies + * the record in place, or returns a new record. + */ + updateRecord(id, updateCallback, optTimestamp) { + let wbo = this.wbo(id); + if (!wbo) { + throw new Error("No record with provided ID"); + } + let curCleartext = wbo.getCleartext(); + // Allow update callback to either return a new cleartext, or modify in place. + let newCleartext = updateCallback(curCleartext) || curCleartext; + wbo.setCleartext(newCleartext, optTimestamp); + // It is already inserted, but we might need to update our timestamp based + // on it's `modified` value, if `optTimestamp` was provided. + return this.insertWBO(wbo); + }, + + /** + * Insert a record, which may either an object with a cleartext property, or + * the cleartext property itself. + */ + insertRecord(record, timestamp = Math.round(Date.now() / 10) / 100) { + if (typeof timestamp != "number") { + throw new TypeError("insertRecord: Timestamp is not a number."); + } + if (!record.id) { + throw new Error("Attempt to insert record with no id"); + } + // Allow providing either the cleartext directly, or the CryptoWrapper-like. + let cleartext = record.cleartext || record; + return this.insert(record.id, encryptPayload(cleartext), timestamp); + }, + + /** + * Insert the provided payload as part of a new ServerWBO with the provided + * ID. + * + * @param id + * The GUID for the WBO. + * @param payload + * The payload, as provided to the ServerWBO constructor. + * @param modified + * An optional modified time for the ServerWBO. + * + * @return the inserted WBO. + */ + insert: function insert(id, payload, modified) { + return this.insertWBO(new ServerWBO(id, payload, modified)); + }, + + /** + * Removes an object entirely from the collection. + * + * @param id + * (string) ID to remove. + */ + remove: function remove(id) { + delete this._wbos[id]; + }, + + _inResultSet(wbo, options) { + return ( + wbo.payload && + (!options.ids || options.ids.includes(wbo.id)) && + (!options.newer || wbo.modified > options.newer) && + (!options.older || wbo.modified < options.older) + ); + }, + + count(options) { + options = options || {}; + let c = 0; + for (let wbo of Object.values(this._wbos)) { + if (wbo.modified && this._inResultSet(wbo, options)) { + c++; + } + } + return c; + }, + + get(options, request) { + let data = []; + for (let wbo of Object.values(this._wbos)) { + if (wbo.modified && this._inResultSet(wbo, options)) { + data.push(wbo); + } + } + switch (options.sort) { + case "newest": + data.sort((a, b) => b.modified - a.modified); + break; + + case "oldest": + data.sort((a, b) => a.modified - b.modified); + break; + + case "index": + data.sort((a, b) => b.sortindex - a.sortindex); + break; + + default: + if (options.sort) { + this._log.error( + "Error: client requesting unknown sort order", + options.sort + ); + throw new Error("Unknown sort order"); + } + // If the client didn't request a sort order, shuffle the records + // to ensure that we don't accidentally depend on the default order. + TestUtils.shuffle(data); + } + if (options.full) { + data = data.map(wbo => wbo.get()); + let start = options.offset || 0; + if (options.limit) { + let numItemsPastOffset = data.length - start; + data = data.slice(start, start + options.limit); + // use options as a backchannel to set x-weave-next-offset + if (numItemsPastOffset > options.limit) { + options.nextOffset = start + options.limit; + } + } else if (start) { + data = data.slice(start); + } + + if (request && request.getHeader("accept") == "application/newlines") { + this._log.error( + "Error: client requesting application/newlines content" + ); + throw new Error( + "This server should not serve application/newlines content" + ); + } + + // Use options as a backchannel to report count. + options.recordCount = data.length; + } else { + data = data.map(wbo => wbo.id); + let start = options.offset || 0; + if (options.limit) { + data = data.slice(start, start + options.limit); + options.nextOffset = start + options.limit; + } else if (start) { + data = data.slice(start); + } + options.recordCount = data.length; + } + return JSON.stringify(data); + }, + + post(input) { + input = JSON.parse(input); + let success = []; + let failed = {}; + + // This will count records where we have an existing ServerWBO + // registered with us as successful and all other records as failed. + for (let key in input) { + let record = input[key]; + let wbo = this.wbo(record.id); + if (!wbo && this.acceptNew) { + this._log.debug( + "Creating WBO " + JSON.stringify(record.id) + " on the fly." + ); + wbo = new ServerWBO(record.id); + this.insertWBO(wbo); + } + if (wbo) { + wbo.payload = record.payload; + wbo.modified = new_timestamp(); + wbo.sortindex = record.sortindex || 0; + success.push(record.id); + } else { + failed[record.id] = "no wbo configured"; + } + } + return { modified: new_timestamp(), success, failed }; + }, + + delete(options) { + let deleted = []; + for (let wbo of Object.values(this._wbos)) { + if (this._inResultSet(wbo, options)) { + this._log.debug("Deleting " + JSON.stringify(wbo)); + deleted.push(wbo.id); + wbo.delete(); + } + } + return deleted; + }, + + // This handler sets `newModified` on the response body if the collection + // timestamp has changed. + handler() { + let self = this; + + return function (request, response) { + var statusCode = 200; + var status = "OK"; + var body; + + // Parse queryString + let options = {}; + for (let chunk of request.queryString.split("&")) { + if (!chunk) { + continue; + } + chunk = chunk.split("="); + if (chunk.length == 1) { + options[chunk[0]] = ""; + } else { + options[chunk[0]] = chunk[1]; + } + } + // The real servers return 400 if ids= is specified without a list of IDs. + if (options.hasOwnProperty("ids")) { + if (!options.ids) { + response.setStatusLine(request.httpVersion, "400", "Bad Request"); + body = "Bad Request"; + writeBytesToOutputStream(response.bodyOutputStream, body); + return; + } + options.ids = options.ids.split(","); + } + if (options.newer) { + options.newer = parseFloat(options.newer); + } + if (options.older) { + options.older = parseFloat(options.older); + } + if (options.limit) { + options.limit = parseInt(options.limit, 10); + } + if (options.offset) { + options.offset = parseInt(options.offset, 10); + } + + switch (request.method) { + case "GET": + body = self.get(options, request); + // see http://moz-services-docs.readthedocs.io/en/latest/storage/apis-1.5.html + // for description of these headers. + let { recordCount: records, nextOffset } = options; + + self._log.info("Records: " + records + ", nextOffset: " + nextOffset); + if (records != null) { + response.setHeader("X-Weave-Records", "" + records); + } + if (nextOffset) { + response.setHeader("X-Weave-Next-Offset", "" + nextOffset); + } + response.setHeader("X-Last-Modified", "" + self.timestamp); + break; + + case "POST": + let res = self.post( + readBytesFromInputStream(request.bodyInputStream), + request + ); + body = JSON.stringify(res); + response.newModified = res.modified; + break; + + case "DELETE": + self._log.debug("Invoking ServerCollection.DELETE."); + let deleted = self.delete(options, request); + let ts = new_timestamp(); + body = JSON.stringify(ts); + response.newModified = ts; + response.deleted = deleted; + break; + } + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + + // Update the collection timestamp to the appropriate modified time. + // This is either a value set by the handler, or the current time. + if (request.method != "GET") { + self.timestamp = + response.newModified >= 0 ? response.newModified : new_timestamp(); + } + response.setHeader("X-Last-Modified", "" + self.timestamp, false); + + response.setStatusLine(request.httpVersion, statusCode, status); + writeBytesToOutputStream(response.bodyOutputStream, body); + }; + }, +}; + +/* + * Test setup helpers. + */ +function sync_httpd_setup(handlers) { + handlers["/1.1/foo/storage/meta/global"] = new ServerWBO( + "global", + {} + ).handler(); + return httpd_setup(handlers); +} + +/* + * Track collection modified times. Return closures. + * + * XXX - DO NOT USE IN NEW TESTS + * + * This code has very limited and very hacky timestamp support - the test + * server now has more complete and correct support - using this helper + * may cause strangeness wrt timestamp headers and 412 responses. + */ +function track_collections_helper() { + /* + * Our tracking object. + */ + let collections = {}; + + /* + * Update the timestamp of a collection. + */ + function update_collection(coll, ts) { + _("Updating collection " + coll + " to " + ts); + let timestamp = ts || new_timestamp(); + collections[coll] = timestamp; + } + + /* + * Invoke a handler, updating the collection's modified timestamp unless + * it's a GET request. + */ + function with_updated_collection(coll, f) { + return function (request, response) { + f.call(this, request, response); + + // Update the collection timestamp to the appropriate modified time. + // This is either a value set by the handler, or the current time. + if (request.method != "GET") { + update_collection(coll, response.newModified); + } + }; + } + + /* + * Return the info/collections object. + */ + function info_collections(request, response) { + let body = "Error."; + switch (request.method) { + case "GET": + body = JSON.stringify(collections); + break; + default: + throw new Error("Non-GET on info_collections."); + } + + response.setHeader("Content-Type", "application/json"); + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + response.setStatusLine(request.httpVersion, 200, "OK"); + writeBytesToOutputStream(response.bodyOutputStream, body); + } + + return { + collections, + handler: info_collections, + with_updated_collection, + update_collection, + }; +} + +// ===========================================================================// +// httpd.js-based Sync server. // +// ===========================================================================// + +/** + * In general, the preferred way of using SyncServer is to directly introspect + * it. Callbacks are available for operations which are hard to verify through + * introspection, such as deletions. + * + * One of the goals of this server is to provide enough hooks for test code to + * find out what it needs without monkeypatching. Use this object as your + * prototype, and override as appropriate. + */ +var SyncServerCallback = { + onCollectionDeleted: function onCollectionDeleted(user, collection) {}, + onItemDeleted: function onItemDeleted(user, collection, wboID) {}, + + /** + * Called at the top of every request. + * + * Allows the test to inspect the request. Hooks should be careful not to + * modify or change state of the request or they may impact future processing. + * The response is also passed so the callback can set headers etc - but care + * must be taken to not screw with the response body or headers that may + * conflict with normal operation of this server. + */ + onRequest: function onRequest(request, response) {}, +}; + +/** + * Construct a new test Sync server. Takes a callback object (e.g., + * SyncServerCallback) as input. + */ +function SyncServer(callback) { + this.callback = callback || Object.create(SyncServerCallback); + this.server = new HttpServer(); + this.started = false; + this.users = {}; + this._log = Log.repository.getLogger(SYNC_HTTP_LOGGER); + + // Install our own default handler. This allows us to mess around with the + // whole URL space. + let handler = this.server._handler; + handler._handleDefault = this.handleDefault.bind(this, handler); +} +SyncServer.prototype = { + server: null, // HttpServer. + users: null, // Map of username => {collections, password}. + + /** + * Start the SyncServer's underlying HTTP server. + * + * @param port + * The numeric port on which to start. -1 implies the default, a + * randomly chosen port. + * @param cb + * A callback function (of no arguments) which is invoked after + * startup. + */ + start: function start(port = -1, cb) { + if (this.started) { + this._log.warn("Warning: server already started on " + this.port); + return; + } + try { + this.server.start(port); + let i = this.server.identity; + this.port = i.primaryPort; + this.baseURI = + i.primaryScheme + "://" + i.primaryHost + ":" + i.primaryPort + "/"; + this.started = true; + if (cb) { + cb(); + } + } catch (ex) { + _("=========================================="); + _("Got exception starting Sync HTTP server."); + _("Error: " + Log.exceptionStr(ex)); + _("Is there a process already listening on port " + port + "?"); + _("=========================================="); + do_throw(ex); + } + }, + + /** + * Stop the SyncServer's HTTP server. + * + * @param cb + * A callback function. Invoked after the server has been stopped. + * + */ + stop: function stop(cb) { + if (!this.started) { + this._log.warn( + "SyncServer: Warning: server not running. Can't stop me now!" + ); + return; + } + + this.server.stop(cb); + this.started = false; + }, + + /** + * Return a server timestamp for a record. + * The server returns timestamps with 1/100 sec granularity. Note that this is + * subject to change: see Bug 650435. + */ + timestamp: function timestamp() { + return new_timestamp(); + }, + + /** + * Create a new user, complete with an empty set of collections. + * + * @param username + * The username to use. An Error will be thrown if a user by that name + * already exists. + * @param password + * A password string. + * + * @return a user object, as would be returned by server.user(username). + */ + registerUser: function registerUser(username, password) { + if (username in this.users) { + throw new Error("User already exists."); + } + this.users[username] = { + password, + collections: {}, + }; + return this.user(username); + }, + + userExists: function userExists(username) { + return username in this.users; + }, + + getCollection: function getCollection(username, collection) { + return this.users[username].collections[collection]; + }, + + _insertCollection: function _insertCollection(collections, collection, wbos) { + let coll = new ServerCollection(wbos, true); + coll.collectionHandler = coll.handler(); + collections[collection] = coll; + return coll; + }, + + createCollection: function createCollection(username, collection, wbos) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let collections = this.users[username].collections; + if (collection in collections) { + throw new Error("Collection already exists."); + } + return this._insertCollection(collections, collection, wbos); + }, + + /** + * Accept a map like the following: + * { + * meta: {global: {version: 1, ...}}, + * crypto: {"keys": {}, foo: {bar: 2}}, + * bookmarks: {} + * } + * to cause collections and WBOs to be created. + * If a collection already exists, no error is raised. + * If a WBO already exists, it will be updated to the new contents. + */ + createContents: function createContents(username, collections) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for (let [id, contents] of Object.entries(collections)) { + let coll = + userCollections[id] || this._insertCollection(userCollections, id); + for (let [wboID, payload] of Object.entries(contents)) { + coll.insert(wboID, payload); + } + } + }, + + /** + * Insert a WBO in an existing collection. + */ + insertWBO: function insertWBO(username, collection, wbo) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + if (!(collection in userCollections)) { + throw new Error("Unknown collection."); + } + userCollections[collection].insertWBO(wbo); + return wbo; + }, + + /** + * Delete all of the collections for the named user. + * + * @param username + * The name of the affected user. + * + * @return a timestamp. + */ + deleteCollections: function deleteCollections(username) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for (let name in userCollections) { + let coll = userCollections[name]; + this._log.trace("Bulk deleting " + name + " for " + username + "..."); + coll.delete({}); + } + this.users[username].collections = {}; + return this.timestamp(); + }, + + /** + * Simple accessor to allow collective binding and abbreviation of a bunch of + * methods. Yay! + * Use like this: + * + * let u = server.user("john"); + * u.collection("bookmarks").wbo("abcdefg").payload; // Etc. + * + * @return a proxy for the user data stored in this server. + */ + user: function user(username) { + let collection = this.getCollection.bind(this, username); + let createCollection = this.createCollection.bind(this, username); + let createContents = this.createContents.bind(this, username); + let modified = function (collectionName) { + return collection(collectionName).timestamp; + }; + let deleteCollections = this.deleteCollections.bind(this, username); + return { + collection, + createCollection, + createContents, + deleteCollections, + modified, + }; + }, + + /* + * Regular expressions for splitting up Sync request paths. + * Sync URLs are of the form: + * /$apipath/$version/$user/$further + * where $further is usually: + * storage/$collection/$wbo + * or + * storage/$collection + * or + * info/$op + * We assume for the sake of simplicity that $apipath is empty. + * + * N.B., we don't follow any kind of username spec here, because as far as I + * can tell there isn't one. See Bug 689671. Instead we follow the Python + * server code. + * + * Path: [all, version, username, first, rest] + * Storage: [all, collection?, id?] + */ + pathRE: + /^\/([0-9]+(?:\.[0-9]+)?)\/([-._a-zA-Z0-9]+)(?:\/([^\/]+)(?:\/(.+))?)?$/, + storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/, + + defaultHeaders: {}, + + /** + * HTTP response utility. + */ + respond: function respond(req, resp, code, status, body, headers) { + resp.setStatusLine(req.httpVersion, code, status); + if (!headers) { + headers = this.defaultHeaders; + } + for (let header in headers) { + let value = headers[header]; + resp.setHeader(header, value); + } + resp.setHeader("X-Weave-Timestamp", "" + this.timestamp(), false); + writeBytesToOutputStream(resp.bodyOutputStream, body); + }, + + /** + * This is invoked by the HttpServer. `this` is bound to the SyncServer; + * `handler` is the HttpServer's handler. + * + * TODO: need to use the correct Sync API response codes and errors here. + * TODO: Basic Auth. + * TODO: check username in path against username in BasicAuth. + */ + handleDefault: function handleDefault(handler, req, resp) { + try { + this._handleDefault(handler, req, resp); + } catch (e) { + if (e instanceof HttpError) { + this.respond(req, resp, e.code, e.description, "", {}); + } else { + throw e; + } + } + }, + + _handleDefault: function _handleDefault(handler, req, resp) { + this._log.debug( + "SyncServer: Handling request: " + req.method + " " + req.path + ); + + if (this.callback.onRequest) { + this.callback.onRequest(req, resp); + } + + let parts = this.pathRE.exec(req.path); + if (!parts) { + this._log.debug("SyncServer: Unexpected request: bad URL " + req.path); + throw HTTP_404; + } + + let [, version, username, first, rest] = parts; + // Doing a float compare of the version allows for us to pretend there was + // a node-reassignment - eg, we could re-assign from "1.1/user/" to + // "1.10/user" - this server will then still accept requests with the new + // URL while any code in sync itself which compares URLs will see a + // different URL. + if (parseFloat(version) != parseFloat(SYNC_API_VERSION)) { + this._log.debug("SyncServer: Unknown version."); + throw HTTP_404; + } + + if (!this.userExists(username)) { + this._log.debug("SyncServer: Unknown user."); + throw HTTP_401; + } + + // Hand off to the appropriate handler for this path component. + if (first in this.toplevelHandlers) { + let newHandler = this.toplevelHandlers[first]; + return newHandler.call( + this, + newHandler, + req, + resp, + version, + username, + rest + ); + } + this._log.debug("SyncServer: Unknown top-level " + first); + throw HTTP_404; + }, + + /** + * Compute the object that is returned for an info/collections request. + */ + infoCollections: function infoCollections(username) { + let responseObject = {}; + let colls = this.users[username].collections; + for (let coll in colls) { + responseObject[coll] = colls[coll].timestamp; + } + this._log.trace( + "SyncServer: info/collections returning " + JSON.stringify(responseObject) + ); + return responseObject; + }, + + /** + * Collection of the handler methods we use for top-level path components. + */ + toplevelHandlers: { + storage: function handleStorage( + handler, + req, + resp, + version, + username, + rest + ) { + let respond = this.respond.bind(this, req, resp); + if (!rest || !rest.length) { + this._log.debug( + "SyncServer: top-level storage " + req.method + " request." + ); + + // TODO: verify if this is spec-compliant. + if (req.method != "DELETE") { + respond(405, "Method Not Allowed", "[]", { Allow: "DELETE" }); + return undefined; + } + + // Delete all collections and track the timestamp for the response. + let timestamp = this.user(username).deleteCollections(); + + // Return timestamp and OK for deletion. + respond(200, "OK", JSON.stringify(timestamp)); + return undefined; + } + + let match = this.storageRE.exec(rest); + if (!match) { + this._log.warn("SyncServer: Unknown storage operation " + rest); + throw HTTP_404; + } + let [, collection, wboID] = match; + let coll = this.getCollection(username, collection); + + let checkXIUSFailure = () => { + if (req.hasHeader("x-if-unmodified-since")) { + let xius = parseFloat(req.getHeader("x-if-unmodified-since")); + // Sadly the way our tests are setup, we often end up with xius of + // zero (typically when syncing just one engine, so the date from + // info/collections isn't used) - so we allow that to work. + // Further, the Python server treats non-existing collections as + // having a timestamp of 0. + let collTimestamp = coll ? coll.timestamp : 0; + if (xius && xius < collTimestamp) { + this._log.info( + `x-if-unmodified-since mismatch - request wants ${xius} but our collection has ${collTimestamp}` + ); + respond(412, "precondition failed", "precondition failed"); + return true; + } + } + return false; + }; + + switch (req.method) { + case "GET": { + if (!coll) { + if (wboID) { + respond(404, "Not found", "Not found"); + return undefined; + } + // *cries inside*: - apparently the real sync server returned 200 + // here for some time, then returned 404 for some time (bug 687299), + // and now is back to 200 (bug 963332). + respond(200, "OK", "[]"); + return undefined; + } + if (!wboID) { + return coll.collectionHandler(req, resp); + } + let wbo = coll.wbo(wboID); + if (!wbo) { + respond(404, "Not found", "Not found"); + return undefined; + } + return wbo.handler()(req, resp); + } + case "DELETE": { + if (!coll) { + respond(200, "OK", "{}"); + return undefined; + } + if (checkXIUSFailure()) { + return undefined; + } + if (wboID) { + let wbo = coll.wbo(wboID); + if (wbo) { + wbo.delete(); + this.callback.onItemDeleted(username, collection, wboID); + } + respond(200, "OK", "{}"); + return undefined; + } + coll.collectionHandler(req, resp); + + // Spot if this is a DELETE for some IDs, and don't blow away the + // whole collection! + // + // We already handled deleting the WBOs by invoking the deleted + // collection's handler. However, in the case of + // + // DELETE storage/foobar + // + // we also need to remove foobar from the collections map. This + // clause tries to differentiate the above request from + // + // DELETE storage/foobar?ids=foo,baz + // + // and do the right thing. + // TODO: less hacky method. + if (-1 == req.queryString.indexOf("ids=")) { + // When you delete the entire collection, we drop it. + this._log.debug("Deleting entire collection."); + delete this.users[username].collections[collection]; + this.callback.onCollectionDeleted(username, collection); + } + + // Notify of item deletion. + let deleted = resp.deleted || []; + for (let i = 0; i < deleted.length; ++i) { + this.callback.onItemDeleted(username, collection, deleted[i]); + } + return undefined; + } + case "PUT": + // PUT and POST have slightly different XIUS semantics - for PUT, + // the check is against the item, whereas for POST it is against + // the collection. So first, a special-case for PUT. + if (req.hasHeader("x-if-unmodified-since")) { + let xius = parseFloat(req.getHeader("x-if-unmodified-since")); + // treat and xius of zero as if it wasn't specified - this happens + // in some of our tests for a new collection. + if (xius > 0) { + let wbo = coll.wbo(wboID); + if (xius < wbo.modified) { + this._log.info( + `x-if-unmodified-since mismatch - request wants ${xius} but wbo has ${wbo.modified}` + ); + respond(412, "precondition failed", "precondition failed"); + return undefined; + } + wbo.handler()(req, resp); + coll.timestamp = resp.newModified; + return resp; + } + } + // fall through to post. + case "POST": + if (checkXIUSFailure()) { + return undefined; + } + if (!coll) { + coll = this.createCollection(username, collection); + } + + if (wboID) { + let wbo = coll.wbo(wboID); + if (!wbo) { + this._log.trace( + "SyncServer: creating WBO " + collection + "/" + wboID + ); + wbo = coll.insert(wboID); + } + // Rather than instantiate each WBO's handler function, do it once + // per request. They get hit far less often than do collections. + wbo.handler()(req, resp); + coll.timestamp = resp.newModified; + return resp; + } + return coll.collectionHandler(req, resp); + default: + throw new Error("Request method " + req.method + " not implemented."); + } + }, + + info: function handleInfo(handler, req, resp, version, username, rest) { + switch (rest) { + case "collections": + let body = JSON.stringify(this.infoCollections(username)); + this.respond(req, resp, 200, "OK", body, { + "Content-Type": "application/json", + }); + return; + case "collection_usage": + case "collection_counts": + case "quota": + // TODO: implement additional info methods. + this.respond(req, resp, 200, "OK", "TODO"); + return; + default: + // TODO + this._log.warn("SyncServer: Unknown info operation " + rest); + throw HTTP_404; + } + }, + }, +}; + +/** + * Test helper. + */ +function serverForUsers(users, contents, callback) { + let server = new SyncServer(callback); + for (let [user, pass] of Object.entries(users)) { + server.registerUser(user, pass); + server.createContents(user, contents); + } + server.start(); + return server; +} diff --git a/services/sync/tests/unit/missing-sourceuri.json b/services/sync/tests/unit/missing-sourceuri.json new file mode 100644 index 0000000000..dcd487726a --- /dev/null +++ b/services/sync/tests/unit/missing-sourceuri.json @@ -0,0 +1,20 @@ +{ + "next": null, + "results": [ + { + "name": "Restartless Test Extension", + "type": "extension", + "guid": "missing-sourceuri@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485 + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/missing-xpi-search.json b/services/sync/tests/unit/missing-xpi-search.json new file mode 100644 index 0000000000..55f6432b29 --- /dev/null +++ b/services/sync/tests/unit/missing-xpi-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Non-Restartless Test Extension", + "type": "extension", + "guid": "missing-xpi@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/THIS_DOES_NOT_EXIST.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/prefs_test_prefs_store.js b/services/sync/tests/unit/prefs_test_prefs_store.js new file mode 100644 index 0000000000..63851a6934 --- /dev/null +++ b/services/sync/tests/unit/prefs_test_prefs_store.js @@ -0,0 +1,47 @@ +// This is a "preferences" file used by test_prefs_store.js + +/* global pref, user_pref */ + +// The prefs that control what should be synced. +// Most of these are "default" prefs, so the value itself will not sync. +pref("services.sync.prefs.sync.testing.int", true); +pref("services.sync.prefs.sync.testing.string", true); +pref("services.sync.prefs.sync.testing.bool", true); +pref("services.sync.prefs.sync.testing.dont.change", true); +// This is a default pref, but has the special "sync-seen" pref. +pref("services.sync.prefs.sync.testing.seen", true); +pref("services.sync.prefs.sync-seen.testing.seen", false); + +// this one is a user pref, so it *will* sync. +user_pref("services.sync.prefs.sync.testing.turned.off", false); +pref("services.sync.prefs.sync.testing.nonexistent", true); +pref("services.sync.prefs.sync.testing.default", true); +pref("services.sync.prefs.sync.testing.synced.url", true); +// We shouldn't sync the URL, or the flag that says we should sync the pref +// (otherwise some other client might overwrite our local value). +user_pref("services.sync.prefs.sync.testing.unsynced.url", true); + +// The preference values - these are all user_prefs, otherwise their value +// will not be synced. +user_pref("testing.int", 123); +user_pref("testing.string", "ohai"); +user_pref("testing.bool", true); +user_pref("testing.dont.change", "Please don't change me."); +user_pref("testing.turned.off", "I won't get synced."); +user_pref("testing.not.turned.on", "I won't get synced either!"); +// Some url we don't want to sync +user_pref( + "testing.unsynced.url", + "moz-extension://d5d31b00-b944-4afb-bd3d-d0326551a0ae" +); +user_pref("testing.synced.url", "https://www.example.com"); + +// A pref that exists but still has the default value - will be synced with +// null as the value. +pref("testing.default", "I'm the default value"); + +// A pref that has the default value - it will start syncing as soon as +// we see a change, even if the change is to the default. +pref("testing.seen", "the value"); + +// A pref that shouldn't be synced diff --git a/services/sync/tests/unit/rewrite-search.json b/services/sync/tests/unit/rewrite-search.json new file mode 100644 index 0000000000..740b6f2c30 --- /dev/null +++ b/services/sync/tests/unit/rewrite-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Rewrite Test Extension", + "type": "extension", + "guid": "rewrite@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/require.xpi?src=api" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/sync_ping_schema.json b/services/sync/tests/unit/sync_ping_schema.json new file mode 100644 index 0000000000..a9866e3550 --- /dev/null +++ b/services/sync/tests/unit/sync_ping_schema.json @@ -0,0 +1,262 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "schema for Sync pings, documentation avaliable in toolkit/components/telemetry/docs/sync-ping.rst", + "type": "object", + "additionalProperties": false, + "required": ["version", "syncs", "why", "uid"], + "properties": { + "version": { "type": "integer", "minimum": 0 }, + "os": { "type": "object" }, + "discarded": { "type": "integer", "minimum": 1 }, + "why": { "enum": ["shutdown", "schedule", "idchange"] }, + "uid": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "deviceID": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "devices": { + "type": "array", + "items": { "$ref": "#/definitions/device" } + }, + "sessionStartDate": { "type": "string" }, + "syncs": { + "type": "array", + "minItems": 0, + "items": { "$ref": "#/definitions/payload" } + }, + "syncNodeType": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/event" } + }, + "migrations": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/migration" } + }, + "histograms": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "min": { "type": "integer" }, + "max": { "type": "integer" }, + "histogram_type": { "type": "integer" }, + "sum": { "type": "integer" }, + "ranges": { "type": "array" }, + "counts": { "type": "array" } + } + } + } + }, + "definitions": { + "payload": { + "type": "object", + "additionalProperties": false, + "required": ["when", "took"], + "properties": { + "didLogin": { "type": "boolean" }, + "when": { "type": "integer" }, + "status": { + "type": "object", + "anyOf": [{ "required": ["sync"] }, { "required": ["service"] }], + "additionalProperties": false, + "properties": { + "sync": { "type": "string" }, + "service": { "type": "string" } + } + }, + "why": { "type": "string" }, + "took": { "type": "integer", "minimum": -1 }, + "failureReason": { "$ref": "#/definitions/error" }, + "engines": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/engine" } + } + } + }, + "device": { + "required": ["id"], + "additionalProperties": false, + "type": "object", + "properties": { + "id": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "os": { "type": "string" }, + "version": { "type": "string" }, + "type": { "type": "string" }, + "syncID": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + "engine": { + "required": ["name"], + "additionalProperties": false, + "properties": { + "failureReason": { "$ref": "#/definitions/error" }, + "name": { "type": "string" }, + "took": { "type": "integer", "minimum": 1 }, + "status": { "type": "string" }, + "incoming": { + "type": "object", + "additionalProperties": false, + "anyOf": [{ "required": ["applied"] }, { "required": ["failed"] }], + "properties": { + "applied": { "type": "integer", "minimum": 1 }, + "failed": { "type": "integer", "minimum": 1 }, + "failedReasons": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "outgoing": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/outgoingBatch" } + }, + "steps": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/step" + }, + "validation": { + "type": "object", + "additionalProperties": false, + "anyOf": [ + { "required": ["checked"] }, + { "required": ["failureReason"] } + ], + "properties": { + "checked": { "type": "integer", "minimum": 0 }, + "failureReason": { "$ref": "#/definitions/error" }, + "took": { "type": "integer" }, + "version": { "type": "integer" }, + "problems": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + } + } + }, + "outgoingBatch": { + "type": "object", + "additionalProperties": false, + "anyOf": [{ "required": ["sent"] }, { "required": ["failed"] }], + "properties": { + "sent": { "type": "integer", "minimum": 1 }, + "failed": { "type": "integer", "minimum": 1 }, + "failedReasons": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "event": { + "type": "array", + "minItems": 4, + "maxItems": 6 + }, + "migration": { + "oneOf": [{ "$ref": "#/definitions/webextMigration" }] + }, + "webextMigration": { + "required": ["type"], + "properties": { + "type": { "enum": ["webext-storage"] }, + "entries": { "type": "integer" }, + "entriesSuccessful": { "type": "integer" }, + "extensions": { "type": "integer" }, + "extensionsSuccessful": { "type": "integer" }, + "openFailure": { "type": "boolean" } + } + }, + "error": { + "oneOf": [ + { "$ref": "#/definitions/httpError" }, + { "$ref": "#/definitions/nsError" }, + { "$ref": "#/definitions/shutdownError" }, + { "$ref": "#/definitions/authError" }, + { "$ref": "#/definitions/otherError" }, + { "$ref": "#/definitions/unexpectedError" }, + { "$ref": "#/definitions/sqlError" } + ] + }, + "httpError": { + "required": ["name", "code"], + "properties": { + "name": { "enum": ["httperror"] }, + "code": { "type": "integer" } + } + }, + "nsError": { + "required": ["name", "code"], + "properties": { + "name": { "enum": ["nserror"] }, + "code": { "type": "integer" } + } + }, + "shutdownError": { + "required": ["name"], + "properties": { + "name": { "enum": ["shutdownerror"] } + } + }, + "authError": { + "required": ["name"], + "properties": { + "name": { "enum": ["autherror"] }, + "from": { "enum": ["tokenserver", "fxaccounts", "hawkclient"] } + } + }, + "otherError": { + "required": ["name"], + "properties": { + "name": { "enum": ["othererror"] }, + "error": { "type": "string" } + } + }, + "unexpectedError": { + "required": ["name"], + "properties": { + "name": { "enum": ["unexpectederror"] }, + "error": { "type": "string" } + } + }, + "sqlError": { + "required": ["name"], + "properties": { + "name": { "enum": ["sqlerror"] }, + "code": { "type": "integer" } + } + }, + "step": { + "required": ["name"], + "properties": { + "name": { "type": "string" }, + "took": { "type": "integer", "minimum": 1 }, + "counts": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "namedCount": { + "required": ["name", "count"], + "properties": { + "name": { "type": "string" }, + "count": { "type": "integer" } + } + } + } +} diff --git a/services/sync/tests/unit/systemaddon-search.json b/services/sync/tests/unit/systemaddon-search.json new file mode 100644 index 0000000000..a812714918 --- /dev/null +++ b/services/sync/tests/unit/systemaddon-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "System Add-on Test", + "type": "extension", + "guid": "system1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 999, + "url": "http://127.0.0.1:8888/system.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/test_412.js b/services/sync/tests/unit/test_412.js new file mode 100644 index 0000000000..de0a2c087e --- /dev/null +++ b/services/sync/tests/unit/test_412.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +add_task(async function test_412_not_treated_as_failure() { + await Service.engineManager.register(RotaryEngine); + let engine = Service.engineManager.get("rotary"); + + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + // add an item to the server to the first sync advances lastModified. + let collection = server.getCollection("foo", "rotary"); + let payload = encryptPayload({ + id: "existing", + something: "existing record", + }); + collection.insert("existing", payload); + + let promiseObserved = promiseOneObserver("weave:engine:sync:finish"); + try { + // Do sync. + _("initial sync to initialize the world"); + await Service.sync(); + + // create a new record that should be uploaded and arrange for our lastSync + // timestamp to be wrong so we get a 412. + engine._store.items = { new: "new record" }; + await engine._tracker.addChangedID("new", 0); + + let saw412 = false; + let _uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async () => { + let lastSync = await engine.getLastSync(); + await engine.setLastSync(lastSync - 2); + try { + await _uploadOutgoing.call(engine); + } catch (ex) { + saw412 = ex.status == 412; + throw ex; + } + }; + _("Second sync - expecting a 412"); + await Service.sync(); + await promiseObserved; + ok(saw412, "did see a 412 error"); + // But service status should be OK as the 412 shouldn't be treated as an error. + equal(Service.status.service, STATUS_OK); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_addon_utils.js b/services/sync/tests/unit/test_addon_utils.js new file mode 100644 index 0000000000..c039bee16c --- /dev/null +++ b/services/sync/tests/unit/test_addon_utils.js @@ -0,0 +1,156 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonUtils } = ChromeUtils.importESModule( + "resource://services-sync/addonutils.sys.mjs" +); + +const HTTP_PORT = 8888; +const SERVER_ADDRESS = "http://127.0.0.1:8888"; + +Services.prefs.setStringPref( + "extensions.getAddons.get.url", + SERVER_ADDRESS + "/search/guid:%IDS%" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); +}); + +function createAndStartHTTPServer(port = HTTP_PORT) { + try { + let server = new HttpServer(); + + server.registerFile( + "/search/guid:missing-sourceuri%40tests.mozilla.org", + do_get_file("missing-sourceuri.json") + ); + + server.registerFile( + "/search/guid:rewrite%40tests.mozilla.org", + do_get_file("rewrite-search.json") + ); + + server.start(port); + + return server; + } catch (ex) { + _("Got exception starting HTTP server on port " + port); + _("Error: " + Log.exceptionStr(ex)); + do_throw(ex); + } + return null; /* not hit, but keeps eslint happy! */ +} + +function run_test() { + syncTestLogging(); + + run_next_test(); +} + +add_task(async function test_handle_empty_source_uri() { + _("Ensure that search results without a sourceURI are properly ignored."); + + let server = createAndStartHTTPServer(); + + const ID = "missing-sourceuri@tests.mozilla.org"; + + const result = await AddonUtils.installAddons([ + { id: ID, requireSecureURI: false }, + ]); + + Assert.ok("installedIDs" in result); + Assert.equal(0, result.installedIDs.length); + + Assert.ok("skipped" in result); + Assert.ok(result.skipped.includes(ID)); + + await promiseStopServer(server); +}); + +add_test(function test_ignore_untrusted_source_uris() { + _("Ensures that source URIs from insecure schemes are rejected."); + + const bad = [ + "http://example.com/foo.xpi", + "ftp://example.com/foo.xpi", + "silly://example.com/foo.xpi", + ]; + + const good = ["https://example.com/foo.xpi"]; + + for (let s of bad) { + let sourceURI = Services.io.newURI(s); + let addon = { sourceURI, name: "bad", id: "bad" }; + + let canInstall = AddonUtils.canInstallAddon(addon); + Assert.ok(!canInstall, "Correctly rejected a bad URL"); + } + + for (let s of good) { + let sourceURI = Services.io.newURI(s); + let addon = { sourceURI, name: "good", id: "good" }; + + let canInstall = AddonUtils.canInstallAddon(addon); + Assert.ok(canInstall, "Correctly accepted a good URL"); + } + run_next_test(); +}); + +add_task(async function test_source_uri_rewrite() { + _("Ensure that a 'src=api' query string is rewritten to 'src=sync'"); + + // This tests for conformance with bug 708134 so server-side metrics aren't + // skewed. + + // We resort to monkeypatching because of the API design. + let oldFunction = + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult; + + let installCalled = false; + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult = + async function testInstallAddon(addon, metadata) { + Assert.equal( + SERVER_ADDRESS + "/require.xpi?src=sync", + addon.sourceURI.spec + ); + + installCalled = true; + + const install = await AddonUtils.getInstallFromSearchResult(addon); + Assert.equal( + SERVER_ADDRESS + "/require.xpi?src=sync", + install.sourceURI.spec + ); + Assert.deepEqual( + install.installTelemetryInfo, + { source: "sync" }, + "Got the expected installTelemetryInfo" + ); + + return { id: addon.id, addon, install }; + }; + + let server = createAndStartHTTPServer(); + + let installOptions = { + id: "rewrite@tests.mozilla.org", + requireSecureURI: false, + }; + await AddonUtils.installAddons([installOptions]); + + Assert.ok(installCalled); + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult = oldFunction; + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_addons_engine.js b/services/sync/tests/unit/test_addons_engine.js new file mode 100644 index 0000000000..d957ed8fd3 --- /dev/null +++ b/services/sync/tests/unit/test_addons_engine.js @@ -0,0 +1,277 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonManager } = ChromeUtils.importESModule( + "resource://gre/modules/AddonManager.sys.mjs" +); +const { CHANGE_INSTALLED } = ChromeUtils.importESModule( + "resource://services-sync/addonsreconciler.sys.mjs" +); +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +Services.prefs.setStringPref( + "extensions.getAddons.get.url", + "http://localhost:8888/search/guid:%IDS%" +); +Services.prefs.setBoolPref("extensions.install.requireSecureOrigin", false); + +let engine; +let syncID; +let reconciler; +let tracker; + +AddonTestUtils.init(this); + +const ADDON_ID = "addon1@tests.mozilla.org"; +const XPI = AddonTestUtils.createTempWebExtensionFile({ + manifest: { + name: "Test 1", + description: "Test Description", + browser_specific_settings: { gecko: { id: ADDON_ID } }, + }, +}); + +async function resetReconciler() { + reconciler._addons = {}; + reconciler._changes = []; + + await reconciler.saveState(); + + await tracker.clearChangedIDs(); +} + +add_task(async function setup() { + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" + ); + AddonTestUtils.overrideCertDB(); + await AddonTestUtils.promiseStartupManager(); + + await Service.engineManager.register(AddonsEngine); + engine = Service.engineManager.get("addons"); + syncID = await engine.resetLocalSyncID(); + reconciler = engine._reconciler; + tracker = engine._tracker; + + reconciler.startListening(); + + // Don't flush to disk in the middle of an event listener! + // This causes test hangs on WinXP. + reconciler._shouldPersist = false; + + await resetReconciler(); +}); + +// This is a basic sanity test for the unit test itself. If this breaks, the +// add-ons API likely changed upstream. +add_task(async function test_addon_install() { + _("Ensure basic add-on APIs work as expected."); + + let install = await AddonManager.getInstallForFile(XPI); + Assert.notEqual(install, null); + Assert.equal(install.type, "extension"); + Assert.equal(install.name, "Test 1"); + + await resetReconciler(); +}); + +add_task(async function test_find_dupe() { + _("Ensure the _findDupe() implementation is sane."); + + // This gets invoked at the top of sync, which is bypassed by this + // test, so we do it manually. + await engine._refreshReconcilerState(); + + let addon = await installAddon(XPI, reconciler); + + let record = { + id: Utils.makeGUID(), + addonID: ADDON_ID, + enabled: true, + applicationID: Services.appinfo.ID, + source: "amo", + }; + + let dupe = await engine._findDupe(record); + Assert.equal(addon.syncGUID, dupe); + + record.id = addon.syncGUID; + dupe = await engine._findDupe(record); + Assert.equal(null, dupe); + + await uninstallAddon(addon, reconciler); + await resetReconciler(); +}); + +add_task(async function test_get_changed_ids() { + let timerPrecision = Services.prefs.getBoolPref( + "privacy.reduceTimerPrecision" + ); + Services.prefs.setBoolPref("privacy.reduceTimerPrecision", false); + + registerCleanupFunction(function () { + Services.prefs.setBoolPref("privacy.reduceTimerPrecision", timerPrecision); + }); + + _("Ensure getChangedIDs() has the appropriate behavior."); + + _("Ensure getChangedIDs() returns an empty object by default."); + let changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(0, Object.keys(changes).length); + + _("Ensure tracker changes are populated."); + let now = new Date(); + let changeTime = now.getTime() / 1000; + let guid1 = Utils.makeGUID(); + await tracker.addChangedID(guid1, changeTime); + + changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(guid1 in changes); + Assert.equal(changeTime, changes[guid1]); + + await tracker.clearChangedIDs(); + + _("Ensure reconciler changes are populated."); + let addon = await installAddon(XPI, reconciler); + await tracker.clearChangedIDs(); // Just in case. + changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(addon.syncGUID in changes); + _( + "Change time: " + changeTime + ", addon change: " + changes[addon.syncGUID] + ); + Assert.ok(changes[addon.syncGUID] >= changeTime); + + let oldTime = changes[addon.syncGUID]; + let guid2 = addon.syncGUID; + await uninstallAddon(addon, reconciler); + changes = await engine.getChangedIDs(); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(guid2 in changes); + Assert.ok(changes[guid2] > oldTime); + + _("Ensure non-syncable add-ons aren't picked up by reconciler changes."); + reconciler._addons = {}; + reconciler._changes = []; + let record = { + id: "DUMMY", + guid: Utils.makeGUID(), + enabled: true, + installed: true, + modified: new Date(), + type: "UNSUPPORTED", + scope: 0, + foreignInstall: false, + }; + reconciler.addons.DUMMY = record; + await reconciler._addChange(record.modified, CHANGE_INSTALLED, record); + + changes = await engine.getChangedIDs(); + _(JSON.stringify(changes)); + Assert.equal(0, Object.keys(changes).length); + + await resetReconciler(); +}); + +add_task(async function test_disabled_install_semantics() { + _("Ensure that syncing a disabled add-on preserves proper state."); + + // This is essentially a test for bug 712542, which snuck into the original + // add-on sync drop. It ensures that when an add-on is installed that the + // disabled state and incoming syncGUID is preserved, even on the next sync. + const USER = "foo"; + const PASSWORD = "password"; + + let server = new SyncServer(); + server.start(); + await SyncTestingInfrastructure(server, USER, PASSWORD); + + await generateNewKeys(Service.collectionKeys); + + let contents = { + meta: { + global: { engines: { addons: { version: engine.version, syncID } } }, + }, + crypto: {}, + addons: {}, + }; + + server.registerUser(USER, "password"); + server.createContents(USER, contents); + + let amoServer = new HttpServer(); + amoServer.registerFile( + "/search/guid:addon1%40tests.mozilla.org", + do_get_file("addon1-search.json") + ); + + amoServer.registerFile("/addon1.xpi", XPI); + amoServer.start(8888); + + // Insert an existing record into the server. + let id = Utils.makeGUID(); + let now = Date.now() / 1000; + + let record = encryptPayload({ + id, + applicationID: Services.appinfo.ID, + addonID: ADDON_ID, + enabled: false, + deleted: false, + source: "amo", + }); + let wbo = new ServerWBO(id, record, now - 2); + server.insertWBO(USER, "addons", wbo); + + _("Performing sync of add-ons engine."); + await engine._sync(); + + // At this point the non-restartless extension should be staged for install. + + // Don't need this server any more. + await promiseStopServer(amoServer); + + // We ensure the reconciler has recorded the proper ID and enabled state. + let addon = reconciler.getAddonStateFromSyncGUID(id); + Assert.notEqual(null, addon); + Assert.equal(false, addon.enabled); + + // We fake an app restart and perform another sync, just to make sure things + // are sane. + await AddonTestUtils.promiseRestartManager(); + + let collection = server.getCollection(USER, "addons"); + engine.lastModified = collection.timestamp; + await engine._sync(); + + // The client should not upload a new record. The old record should be + // retained and unmodified. + Assert.equal(1, collection.count()); + + let payload = collection.payloads()[0]; + Assert.notEqual(null, collection.wbo(id)); + Assert.equal(ADDON_ID, payload.addonID); + Assert.ok(!payload.enabled); + + await promiseStopServer(server); +}); + +add_test(function cleanup() { + // There's an xpcom-shutdown hook for this, but let's give this a shot. + reconciler.stopListening(); + run_next_test(); +}); diff --git a/services/sync/tests/unit/test_addons_reconciler.js b/services/sync/tests/unit/test_addons_reconciler.js new file mode 100644 index 0000000000..c72b18b00e --- /dev/null +++ b/services/sync/tests/unit/test_addons_reconciler.js @@ -0,0 +1,209 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonsReconciler, CHANGE_INSTALLED, CHANGE_UNINSTALLED } = + ChromeUtils.importESModule( + "resource://services-sync/addonsreconciler.sys.mjs" + ); +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +const ADDON_ID = "addon1@tests.mozilla.org"; +const XPI = AddonTestUtils.createTempWebExtensionFile({ + manifest: { + name: "Test 1", + description: "Test Description", + browser_specific_settings: { gecko: { id: ADDON_ID } }, + }, +}); + +function makeAddonsReconciler() { + const log = Service.engineManager.get("addons")._log; + const queueCaller = Async.asyncQueueCaller(log); + return new AddonsReconciler(queueCaller); +} + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); + Svc.PrefBranch.setBoolPref("engine.addons", true); + await Service.engineManager.register(AddonsEngine); +}); + +add_task(async function test_defaults() { + _("Ensure new objects have reasonable defaults."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + Assert.ok(!reconciler._listening); + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); + Assert.equal(0, reconciler._listeners.length); +}); + +add_task(async function test_load_state_empty_file() { + _("Ensure loading from a missing file results in defaults being set."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + let loaded = await reconciler.loadState(); + Assert.ok(!loaded); + + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); +}); + +add_task(async function test_install_detection() { + _("Ensure that add-on installation results in appropriate side-effects."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler.startListening(); + + let before = new Date(); + let addon = await installAddon(XPI); + let after = new Date(); + + Assert.equal(1, Object.keys(reconciler.addons).length); + Assert.ok(addon.id in reconciler.addons); + let record = reconciler.addons[ADDON_ID]; + + const KEYS = [ + "id", + "guid", + "enabled", + "installed", + "modified", + "type", + "scope", + "foreignInstall", + ]; + for (let key of KEYS) { + Assert.ok(key in record); + Assert.notEqual(null, record[key]); + } + + Assert.equal(addon.id, record.id); + Assert.equal(addon.syncGUID, record.guid); + Assert.ok(record.enabled); + Assert.ok(record.installed); + Assert.ok(record.modified >= before && record.modified <= after); + Assert.equal("extension", record.type); + Assert.ok(!record.foreignInstall); + + Assert.equal(1, reconciler._changes.length); + let change = reconciler._changes[0]; + Assert.ok(change[0] >= before && change[1] <= after); + Assert.equal(CHANGE_INSTALLED, change[1]); + Assert.equal(addon.id, change[2]); + + await uninstallAddon(addon); +}); + +add_task(async function test_uninstall_detection() { + _("Ensure that add-on uninstallation results in appropriate side-effects."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler.startListening(); + + reconciler._addons = {}; + reconciler._changes = []; + + let addon = await installAddon(XPI); + let id = addon.id; + + reconciler._changes = []; + await uninstallAddon(addon, reconciler); + + Assert.equal(1, Object.keys(reconciler.addons).length); + Assert.ok(id in reconciler.addons); + + let record = reconciler.addons[id]; + Assert.ok(!record.installed); + + Assert.equal(1, reconciler._changes.length); + let change = reconciler._changes[0]; + Assert.equal(CHANGE_UNINSTALLED, change[1]); + Assert.equal(id, change[2]); +}); + +add_task(async function test_load_state_future_version() { + _("Ensure loading a file from a future version results in no data loaded."); + + const FILENAME = "TEST_LOAD_STATE_FUTURE_VERSION"; + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + // First we populate our new file. + let state = { version: 100, addons: { foo: {} }, changes: [[1, 1, "foo"]] }; + + // jsonSave() expects an object with ._log, so we give it a reconciler + // instance. + await Utils.jsonSave(FILENAME, reconciler, state); + + let loaded = await reconciler.loadState(FILENAME); + Assert.ok(!loaded); + + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); +}); + +add_task(async function test_prune_changes_before_date() { + _("Ensure that old changes are pruned properly."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler._changes = []; + + let now = new Date(); + const HOUR_MS = 1000 * 60 * 60; + + _("Ensure pruning an empty changes array works."); + reconciler.pruneChangesBeforeDate(now); + Assert.equal(0, reconciler._changes.length); + + let old = new Date(now.getTime() - HOUR_MS); + let young = new Date(now.getTime() - 1000); + reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]); + reconciler._changes.push([young, CHANGE_INSTALLED, "bar"]); + Assert.equal(2, reconciler._changes.length); + + _("Ensure pruning with an old time won't delete anything."); + let threshold = new Date(old.getTime() - 1); + reconciler.pruneChangesBeforeDate(threshold); + Assert.equal(2, reconciler._changes.length); + + _("Ensure pruning a single item works."); + threshold = new Date(young.getTime() - 1000); + reconciler.pruneChangesBeforeDate(threshold); + Assert.equal(1, reconciler._changes.length); + Assert.notEqual(undefined, reconciler._changes[0]); + Assert.equal(young, reconciler._changes[0][0]); + Assert.equal("bar", reconciler._changes[0][2]); + + _("Ensure pruning all changes works."); + reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]); + reconciler.pruneChangesBeforeDate(now); + Assert.equal(0, reconciler._changes.length); +}); diff --git a/services/sync/tests/unit/test_addons_store.js b/services/sync/tests/unit/test_addons_store.js new file mode 100644 index 0000000000..a0a3ac6c69 --- /dev/null +++ b/services/sync/tests/unit/test_addons_store.js @@ -0,0 +1,750 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +const HTTP_PORT = 8888; + +Services.prefs.setStringPref( + "extensions.getAddons.get.url", + "http://localhost:8888/search/guid:%IDS%" +); +// Note that all compat-override URLs currently 404, but that's OK - the main +// thing is to avoid us hitting the real AMO. +Services.prefs.setStringPref( + "extensions.getAddons.compatOverides.url", + "http://localhost:8888/compat-override/guid:%IDS%" +); +Services.prefs.setBoolPref("extensions.install.requireSecureOrigin", false); +Services.prefs.setBoolPref("extensions.checkUpdateSecurity", false); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +Services.prefs.setBoolPref("extensions.experiments.enabled", true); + +const SYSTEM_ADDON_ID = "system1@tests.mozilla.org"; +add_task(async function setupSystemAddon() { + const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "app0"]); + distroDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); + AddonTestUtils.registerDirectory("XREAppFeat", distroDir); + + let xpi = await AddonTestUtils.createTempWebExtensionFile({ + manifest: { + browser_specific_settings: { gecko: { id: SYSTEM_ADDON_ID } }, + }, + }); + + xpi.copyTo(distroDir, `${SYSTEM_ADDON_ID}.xpi`); + + await AddonTestUtils.overrideBuiltIns({ system: [SYSTEM_ADDON_ID] }); + await AddonTestUtils.promiseStartupManager(); +}); + +const ID1 = "addon1@tests.mozilla.org"; +const ID2 = "addon2@tests.mozilla.org"; +const ID3 = "addon3@tests.mozilla.org"; + +const ADDONS = { + test_addon1: { + manifest: { + browser_specific_settings: { + gecko: { + id: ID1, + update_url: "http://example.com/data/test_install.json", + }, + }, + }, + }, + + test_addon2: { + manifest: { + browser_specific_settings: { gecko: { id: ID2 } }, + }, + }, + + test_addon3: { + manifest: { + browser_specific_settings: { + gecko: { + id: ID3, + strict_max_version: "0", + }, + }, + }, + }, +}; + +const SEARCH_RESULT = { + next: null, + results: [ + { + name: "Test Extension", + type: "extension", + guid: "addon1@tests.mozilla.org", + current_version: { + version: "1.0", + files: [ + { + platform: "all", + size: 485, + url: "http://localhost:8888/addon1.xpi", + }, + ], + }, + last_updated: "2018-10-27T04:12:00.826Z", + }, + ], +}; + +const MISSING_SEARCH_RESULT = { + next: null, + results: [ + { + name: "Test", + type: "extension", + guid: "missing-xpi@tests.mozilla.org", + current_version: { + version: "1.0", + files: [ + { + platform: "all", + size: 123, + url: "http://localhost:8888/THIS_DOES_NOT_EXIST.xpi", + }, + ], + }, + }, + ], +}; + +const XPIS = {}; +for (let [name, files] of Object.entries(ADDONS)) { + XPIS[name] = AddonTestUtils.createTempWebExtensionFile(files); +} + +let engine; +let store; +let reconciler; + +const proxyService = Cc[ + "@mozilla.org/network/protocol-proxy-service;1" +].getService(Ci.nsIProtocolProxyService); + +const proxyFilter = { + proxyInfo: proxyService.newProxyInfo( + "http", + "localhost", + HTTP_PORT, + "", + "", + 0, + 4096, + null + ), + + applyFilter(channel, defaultProxyInfo, callback) { + if (channel.URI.host === "example.com") { + callback.onProxyFilterResult(this.proxyInfo); + } else { + callback.onProxyFilterResult(defaultProxyInfo); + } + }, +}; + +proxyService.registerChannelFilter(proxyFilter, 0); +registerCleanupFunction(() => { + proxyService.unregisterChannelFilter(proxyFilter); +}); + +/** + * Create a AddonsRec for this application with the fields specified. + * + * @param id Sync GUID of record + * @param addonId ID of add-on + * @param enabled Boolean whether record is enabled + * @param deleted Boolean whether record was deleted + */ +function createRecordForThisApp(id, addonId, enabled, deleted) { + return { + id, + addonID: addonId, + enabled, + deleted: !!deleted, + applicationID: Services.appinfo.ID, + source: "amo", + }; +} + +function createAndStartHTTPServer(port) { + try { + let server = new HttpServer(); + + server.registerPathHandler( + "/search/guid:addon1%40tests.mozilla.org", + (req, resp) => { + resp.setHeader("Content-type", "application/json", true); + resp.write(JSON.stringify(SEARCH_RESULT)); + } + ); + server.registerPathHandler( + "/search/guid:missing-xpi%40tests.mozilla.org", + (req, resp) => { + resp.setHeader("Content-type", "application/json", true); + resp.write(JSON.stringify(MISSING_SEARCH_RESULT)); + } + ); + server.registerFile("/addon1.xpi", XPIS.test_addon1); + + server.start(port); + + return server; + } catch (ex) { + _("Got exception starting HTTP server on port " + port); + _("Error: " + Log.exceptionStr(ex)); + do_throw(ex); + } + return null; /* not hit, but keeps eslint happy! */ +} + +// A helper function to ensure that the reconciler's current view of the addon +// is the same as the addon itself. If it's not, then the reconciler missed a +// change, and is likely to re-upload the addon next sync because of the change +// it missed. +async function checkReconcilerUpToDate(addon) { + let stateBefore = Object.assign({}, store.reconciler.addons[addon.id]); + await store.reconciler.rectifyStateFromAddon(addon); + let stateAfter = store.reconciler.addons[addon.id]; + deepEqual(stateBefore, stateAfter); +} + +add_task(async function setup() { + await Service.engineManager.register(AddonsEngine); + engine = Service.engineManager.get("addons"); + store = engine._store; + reconciler = engine._reconciler; + + reconciler.startListening(); + + // Don't flush to disk in the middle of an event listener! + // This causes test hangs on WinXP. + reconciler._shouldPersist = false; +}); + +add_task(async function test_remove() { + _("Ensure removing add-ons from deleted records works."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + let record = createRecordForThisApp(addon.syncGUID, ID1, true, true); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(null, countTelemetry.failedReasons); + Assert.equal(0, countTelemetry.incomingCounts.failed); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, newAddon); +}); + +add_task(async function test_apply_enabled() { + let countTelemetry = new SyncedRecordsTelemetry(); + _("Ensures that changes to the userEnabled flag apply."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(addon.isActive); + Assert.ok(!addon.userDisabled); + + _("Ensure application of a disable record works as expected."); + let records = []; + records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false)); + + let [failed] = await Promise.all([ + store.applyIncomingBatch(records, countTelemetry), + AddonTestUtils.promiseAddonEvent("onDisabled"), + ]); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enable record works as expected."); + records.push(createRecordForThisApp(addon.syncGUID, ID1, true, false)); + [failed] = await Promise.all([ + store.applyIncomingBatch(records, countTelemetry), + AddonTestUtils.promiseWebExtensionStartup(ID1), + ]); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(!addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enabled state updates don't apply if the ignore pref is set."); + records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false)); + Svc.PrefBranch.setBoolPref("addons.ignoreUserEnabledChanges", true); + failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(!addon.userDisabled); + records = []; + + await uninstallAddon(addon, reconciler); + Svc.PrefBranch.clearUserPref("addons.ignoreUserEnabledChanges"); +}); + +add_task(async function test_apply_enabled_appDisabled() { + _( + "Ensures that changes to the userEnabled flag apply when the addon is appDisabled." + ); + + // this addon is appDisabled by default. + let addon = await installAddon(XPIS.test_addon3); + Assert.ok(addon.appDisabled); + Assert.ok(!addon.isActive); + Assert.ok(!addon.userDisabled); + + _("Ensure application of a disable record works as expected."); + store.reconciler.pruneChangesBeforeDate(Date.now() + 10); + store.reconciler._changes = []; + let records = []; + let countTelemetry = new SyncedRecordsTelemetry(); + records.push(createRecordForThisApp(addon.syncGUID, ID3, false, false)); + let failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID3); + Assert.ok(addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enable record works as expected."); + records.push(createRecordForThisApp(addon.syncGUID, ID3, true, false)); + failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID3); + Assert.ok(!addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_ignore_different_appid() { + _( + "Ensure that incoming records with a different application ID are ignored." + ); + + // We test by creating a record that should result in an update. + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(!addon.userDisabled); + + let record = createRecordForThisApp(addon.syncGUID, ID1, false, false); + record.applicationID = "FAKE_ID"; + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_ignore_unknown_source() { + _("Ensure incoming records with unknown source are ignored."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let record = createRecordForThisApp(addon.syncGUID, ID1, false, false); + record.source = "DUMMY_SOURCE"; + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_apply_uninstall() { + _("Ensures that uninstalling an add-on from a record works."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let records = []; + let countTelemetry = new SyncedRecordsTelemetry(); + records.push(createRecordForThisApp(addon.syncGUID, ID1, true, true)); + let failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + + addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); +}); + +add_task(async function test_addon_syncability() { + _("Ensure isAddonSyncable functions properly."); + + Svc.PrefBranch.setStringPref( + "addons.trustedSourceHostnames", + "addons.mozilla.org,other.example.com" + ); + + Assert.ok(!(await store.isAddonSyncable(null))); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(await store.isAddonSyncable(addon)); + + let dummy = {}; + const KEYS = [ + "id", + "syncGUID", + "type", + "scope", + "foreignInstall", + "isSyncable", + ]; + for (let k of KEYS) { + dummy[k] = addon[k]; + } + + Assert.ok(await store.isAddonSyncable(dummy)); + + dummy.type = "UNSUPPORTED"; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.type = addon.type; + + dummy.scope = 0; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.scope = addon.scope; + + dummy.isSyncable = false; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.isSyncable = addon.isSyncable; + + dummy.foreignInstall = true; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.foreignInstall = false; + + await uninstallAddon(addon, reconciler); + + Assert.ok(!store.isSourceURITrusted(null)); + + let trusted = [ + "https://addons.mozilla.org/foo", + "https://other.example.com/foo", + ]; + + let untrusted = [ + "http://addons.mozilla.org/foo", // non-https + "ftps://addons.mozilla.org/foo", // non-https + "https://untrusted.example.com/foo", // non-trusted hostname` + ]; + + for (let uri of trusted) { + Assert.ok(store.isSourceURITrusted(Services.io.newURI(uri))); + } + + for (let uri of untrusted) { + Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri))); + } + + Svc.PrefBranch.setStringPref("addons.trustedSourceHostnames", ""); + for (let uri of trusted) { + Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri))); + } + + Svc.PrefBranch.setStringPref( + "addons.trustedSourceHostnames", + "addons.mozilla.org" + ); + Assert.ok( + store.isSourceURITrusted( + Services.io.newURI("https://addons.mozilla.org/foo") + ) + ); + + Svc.PrefBranch.clearUserPref("addons.trustedSourceHostnames"); +}); + +add_task(async function test_get_all_ids() { + _("Ensures that getAllIDs() returns an appropriate set."); + + _("Installing two addons."); + // XXX - this test seems broken - at this point, before we've installed the + // addons below, store.getAllIDs() returns all addons installed by previous + // tests, even though those tests uninstalled the addon. + // So if any tests above ever add a new addon ID, they are going to need to + // be added here too. + // Assert.equal(0, Object.keys(store.getAllIDs()).length); + let addon1 = await installAddon(XPIS.test_addon1, reconciler); + let addon2 = await installAddon(XPIS.test_addon2, reconciler); + let addon3 = await installAddon(XPIS.test_addon3, reconciler); + + _("Ensure they're syncable."); + Assert.ok(await store.isAddonSyncable(addon1)); + Assert.ok(await store.isAddonSyncable(addon2)); + Assert.ok(await store.isAddonSyncable(addon3)); + + let ids = await store.getAllIDs(); + + Assert.equal("object", typeof ids); + Assert.equal(3, Object.keys(ids).length); + Assert.ok(addon1.syncGUID in ids); + Assert.ok(addon2.syncGUID in ids); + Assert.ok(addon3.syncGUID in ids); + + await uninstallAddon(addon1, reconciler); + await uninstallAddon(addon2, reconciler); + await uninstallAddon(addon3, reconciler); +}); + +add_task(async function test_change_item_id() { + _("Ensures that changeItemID() works properly."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let oldID = addon.syncGUID; + let newID = Utils.makeGUID(); + + await store.changeItemID(oldID, newID); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.notEqual(null, newAddon); + Assert.equal(newID, newAddon.syncGUID); + + await uninstallAddon(newAddon, reconciler); +}); + +add_task(async function test_create() { + _("Ensure creating/installing an add-on from a record works."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, ID1, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.notEqual(null, newAddon); + Assert.equal(guid, newAddon.syncGUID); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(newAddon, reconciler); + + await promiseStopServer(server); +}); + +add_task(async function test_create_missing_search() { + _("Ensures that failed add-on searches are handled gracefully."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + // The handler for this ID is not installed, so a search should 404. + const id = "missing@tests.mozilla.org"; + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, id, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(1, failed.length); + Assert.equal(guid, failed[0]); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "GET <URL> failed (status 404)" + ); + Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1); + + let addon = await AddonManager.getAddonByID(id); + Assert.equal(null, addon); + + await promiseStopServer(server); +}); + +add_task(async function test_create_bad_install() { + _("Ensures that add-ons without a valid install are handled gracefully."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + // The handler returns a search result but the XPI will 404. + const id = "missing-xpi@tests.mozilla.org"; + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, id, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + /* let failed = */ await store.applyIncomingBatch([record], countTelemetry); + // This addon had no source URI so was skipped - but it's not treated as + // failure. + // XXX - this test isn't testing what we thought it was. Previously the addon + // was not being installed due to requireSecureURL checking *before* we'd + // attempted to get the XPI. + // With requireSecureURL disabled we do see a download failure, but the addon + // *does* get added to |failed|. + // FTR: onDownloadFailed() is called with ERROR_NETWORK_FAILURE, so it's going + // to be tricky to distinguish a 404 from other transient network errors + // where we do want the addon to end up in |failed|. + // This is being tracked in bug 1284778. + // Assert.equal(0, failed.length); + + let addon = await AddonManager.getAddonByID(id); + Assert.equal(null, addon); + + await promiseStopServer(server); +}); + +add_task(async function test_ignore_system() { + _("Ensure we ignore system addons"); + // Our system addon should not appear in getAllIDs + await engine._refreshReconcilerState(); + let num = 0; + let ids = await store.getAllIDs(); + for (let guid in ids) { + num += 1; + let addon = reconciler.getAddonStateFromSyncGUID(guid); + Assert.notEqual(addon.id, SYSTEM_ADDON_ID); + } + Assert.greater(num, 1, "should have seen at least one."); +}); + +add_task(async function test_incoming_system() { + _("Ensure we handle incoming records that refer to a system addon"); + // eg, loop initially had a normal addon but it was then "promoted" to be a + // system addon but wanted to keep the same ID. The server record exists due + // to this. + + // before we start, ensure the system addon isn't disabled. + Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled)); + + // Now simulate an incoming record with the same ID as the system addon, + // but flagged as disabled - it should not be applied. + let server = createAndStartHTTPServer(HTTP_PORT); + // We make the incoming record flag the system addon as disabled - it should + // be ignored. + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, SYSTEM_ADDON_ID, false, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + // The system addon should still not be userDisabled. + Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled)); + + await promiseStopServer(server); +}); + +add_task(async function test_wipe() { + _("Ensures that wiping causes add-ons to be uninstalled."); + + await installAddon(XPIS.test_addon1, reconciler); + + await store.wipe(); + + let addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); +}); + +add_task(async function test_wipe_and_install() { + _("Ensure wipe followed by install works."); + + // This tests the reset sync flow where remote data is replaced by local. The + // receiving client will see a wipe followed by a record which should undo + // the wipe. + let installed = await installAddon(XPIS.test_addon1, reconciler); + + let record = createRecordForThisApp(installed.syncGUID, ID1, true, false); + + await store.wipe(); + + let deleted = await AddonManager.getAddonByID(ID1); + Assert.equal(null, deleted); + + // Re-applying the record can require re-fetching the XPI. + let server = createAndStartHTTPServer(HTTP_PORT); + + await store.applyIncoming(record); + + let fetched = await AddonManager.getAddonByID(record.addonID); + Assert.ok(!!fetched); + + // wipe again to we are left with a clean slate. + await store.wipe(); + + await promiseStopServer(server); +}); + +// STR for what this is testing: +// * Either: +// * Install then remove an addon, then delete addons.json from the profile +// or corrupt it (in which case the addon manager will remove it) +// * Install then remove an addon while addon caching is disabled, then +// re-enable addon caching. +// * Install the same addon in a different profile, sync it. +// * Sync this profile +// Before bug 1467904, the addon would fail to install because this profile +// has a copy of the addon in our addonsreconciler.json, but the addon manager +// does *not* have a copy in its cache, and repopulating that cache would not +// re-add it as the addon is no longer installed locally. +add_task(async function test_incoming_reconciled_but_not_cached() { + _( + "Ensure we handle incoming records our reconciler has but the addon cache does not" + ); + + // Make sure addon is not installed. + let addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); + + Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", false); + + addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.notEqual(await AddonManager.getAddonByID(ID1), null); + await uninstallAddon(addon, reconciler); + + Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", true); + + // now pretend it is incoming. + let server = createAndStartHTTPServer(HTTP_PORT); + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, ID1, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + Assert.notEqual(await AddonManager.getAddonByID(ID1), null); + + await promiseStopServer(server); +}); + +// NOTE: The test above must be the last test run due to the addon cache +// being trashed. It is probably possible to fix that by running, eg, +// AddonRespository.backgroundUpdateCheck() to rebuild the cache, but that +// requires implementing more AMO functionality in our test server + +add_task(async function cleanup() { + // There's an xpcom-shutdown hook for this, but let's give this a shot. + reconciler.stopListening(); +}); diff --git a/services/sync/tests/unit/test_addons_tracker.js b/services/sync/tests/unit/test_addons_tracker.js new file mode 100644 index 0000000000..f8473e4cfa --- /dev/null +++ b/services/sync/tests/unit/test_addons_tracker.js @@ -0,0 +1,174 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +Services.prefs.setBoolPref("extensions.experiments.enabled", true); + +Svc.PrefBranch.setBoolPref("engine.addons", true); + +let reconciler; +let tracker; + +const addon1ID = "addon1@tests.mozilla.org"; + +const ADDONS = { + test_addon1: { + manifest: { + browser_specific_settings: { gecko: { id: addon1ID } }, + }, + }, +}; + +const XPIS = {}; + +async function cleanup() { + tracker.stop(); + + tracker.resetScore(); + await tracker.clearChangedIDs(); + + reconciler._addons = {}; + reconciler._changes = []; + await reconciler.saveState(); +} + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); + for (let [name, data] of Object.entries(ADDONS)) { + XPIS[name] = AddonTestUtils.createTempWebExtensionFile(data); + } + await Service.engineManager.register(AddonsEngine); + let engine = Service.engineManager.get("addons"); + reconciler = engine._reconciler; + tracker = engine._tracker; + + await cleanup(); +}); + +add_task(async function test_empty() { + _("Verify the tracker is empty to start with."); + + Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length); + Assert.equal(0, tracker.score); + + await cleanup(); +}); + +add_task(async function test_not_tracking() { + _("Ensures the tracker doesn't do anything when it isn't tracking."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + await uninstallAddon(addon, reconciler); + + Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length); + Assert.equal(0, tracker.score); + + await cleanup(); +}); + +add_task(async function test_track_install() { + _("Ensure that installing an add-on notifies tracker."); + + reconciler.startListening(); + + tracker.start(); + + Assert.equal(0, tracker.score); + let addon = await installAddon(XPIS.test_addon1, reconciler); + let changed = await tracker.getChangedIDs(); + + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); + +add_task(async function test_track_uninstall() { + _("Ensure that uninstalling an add-on notifies tracker."); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + let guid = addon.syncGUID; + Assert.equal(0, tracker.score); + + tracker.start(); + + await uninstallAddon(addon, reconciler); + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(guid in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await cleanup(); +}); + +add_task(async function test_track_user_disable() { + _("Ensure that tracker sees disabling of add-on"); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(!addon.userDisabled); + Assert.ok(!addon.appDisabled); + Assert.ok(addon.isActive); + + tracker.start(); + Assert.equal(0, tracker.score); + + _("Disabling add-on"); + await addon.disable(); + await reconciler.queueCaller.promiseCallsComplete(); + + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); + +add_task(async function test_track_enable() { + _("Ensure that enabling a disabled add-on notifies tracker."); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + await addon.disable(); + await Async.promiseYield(); + + Assert.equal(0, tracker.score); + + tracker.start(); + await addon.enable(); + await Async.promiseYield(); + await reconciler.queueCaller.promiseCallsComplete(); + + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); diff --git a/services/sync/tests/unit/test_addons_validator.js b/services/sync/tests/unit/test_addons_validator.js new file mode 100644 index 0000000000..60f2f8bf43 --- /dev/null +++ b/services/sync/tests/unit/test_addons_validator.js @@ -0,0 +1,65 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AddonValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "1111", + applicationID: Services.appinfo.ID, + addonID: "synced-addon@example.com", + enabled: true, + source: "amo", + understood: true, + }, + ], + client: [ + { + syncGUID: "1111", + id: "synced-addon@example.com", + type: "extension", + isSystem: false, + isSyncable: true, + }, + { + syncGUID: "2222", + id: "system-addon@example.com", + type: "extension", + isSystem: true, + isSyncable: false, + }, + { + // Plugins don't have a `syncedGUID`, but we don't sync them, so we + // shouldn't report them as client duplicates. + id: "some-plugin", + type: "plugin", + }, + { + id: "another-plugin", + type: "plugin", + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new AddonValidator({ + _findDupe(item) { + return null; + }, + isAddonSyncable(item) { + return item.type != "plugin"; + }, + }); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 4); + equal(records.length, 1); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); diff --git a/services/sync/tests/unit/test_bookmark_batch_fail.js b/services/sync/tests/unit/test_bookmark_batch_fail.js new file mode 100644 index 0000000000..9644d730e4 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_batch_fail.js @@ -0,0 +1,25 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Making sure a failing sync reports a useful error"); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_bookmark_test(async function run_test(engine) { + await engine.initialize(); + engine._syncStartup = async function () { + throw new Error("FAIL!"); + }; + + try { + _("Try calling the sync that should throw right away"); + await engine._sync(); + do_throw("Should have failed sync!"); + } catch (ex) { + _("Making sure what we threw ended up as the exception:", ex); + Assert.equal(ex.message, "FAIL!"); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_decline_undecline.js b/services/sync/tests/unit/test_bookmark_decline_undecline.js new file mode 100644 index 0000000000..12139dd163 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_decline_undecline.js @@ -0,0 +1,48 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// A stored reference to the collection won't be valid after disabling. +function getBookmarkWBO(server, guid) { + let coll = server.user("foo").collection("bookmarks"); + if (!coll) { + return null; + } + return coll.wbo(guid); +} + +add_task(async function test_decline_undecline() { + let engine = Service.engineManager.get("bookmarks"); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let { guid: bzGuid } = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org", + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + title: "bugzilla", + }); + + ok(!getBookmarkWBO(server, bzGuid), "Shouldn't have been uploaded yet"); + await Service.sync(); + ok(getBookmarkWBO(server, bzGuid), "Should be present on server"); + + engine.enabled = false; + await Service.sync(); + ok( + !getBookmarkWBO(server, bzGuid), + "Shouldn't be present on server anymore" + ); + + engine.enabled = true; + await Service.sync(); + ok(getBookmarkWBO(server, bzGuid), "Should be present on server again"); + } finally { + await PlacesSyncUtils.bookmarks.reset(); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_engine.js b/services/sync/tests/unit/test_bookmark_engine.js new file mode 100644 index 0000000000..6274a6b836 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_engine.js @@ -0,0 +1,1555 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { BookmarkHTMLUtils } = ChromeUtils.importESModule( + "resource://gre/modules/BookmarkHTMLUtils.sys.mjs" +); +const { BookmarkJSONUtils } = ChromeUtils.importESModule( + "resource://gre/modules/BookmarkJSONUtils.sys.mjs" +); +const { Bookmark, BookmarkFolder, BookmarksEngine, Livemark } = + ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +var recordedEvents = []; + +function checkRecordedEvents(object, expected, message) { + // Ignore event telemetry from the merger. + let checkEvents = recordedEvents.filter(event => event.object == object); + deepEqual(checkEvents, expected, message); + // and clear the list so future checks are easier to write. + recordedEvents = []; +} + +async function fetchAllRecordIds() { + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached(` + WITH RECURSIVE + syncedItems(id, guid) AS ( + SELECT b.id, b.guid FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id, b.guid FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ) + SELECT guid FROM syncedItems`); + let recordIds = new Set(); + for (let row of rows) { + let recordId = PlacesSyncUtils.bookmarks.guidToRecordId( + row.getResultByName("guid") + ); + recordIds.add(recordId); + } + return recordIds; +} + +async function cleanupEngine(engine) { + await engine.resetClient(); + await engine._store.wipe(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + // Note we don't finalize the engine here as add_bookmark_test() does. +} + +async function cleanup(engine, server) { + await promiseStopServer(server); + await cleanupEngine(engine); +} + +add_task(async function setup() { + await generateNewKeys(Service.collectionKeys); + await Service.engineManager.unregister("bookmarks"); + + Service.recordTelemetryEvent = (object, method, value, extra = undefined) => { + recordedEvents.push({ object, method, value, extra }); + }; +}); + +add_task(async function test_buffer_timeout() { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let engine = new BookmarksEngine(Service); + engine._newWatchdog = function () { + // Return an already-aborted watchdog, so that we can abort merges + // immediately. + let watchdog = Async.watchdog(); + watchdog.controller.abort(); + return watchdog; + }; + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + + try { + info("Insert local bookmarks"); + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + guid: "bookmarkAAAA", + url: "http://example.com/a", + title: "A", + }, + { + guid: "bookmarkBBBB", + url: "http://example.com/b", + title: "B", + }, + ], + }); + + info("Insert remote bookmarks"); + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentid: "places", + title: "menu", + children: ["bookmarkCCCC", "bookmarkDDDD"], + }) + ); + collection.insert( + "bookmarkCCCC", + encryptPayload({ + id: "bookmarkCCCC", + type: "bookmark", + parentid: "menu", + bmkUri: "http://example.com/c", + title: "C", + }) + ); + collection.insert( + "bookmarkDDDD", + encryptPayload({ + id: "bookmarkDDDD", + type: "bookmark", + parentid: "menu", + bmkUri: "http://example.com/d", + title: "D", + }) + ); + + info("We expect this sync to fail"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex.name == "InterruptedError" + ); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_maintenance_after_failure(engine) { + _("Ensure we try to run maintenance if the engine fails to sync"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let syncStartup = engine._syncStartup; + let syncError = new Error("Something is rotten in the state of Places"); + engine._syncStartup = function () { + throw syncError; + }; + + Services.prefs.clearUserPref("places.database.lastMaintenance"); + + _("Ensure the sync fails and we run maintenance"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "run", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for first maintenance run" + ); + + _("Sync again, but ensure maintenance doesn't run"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [], + "Should not record event if maintenance didn't run" + ); + + _("Fast-forward last maintenance pref; ensure maintenance runs"); + Services.prefs.setIntPref( + "places.database.lastMaintenance", + Date.now() / 1000 - 14400 + ); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "run", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for second maintenance run" + ); + + _("Fix sync failure; ensure we report success after maintenance"); + engine._syncStartup = syncStartup; + await sync_engine_and_validate_telem(engine, false); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "fix", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for successful sync after second maintenance" + ); + + await sync_engine_and_validate_telem(engine, false); + checkRecordedEvents( + "maintenance", + [], + "Should not record maintenance events after successful sync" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_delete_invalid_roots_from_server(engine) { + _("Ensure that we delete the Places and Reading List roots from the server."); + + enableValidationPrefs(); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); + + try { + let placesRecord = await store.createRecord("places"); + collection.insert("places", encryptPayload(placesRecord.cleartext)); + + let listBmk = new Bookmark("bookmarks", Utils.makeGUID()); + listBmk.bmkUri = "https://example.com"; + listBmk.title = "Example reading list entry"; + listBmk.parentName = "Reading List"; + listBmk.parentid = "readinglist"; + collection.insert(listBmk.id, encryptPayload(listBmk.cleartext)); + + let readingList = new BookmarkFolder("bookmarks", "readinglist"); + readingList.title = "Reading List"; + readingList.children = [listBmk.id]; + readingList.parentName = ""; + readingList.parentid = "places"; + collection.insert("readinglist", encryptPayload(readingList.cleartext)); + + // Note that we don't insert a record for the toolbar, so the engine will + // report a parent-child disagreement, since Firefox's `parentid` is + // `toolbar`. + let newBmk = new Bookmark("bookmarks", Utils.makeGUID()); + newBmk.bmkUri = "http://getfirefox.com"; + newBmk.title = "Get Firefox!"; + newBmk.parentName = "Bookmarks Toolbar"; + newBmk.parentid = "toolbar"; + collection.insert(newBmk.id, encryptPayload(newBmk.cleartext)); + + deepEqual( + collection.keys().sort(), + ["places", "readinglist", listBmk.id, newBmk.id].sort(), + "Should store Places root, reading list items, and new bookmark on server" + ); + + let ping = await sync_engine_and_validate_telem(engine, true); + // In a real sync, the engine is named `bookmarks-buffered`. + // However, `sync_engine_and_validate_telem` simulates a sync where + // the engine isn't registered with the engine manager, so the recorder + // doesn't see its `overrideTelemetryName`. + let engineData = ping.engines.find(e => e.name == "bookmarks"); + ok(engineData.validation, "Bookmarks engine should always run validation"); + equal( + engineData.validation.checked, + 6, + "Bookmarks engine should validate all items" + ); + deepEqual( + engineData.validation.problems, + [ + { + name: "parentChildDisagreements", + count: 1, + }, + ], + "Bookmarks engine should report parent-child disagreement" + ); + deepEqual( + engineData.steps.map(step => step.name), + [ + "fetchLocalTree", + "fetchRemoteTree", + "merge", + "apply", + "notifyObservers", + "fetchLocalChangeRecords", + ], + "Bookmarks engine should report all merge steps" + ); + + deepEqual( + collection.keys().sort(), + ["menu", "mobile", "toolbar", "unfiled", newBmk.id].sort(), + "Should remove Places root and reading list items from server; upload local roots" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_processIncoming_error_orderChildren( + engine +) { + _( + "Ensure that _orderChildren() is called even when _processIncoming() throws an error." + ); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + try { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder 1", + }); + + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + let toolbar_record = await store.createRecord("toolbar"); + collection.insert("toolbar", encryptPayload(toolbar_record.cleartext)); + + let bmk1_record = await store.createRecord(bmk1.guid); + collection.insert(bmk1.guid, encryptPayload(bmk1_record.cleartext)); + + let bmk2_record = await store.createRecord(bmk2.guid); + collection.insert(bmk2.guid, encryptPayload(bmk2_record.cleartext)); + + // Create a server record for folder1 where we flip the order of + // the children. + let folder1_record = await store.createRecord(folder1.guid); + let folder1_payload = folder1_record.cleartext; + folder1_payload.children.reverse(); + collection.insert(folder1.guid, encryptPayload(folder1_payload)); + + // Create a bogus record that when synced down will provoke a + // network error which in turn provokes an exception in _processIncoming. + const BOGUS_GUID = "zzzzzzzzzzzz"; + let bogus_record = collection.insert(BOGUS_GUID, "I'm a bogus record!"); + bogus_record.get = function get() { + throw new Error("Sync this!"); + }; + + // Make the 10 minutes old so it will only be synced in the toFetch phase. + bogus_record.modified = new_timestamp() - 60 * 10; + await engine.setLastSync(new_timestamp() - 60); + engine.toFetch = new SerializableSet([BOGUS_GUID]); + + let error; + try { + await sync_engine_and_validate_telem(engine, true); + } catch (ex) { + error = ex; + } + ok(!!error); + + // Verify that the bookmark order has been applied. + folder1_record = await store.createRecord(folder1.guid); + let new_children = folder1_record.children; + Assert.deepEqual( + new_children.sort(), + [folder1_payload.children[0], folder1_payload.children[1]].sort() + ); + + let localChildIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + folder1.guid + ); + Assert.deepEqual(localChildIds.sort(), [bmk2.guid, bmk1.guid].sort()); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_restorePromptsReupload(engine) { + await test_restoreOrImport(engine, { replace: true }); +}); + +add_bookmark_test(async function test_importPromptsReupload(engine) { + await test_restoreOrImport(engine, { replace: false }); +}); + +// Test a JSON restore or HTML import. Use JSON if `replace` is `true`, or +// HTML otherwise. +async function test_restoreOrImport(engine, { replace }) { + let verb = replace ? "restore" : "import"; + let verbing = replace ? "restoring" : "importing"; + let bookmarkUtils = replace ? BookmarkJSONUtils : BookmarkHTMLUtils; + + _(`Ensure that ${verbing} from a backup will reupload all records.`); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + try { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder 1", + }); + + _("Create a single record."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + _(`Get Firefox!: ${bmk1.guid}`); + + let backupFilePath = PathUtils.join( + PathUtils.tempDir, + `t_b_e_${Date.now()}.json` + ); + + _("Make a backup."); + + await bookmarkUtils.exportToFile(backupFilePath); + + _("Create a different record and sync."); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + _(`Get Thunderbird!: ${bmk2.guid}`); + + await PlacesUtils.bookmarks.remove(bmk1.guid); + + let error; + try { + await sync_engine_and_validate_telem(engine, false); + } catch (ex) { + error = ex; + _("Got error: " + Log.exceptionStr(ex)); + } + Assert.ok(!error); + + _( + "Verify that there's only one bookmark on the server, and it's Thunderbird." + ); + // Of course, there's also the Bookmarks Toolbar and Bookmarks Menu... + let wbos = collection.keys(function (id) { + return !["menu", "toolbar", "mobile", "unfiled", folder1.guid].includes( + id + ); + }); + Assert.equal(wbos.length, 1); + Assert.equal(wbos[0], bmk2.guid); + + _(`Now ${verb} from a backup.`); + await bookmarkUtils.importFromFile(backupFilePath, { replace }); + + // If `replace` is `true`, we'll wipe the server on the next sync. + let bookmarksCollection = server.user("foo").collection("bookmarks"); + _("Verify that we didn't wipe the server."); + Assert.ok(!!bookmarksCollection); + + _("Ensure we have the bookmarks we expect locally."); + let recordIds = await fetchAllRecordIds(); + _("GUIDs: " + JSON.stringify([...recordIds])); + + let bookmarkRecordIds = new Map(); + let count = 0; + for (let recordId of recordIds) { + count++; + let info = await PlacesUtils.bookmarks.fetch( + PlacesSyncUtils.bookmarks.recordIdToGuid(recordId) + ); + // Only one bookmark, so _all_ should be Firefox! + if (info.type == PlacesUtils.bookmarks.TYPE_BOOKMARK) { + _(`Found URI ${info.url.href} for record ID ${recordId}`); + bookmarkRecordIds.set(info.url.href, recordId); + } + } + Assert.ok(bookmarkRecordIds.has("http://getfirefox.com/")); + if (!replace) { + Assert.ok(bookmarkRecordIds.has("http://getthunderbird.com/")); + } + + _("Have the correct number of IDs locally, too."); + let expectedResults = [ + "menu", + "toolbar", + "mobile", + "unfiled", + folder1.guid, + bmk1.guid, + ]; + if (!replace) { + expectedResults.push("toolbar", folder1.guid, bmk2.guid); + } + Assert.equal(count, expectedResults.length); + + _("Sync again. This'll wipe bookmarks from the server."); + try { + await sync_engine_and_validate_telem(engine, false); + } catch (ex) { + error = ex; + _("Got error: " + Log.exceptionStr(ex)); + } + Assert.ok(!error); + + _("Verify that there's the right bookmarks on the server."); + // Of course, there's also the Bookmarks Toolbar and Bookmarks Menu... + let payloads = server.user("foo").collection("bookmarks").payloads(); + let bookmarkWBOs = payloads.filter(function (wbo) { + return wbo.type == "bookmark"; + }); + + let folderWBOs = payloads.filter(function (wbo) { + return ( + wbo.type == "folder" && + wbo.id != "menu" && + wbo.id != "toolbar" && + wbo.id != "unfiled" && + wbo.id != "mobile" && + wbo.parentid != "menu" + ); + }); + + let expectedFX = { + id: bookmarkRecordIds.get("http://getfirefox.com/"), + bmkUri: "http://getfirefox.com/", + title: "Get Firefox!", + }; + let expectedTB = { + id: bookmarkRecordIds.get("http://getthunderbird.com/"), + bmkUri: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }; + + let expectedBookmarks; + if (replace) { + expectedBookmarks = [expectedFX]; + } else { + expectedBookmarks = [expectedTB, expectedFX]; + } + + doCheckWBOs(bookmarkWBOs, expectedBookmarks); + + _("Our old friend Folder 1 is still in play."); + let expectedFolder1 = { title: "Folder 1" }; + + let expectedFolders; + if (replace) { + expectedFolders = [expectedFolder1]; + } else { + expectedFolders = [expectedFolder1, expectedFolder1]; + } + + doCheckWBOs(folderWBOs, expectedFolders); + } finally { + await cleanup(engine, server); + } +} + +function doCheckWBOs(WBOs, expected) { + Assert.equal(WBOs.length, expected.length); + for (let i = 0; i < expected.length; i++) { + let lhs = WBOs[i]; + let rhs = expected[i]; + if ("id" in rhs) { + Assert.equal(lhs.id, rhs.id); + } + if ("bmkUri" in rhs) { + Assert.equal(lhs.bmkUri, rhs.bmkUri); + } + if ("title" in rhs) { + Assert.equal(lhs.title, rhs.title); + } + } +} + +function FakeRecord(constructor, r) { + this.defaultCleartext = constructor.prototype.defaultCleartext; + constructor.call(this, "bookmarks", r.id); + for (let x in r) { + this[x] = r[x]; + } + // Borrow the constructor's conversion functions. + this.toSyncBookmark = constructor.prototype.toSyncBookmark; + this.cleartextToString = constructor.prototype.cleartextToString; +} + +// Bug 632287. +// (Note that `test_mismatched_folder_types()` in +// toolkit/components/places/tests/sync/test_bookmark_kinds.js is an exact +// copy of this test, so it's fine to remove it as part of bug 1449730) +add_task(async function test_mismatched_types() { + _( + "Ensure that handling a record that changes type causes deletion " + + "then re-adding." + ); + + let oldRecord = { + id: "l1nZZXfB8nC7", + type: "folder", + parentName: "Bookmarks Toolbar", + title: "Innerst i Sneglehode", + description: null, + parentid: "toolbar", + }; + + let newRecord = { + id: "l1nZZXfB8nC7", + type: "livemark", + siteUri: "http://sneglehode.wordpress.com/", + feedUri: "http://sneglehode.wordpress.com/feed/", + parentName: "Bookmarks Toolbar", + title: "Innerst i Sneglehode", + description: null, + children: [ + "HCRq40Rnxhrd", + "YeyWCV1RVsYw", + "GCceVZMhvMbP", + "sYi2hevdArlF", + "vjbZlPlSyGY8", + "UtjUhVyrpeG6", + "rVq8WMG2wfZI", + "Lx0tcy43ZKhZ", + "oT74WwV8_j4P", + "IztsItWVSo3-", + ], + parentid: "toolbar", + }; + + let engine = new BookmarksEngine(Service); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let oldR = new FakeRecord(BookmarkFolder, oldRecord); + let newR = new FakeRecord(Livemark, newRecord); + oldR.parentid = PlacesUtils.bookmarks.toolbarGuid; + newR.parentid = PlacesUtils.bookmarks.toolbarGuid; + + await store.applyIncoming(oldR); + await engine._apply(); + _("Applied old. It's a folder."); + let oldID = await PlacesTestUtils.promiseItemId(oldR.id); + _("Old ID: " + oldID); + let oldInfo = await PlacesUtils.bookmarks.fetch(oldR.id); + Assert.equal(oldInfo.type, PlacesUtils.bookmarks.TYPE_FOLDER); + + await store.applyIncoming(newR); + await engine._apply(); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_misreconciled_root(engine) { + _("Ensure that we don't reconcile an arbitrary record with a root."); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + // Log real hard for this test. + store._log.trace = store._log.debug; + engine._log.trace = engine._log.debug; + + await engine._syncStartup(); + + // Let's find out where the toolbar is right now. + let toolbarBefore = await store.createRecord("toolbar", "bookmarks"); + let toolbarIDBefore = await PlacesTestUtils.promiseItemId( + PlacesUtils.bookmarks.toolbarGuid + ); + Assert.notEqual(-1, toolbarIDBefore); + + let parentRecordIDBefore = toolbarBefore.parentid; + let parentGUIDBefore = + PlacesSyncUtils.bookmarks.recordIdToGuid(parentRecordIDBefore); + let parentIDBefore = await PlacesTestUtils.promiseItemId(parentGUIDBefore); + Assert.equal("string", typeof parentGUIDBefore); + + _("Current parent: " + parentGUIDBefore + " (" + parentIDBefore + ")."); + + let to_apply = { + id: "zzzzzzzzzzzz", + type: "folder", + title: "Bookmarks Toolbar", + description: "Now you're for it.", + parentName: "", + parentid: "mobile", // Why not? + children: [], + }; + + let rec = new FakeRecord(BookmarkFolder, to_apply); + + _("Applying record."); + let countTelemetry = new SyncedRecordsTelemetry(); + await store.applyIncomingBatch([rec], countTelemetry); + + // Ensure that afterwards, toolbar is still there. + // As of 2012-12-05, this only passes because Places doesn't use "toolbar" as + // the real GUID, instead using a generated one. Sync does the translation. + let toolbarAfter = await store.createRecord("toolbar", "bookmarks"); + let parentRecordIDAfter = toolbarAfter.parentid; + let parentGUIDAfter = + PlacesSyncUtils.bookmarks.recordIdToGuid(parentRecordIDAfter); + let parentIDAfter = await PlacesTestUtils.promiseItemId(parentGUIDAfter); + Assert.equal( + await PlacesTestUtils.promiseItemGuid(toolbarIDBefore), + PlacesUtils.bookmarks.toolbarGuid + ); + Assert.equal(parentGUIDBefore, parentGUIDAfter); + Assert.equal(parentIDBefore, parentIDAfter); + + await cleanup(engine, server); +}); + +add_bookmark_test(async function test_invalid_url(engine) { + _("Ensure an incoming invalid bookmark URL causes an outgoing tombstone."); + + let server = await serverForFoo(engine); + let collection = server.user("foo").collection("bookmarks"); + + await SyncTestingInfrastructure(server); + await engine._syncStartup(); + + // check the URL really is invalid. + let url = "https://www.42registry.42/"; + Assert.throws(() => Services.io.newURI(url), /invalid/); + + let guid = "abcdefabcdef"; + + let toolbar = new BookmarkFolder("bookmarks", "toolbar"); + toolbar.title = "toolbar"; + toolbar.parentName = ""; + toolbar.parentid = "places"; + toolbar.children = [guid]; + collection.insert("toolbar", encryptPayload(toolbar.cleartext)); + + let item1 = new Bookmark("bookmarks", guid); + item1.bmkUri = "https://www.42registry.42/"; + item1.title = "invalid url"; + item1.parentName = "Bookmarks Toolbar"; + item1.parentid = "toolbar"; + item1.dateAdded = 1234; + collection.insert(guid, encryptPayload(item1.cleartext)); + + _("syncing."); + await sync_engine_and_validate_telem(engine, false); + + // We should find the record now exists on the server as a tombstone. + let updated = collection.cleartext(guid); + Assert.ok(updated.deleted, "record was deleted"); + + let local = await PlacesUtils.bookmarks.fetch(guid); + Assert.deepEqual(local, null, "no local bookmark exists"); + + await cleanup(engine, server); +}); + +add_bookmark_test(async function test_sync_dateAdded(engine) { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + // TODO: Avoid random orange (bug 1374599), this is only necessary + // intermittently - reset the last sync date so that we'll get all bookmarks. + await engine.setLastSync(1); + + engine._tracker.start(); // We skip usual startup... + + // Just matters that it's in the past, not how far. + let now = Date.now(); + let oneYearMS = 365 * 24 * 60 * 60 * 1000; + + try { + let toolbar = new BookmarkFolder("bookmarks", "toolbar"); + toolbar.title = "toolbar"; + toolbar.parentName = ""; + toolbar.parentid = "places"; + toolbar.children = [ + "abcdefabcdef", + "aaaaaaaaaaaa", + "bbbbbbbbbbbb", + "cccccccccccc", + "dddddddddddd", + "eeeeeeeeeeee", + ]; + collection.insert("toolbar", encryptPayload(toolbar.cleartext)); + + let item1GUID = "abcdefabcdef"; + let item1 = new Bookmark("bookmarks", item1GUID); + item1.bmkUri = "https://example.com"; + item1.title = "asdf"; + item1.parentName = "Bookmarks Toolbar"; + item1.parentid = "toolbar"; + item1.dateAdded = now - oneYearMS; + collection.insert(item1GUID, encryptPayload(item1.cleartext)); + + let item2GUID = "aaaaaaaaaaaa"; + let item2 = new Bookmark("bookmarks", item2GUID); + item2.bmkUri = "https://example.com/2"; + item2.title = "asdf2"; + item2.parentName = "Bookmarks Toolbar"; + item2.parentid = "toolbar"; + item2.dateAdded = now + oneYearMS; + const item2LastModified = now / 1000 - 100; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + item2LastModified + ); + + let item3GUID = "bbbbbbbbbbbb"; + let item3 = new Bookmark("bookmarks", item3GUID); + item3.bmkUri = "https://example.com/3"; + item3.title = "asdf3"; + item3.parentName = "Bookmarks Toolbar"; + item3.parentid = "toolbar"; + // no dateAdded + collection.insert(item3GUID, encryptPayload(item3.cleartext)); + + let item4GUID = "cccccccccccc"; + let item4 = new Bookmark("bookmarks", item4GUID); + item4.bmkUri = "https://example.com/4"; + item4.title = "asdf4"; + item4.parentName = "Bookmarks Toolbar"; + item4.parentid = "toolbar"; + // no dateAdded, but lastModified in past + const item4LastModified = (now - oneYearMS) / 1000; + collection.insert( + item4GUID, + encryptPayload(item4.cleartext), + item4LastModified + ); + + let item5GUID = "dddddddddddd"; + let item5 = new Bookmark("bookmarks", item5GUID); + item5.bmkUri = "https://example.com/5"; + item5.title = "asdf5"; + item5.parentName = "Bookmarks Toolbar"; + item5.parentid = "toolbar"; + // no dateAdded, lastModified in (near) future. + const item5LastModified = (now + 60000) / 1000; + collection.insert( + item5GUID, + encryptPayload(item5.cleartext), + item5LastModified + ); + + let item6GUID = "eeeeeeeeeeee"; + let item6 = new Bookmark("bookmarks", item6GUID); + item6.bmkUri = "https://example.com/6"; + item6.title = "asdf6"; + item6.parentName = "Bookmarks Toolbar"; + item6.parentid = "toolbar"; + const item6LastModified = (now - oneYearMS) / 1000; + collection.insert( + item6GUID, + encryptPayload(item6.cleartext), + item6LastModified + ); + + await sync_engine_and_validate_telem(engine, false); + + let record1 = await store.createRecord(item1GUID); + let record2 = await store.createRecord(item2GUID); + + equal( + item1.dateAdded, + record1.dateAdded, + "dateAdded in past should be synced" + ); + equal( + record2.dateAdded, + item2LastModified * 1000, + "dateAdded in future should be ignored in favor of last modified" + ); + + let record3 = await store.createRecord(item3GUID); + + ok(record3.dateAdded); + // Make sure it's within 24 hours of the right timestamp... This is a little + // dodgey but we only really care that it's basically accurate and has the + // right day. + ok(Math.abs(Date.now() - record3.dateAdded) < 24 * 60 * 60 * 1000); + + let record4 = await store.createRecord(item4GUID); + equal( + record4.dateAdded, + item4LastModified * 1000, + "If no dateAdded is provided, lastModified should be used" + ); + + let record5 = await store.createRecord(item5GUID); + equal( + record5.dateAdded, + item5LastModified * 1000, + "If no dateAdded is provided, lastModified should be used (even if it's in the future)" + ); + + // Update item2 and try resyncing it. + item2.dateAdded = now - 100000; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + now / 1000 - 50 + ); + + // Also, add a local bookmark and make sure its date added makes it up to the server + let bz = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }); + + // last sync did a POST, which doesn't advance its lastModified value. + // Next sync of the engine doesn't hit info/collections, so lastModified + // remains stale. Setting it to null side-steps that. + engine.lastModified = null; + await sync_engine_and_validate_telem(engine, false); + + let newRecord2 = await store.createRecord(item2GUID); + equal( + newRecord2.dateAdded, + item2.dateAdded, + "dateAdded update should work for earlier date" + ); + + let bzWBO = collection.cleartext(bz.guid); + ok(bzWBO.dateAdded, "Locally added dateAdded lost"); + + let localRecord = await store.createRecord(bz.guid); + equal( + bzWBO.dateAdded, + localRecord.dateAdded, + "dateAdded should not change during upload" + ); + + item2.dateAdded += 10000; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + now / 1000 - 10 + ); + + engine.lastModified = null; + await sync_engine_and_validate_telem(engine, false); + + let newerRecord2 = await store.createRecord(item2GUID); + equal( + newerRecord2.dateAdded, + newRecord2.dateAdded, + "dateAdded update should be ignored for later date if we know an earlier one " + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_buffer_hasDupe() { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + engine._tracker.start(); // We skip usual startup... + try { + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + await PlacesUtils.bookmarks.insert({ + guid: guid1, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "https://www.example.com", + title: "example.com", + }); + await PlacesUtils.bookmarks.insert({ + guid: guid2, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "https://www.example.com", + title: "example.com", + }); + + await sync_engine_and_validate_telem(engine, false); + // Make sure we set hasDupe on outgoing records + Assert.ok(collection.payloads().every(payload => payload.hasDupe)); + + await PlacesUtils.bookmarks.remove(guid1); + + await sync_engine_and_validate_telem(engine, false); + + let tombstone = JSON.parse( + JSON.parse(collection.payload(guid1)).ciphertext + ); + // We shouldn't set hasDupe on tombstones. + Assert.ok(tombstone.deleted); + Assert.ok(!tombstone.hasDupe); + + let record = JSON.parse(JSON.parse(collection.payload(guid2)).ciphertext); + // We should set hasDupe on weakly uploaded records. + Assert.ok(!record.deleted); + Assert.ok( + record.hasDupe, + "Bookmarks bookmark engine should set hasDupe for weakly uploaded records." + ); + + await sync_engine_and_validate_telem(engine, false); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +// Bug 890217. +add_bookmark_test(async function test_sync_imap_URLs(engine) { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + try { + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentid: "places", + title: "Bookmarks Menu", + children: ["bookmarkAAAA"], + }) + ); + collection.insert( + "bookmarkAAAA", + encryptPayload({ + id: "bookmarkAAAA", + type: "bookmark", + parentid: "menu", + bmkUri: + "imap://vs@eleven.vs.solnicky.cz:993/fetch%3EUID%3E/" + + "INBOX%3E56291?part=1.2&type=image/jpeg&filename=" + + "invalidazPrahy.jpg", + title: + "invalidazPrahy.jpg (JPEG Image, 1280x1024 pixels) - Scaled (71%)", + }) + ); + + await PlacesUtils.bookmarks.insert({ + guid: "bookmarkBBBB", + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: + "imap://eleven.vs.solnicky.cz:993/fetch%3EUID%3E/" + + "CURRENT%3E2433?part=1.2&type=text/html&filename=TomEdwards.html", + title: "TomEdwards.html", + }); + + await sync_engine_and_validate_telem(engine, false); + + let aInfo = await PlacesUtils.bookmarks.fetch("bookmarkAAAA"); + equal( + aInfo.url.href, + "imap://vs@eleven.vs.solnicky.cz:993/" + + "fetch%3EUID%3E/INBOX%3E56291?part=1.2&type=image/jpeg&filename=" + + "invalidazPrahy.jpg", + "Remote bookmark A with IMAP URL should exist locally" + ); + + let bPayload = collection.cleartext("bookmarkBBBB"); + equal( + bPayload.bmkUri, + "imap://eleven.vs.solnicky.cz:993/" + + "fetch%3EUID%3E/CURRENT%3E2433?part=1.2&type=text/html&filename=" + + "TomEdwards.html", + "Local bookmark B with IMAP URL should exist remotely" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_resume_buffer() { + await Service.recordManager.clearCache(); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + await engine._store.wipe(); + await engine.resetClient(); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + const batchChunkSize = 50; + + engine._store._batchChunkSize = batchChunkSize; + try { + let children = []; + + let timestamp = round_timestamp(Date.now()); + // Add two chunks worth of records to the server + for (let i = 0; i < batchChunkSize * 2; ++i) { + let cleartext = { + id: Utils.makeGUID(), + type: "bookmark", + parentid: "toolbar", + title: `Bookmark ${i}`, + parentName: "Bookmarks Toolbar", + bmkUri: `https://example.com/${i}`, + }; + let wbo = collection.insert( + cleartext.id, + encryptPayload(cleartext), + timestamp + 10 * i + ); + // Something that is effectively random, but deterministic. + // (This is just to ensure we don't accidentally start using the + // sortindex again). + wbo.sortindex = 1000 + Math.round(Math.sin(i / 5) * 100); + children.push(cleartext.id); + } + + // Add the parent of those records, and ensure its timestamp is the most recent. + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + parentid: "places", + title: "Bookmarks Toolbar", + children, + }), + timestamp + 10 * children.length + ); + + // Replace applyIncomingBatch with a custom one that calls the original, + // but forces it to throw on the 2nd chunk. + let origApplyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = function (records) { + if (records.length > batchChunkSize) { + // Hacky way to make reading from the batchChunkSize'th record throw. + delete records[batchChunkSize]; + Object.defineProperty(records, batchChunkSize, { + get() { + throw new Error("D:"); + }, + }); + } + return origApplyIncomingBatch.call(this, records); + }; + + let caughtError; + _("We expect this to fail"); + try { + await sync_engine_and_validate_telem(engine, true); + } catch (e) { + caughtError = e; + } + Assert.ok(caughtError, "Expected engine.sync to throw"); + Assert.equal(caughtError.message, "D:"); + + // The buffer subtracts one second from the actual timestamp. + let lastSync = (await engine.getLastSync()) + 1; + // We poisoned the batchChunkSize'th record, so the last successfully + // applied record will be batchChunkSize - 1. + let expectedLastSync = timestamp + 10 * (batchChunkSize - 1); + Assert.equal(expectedLastSync, lastSync); + + engine._store.applyIncomingBatch = origApplyIncomingBatch; + + await sync_engine_and_validate_telem(engine, false); + + // Check that all the children made it onto the correct record. + let toolbarRecord = await engine._store.createRecord("toolbar"); + Assert.deepEqual(toolbarRecord.children.sort(), children.sort()); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_livemarks(engine) { + _("Ensure we replace new and existing livemarks with tombstones"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + let now = Date.now(); + + try { + _("Insert existing livemark"); + let modifiedForA = now - 5 * 60 * 1000; + await PlacesUtils.bookmarks.insert({ + guid: "livemarkAAAA", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "A", + lastModified: new Date(modifiedForA), + dateAdded: new Date(modifiedForA), + source: PlacesUtils.bookmarks.SOURCE_SYNC, + }); + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentName: "", + title: "menu", + children: ["livemarkAAAA"], + parentid: "places", + }), + round_timestamp(modifiedForA) + ); + collection.insert( + "livemarkAAAA", + encryptPayload({ + id: "livemarkAAAA", + type: "livemark", + feedUri: "http://example.com/a", + parentName: "menu", + title: "A", + parentid: "menu", + }), + round_timestamp(modifiedForA) + ); + + _("Insert remotely updated livemark"); + await PlacesUtils.bookmarks.insert({ + guid: "livemarkBBBB", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "B", + lastModified: new Date(now), + dateAdded: new Date(now), + }); + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + parentName: "", + title: "toolbar", + children: ["livemarkBBBB"], + parentid: "places", + }), + round_timestamp(now) + ); + collection.insert( + "livemarkBBBB", + encryptPayload({ + id: "livemarkBBBB", + type: "livemark", + feedUri: "http://example.com/b", + parentName: "toolbar", + title: "B", + parentid: "toolbar", + }), + round_timestamp(now) + ); + + _("Insert new remote livemark"); + collection.insert( + "unfiled", + encryptPayload({ + id: "unfiled", + type: "folder", + parentName: "", + title: "unfiled", + children: ["livemarkCCCC"], + parentid: "places", + }), + round_timestamp(now) + ); + collection.insert( + "livemarkCCCC", + encryptPayload({ + id: "livemarkCCCC", + type: "livemark", + feedUri: "http://example.com/c", + parentName: "unfiled", + title: "C", + parentid: "unfiled", + }), + round_timestamp(now) + ); + + _("Bump last sync time to ignore A"); + await engine.setLastSync(round_timestamp(now) - 60); + + _("Sync"); + await sync_engine_and_validate_telem(engine, false); + + deepEqual( + collection.keys().sort(), + [ + "livemarkAAAA", + "livemarkBBBB", + "livemarkCCCC", + "menu", + "mobile", + "toolbar", + "unfiled", + ], + "Should store original livemark A and tombstones for B and C on server" + ); + + let payloads = collection.payloads(); + + deepEqual( + payloads.find(payload => payload.id == "menu").children, + ["livemarkAAAA"], + "Should keep A in menu" + ); + ok( + !payloads.find(payload => payload.id == "livemarkAAAA").deleted, + "Should not upload tombstone for A" + ); + + deepEqual( + payloads.find(payload => payload.id == "toolbar").children, + [], + "Should remove B from toolbar" + ); + ok( + payloads.find(payload => payload.id == "livemarkBBBB").deleted, + "Should upload tombstone for B" + ); + + deepEqual( + payloads.find(payload => payload.id == "unfiled").children, + [], + "Should remove C from unfiled" + ); + ok( + payloads.find(payload => payload.id == "livemarkCCCC").deleted, + "Should replace C with tombstone" + ); + + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + children: [ + { + guid: "livemarkAAAA", + index: 0, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "Should keep A and remove B locally" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_unknown_fields(engine) { + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + try { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder 1", + }); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + let toolbar_record = await store.createRecord("toolbar"); + collection.insert("toolbar", encryptPayload(toolbar_record.cleartext)); + + let folder1_record_without_unknown_fields = await store.createRecord( + folder1.guid + ); + collection.insert( + folder1.guid, + encryptPayload(folder1_record_without_unknown_fields.cleartext) + ); + + // First bookmark record has an unknown string field + let bmk1_record = await store.createRecord(bmk1.guid); + console.log("bmk1_record: ", bmk1_record); + bmk1_record.cleartext.unknownStrField = + "an unknown field from another client"; + collection.insert(bmk1.guid, encryptPayload(bmk1_record.cleartext)); + + // Second bookmark record as an unknown object field + let bmk2_record = await store.createRecord(bmk2.guid); + bmk2_record.cleartext.unknownObjField = { + name: "an unknown object from another client", + }; + collection.insert(bmk2.guid, encryptPayload(bmk2_record.cleartext)); + + // Sync the two bookmarks + await sync_engine_and_validate_telem(engine, true); + + // Add a folder could also have an unknown field + let folder1_record = await store.createRecord(folder1.guid); + folder1_record.cleartext.unknownStrField = + "a folder could also have an unknown field!"; + collection.insert(folder1.guid, encryptPayload(folder1_record.cleartext)); + + // sync the new updates + await engine.setLastSync(1); + await sync_engine_and_validate_telem(engine, true); + + let payloads = collection.payloads(); + // Validate the server has the unknown fields at the top level (and now unknownFields) + let server_bmk1 = payloads.find(payload => payload.id == bmk1.guid); + deepEqual( + server_bmk1.unknownStrField, + "an unknown field from another client", + "unknown fields correctly on the record" + ); + Assert.equal(server_bmk1.unknownFields, null); + + // Check that the mirror table has unknown fields + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT guid, title, unknownFields from items WHERE guid IN + (:bmk1, :bmk2, :folder1)`, + { bmk1: bmk1.guid, bmk2: bmk2.guid, folder1: folder1.guid } + ); + // We should have 3 rows that came from the server + Assert.equal(rows.length, 3); + + // Bookmark 1 - unknown string field + let remote_bmk1 = rows.find( + row => row.getResultByName("guid") == bmk1.guid + ); + Assert.equal(remote_bmk1.getResultByName("title"), "Get Firefox!"); + deepEqual(JSON.parse(remote_bmk1.getResultByName("unknownFields")), { + unknownStrField: "an unknown field from another client", + }); + + // Bookmark 2 - unknown object field + let remote_bmk2 = rows.find( + row => row.getResultByName("guid") == bmk2.guid + ); + Assert.equal(remote_bmk2.getResultByName("title"), "Get Thunderbird!"); + deepEqual(JSON.parse(remote_bmk2.getResultByName("unknownFields")), { + unknownObjField: { + name: "an unknown object from another client", + }, + }); + + // Folder with unknown field + + // check the server still has the unknown field + deepEqual( + payloads.find(payload => payload.id == folder1.guid).unknownStrField, + "a folder could also have an unknown field!", + "Server still has the unknown field" + ); + + let remote_folder = rows.find( + row => row.getResultByName("guid") == folder1.guid + ); + Assert.equal(remote_folder.getResultByName("title"), "Folder 1"); + deepEqual(JSON.parse(remote_folder.getResultByName("unknownFields")), { + unknownStrField: "a folder could also have an unknown field!", + }); + } finally { + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_order.js b/services/sync/tests/unit/test_bookmark_order.js new file mode 100644 index 0000000000..fc182b81ef --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_order.js @@ -0,0 +1,586 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Making sure after processing incoming bookmarks, they show up in the right order" +); +const { Bookmark, BookmarkFolder } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function serverForFoo(engine) { + await generateNewKeys(Service.collectionKeys); + + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let engineSyncID = await engine.resetLocalSyncID(); + return serverForUsers( + { foo: "password" }, + { + meta: { + global: { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { + version: clientsEngine.version, + syncID: clientsSyncID, + }, + [engine.name]: { + version: engine.version, + syncID: engineSyncID, + }, + }, + }, + }, + crypto: { + keys: encryptPayload({ + id: "keys", + // Generate a fake default key bundle to avoid resetting the client + // before the first sync. + default: [ + await Weave.Crypto.generateRandomKey(), + await Weave.Crypto.generateRandomKey(), + ], + }), + }, + [engine.name]: {}, + } + ); +} + +async function resolveConflict( + engine, + collection, + timestamp, + buildTree, + message +) { + let guids = { + // These items don't exist on the server. + fx: Utils.makeGUID(), + nightly: Utils.makeGUID(), + support: Utils.makeGUID(), + customize: Utils.makeGUID(), + + // These exist on the server, but in a different order, and `res` + // has completely different children. + res: Utils.makeGUID(), + tb: Utils.makeGUID(), + + // These don't exist locally. + bz: Utils.makeGUID(), + irc: Utils.makeGUID(), + mdn: Utils.makeGUID(), + }; + + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.menuGuid, + children: [ + { + guid: guids.fx, + title: "Get Firefox!", + url: "http://getfirefox.com/", + }, + { + guid: guids.res, + title: "Resources", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children: [ + { + guid: guids.nightly, + title: "Nightly", + url: "https://nightly.mozilla.org/", + }, + { + guid: guids.support, + title: "Support", + url: "https://support.mozilla.org/", + }, + { + guid: guids.customize, + title: "Customize", + url: "https://mozilla.org/firefox/customize/", + }, + ], + }, + { + title: "Get Thunderbird!", + guid: guids.tb, + url: "http://getthunderbird.com/", + }, + ], + }); + + let serverRecords = [ + { + id: "menu", + type: "folder", + title: "Bookmarks Menu", + parentid: "places", + children: [guids.tb, guids.res], + }, + { + id: guids.tb, + type: "bookmark", + parentid: "menu", + bmkUri: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }, + { + id: guids.res, + type: "folder", + parentid: "menu", + title: "Resources", + children: [guids.irc, guids.bz, guids.mdn], + }, + { + id: guids.bz, + type: "bookmark", + parentid: guids.res, + bmkUri: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }, + { + id: guids.mdn, + type: "bookmark", + parentid: guids.res, + bmkUri: "https://developer.mozilla.org/", + title: "MDN", + }, + { + id: guids.irc, + type: "bookmark", + parentid: guids.res, + bmkUri: "ircs://irc.mozilla.org/nightly", + title: "IRC", + }, + ]; + for (let record of serverRecords) { + collection.insert(record.id, encryptPayload(record), timestamp); + } + + engine.lastModified = collection.timestamp; + await sync_engine_and_validate_telem(engine, false); + + let expectedTree = buildTree(guids); + await assertBookmarksTreeMatches( + PlacesUtils.bookmarks.menuGuid, + expectedTree, + message + ); +} + +async function get_engine() { + return Service.engineManager.get("bookmarks"); +} + +add_task(async function test_local_order_newer() { + let engine = await get_engine(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let collection = server.user("foo").collection("bookmarks"); + let serverModified = Date.now() / 1000 - 120; + await resolveConflict( + engine, + collection, + serverModified, + guids => [ + { + guid: guids.fx, + index: 0, + }, + { + guid: guids.res, + index: 1, + children: [ + { + guid: guids.nightly, + index: 0, + }, + { + guid: guids.support, + index: 1, + }, + { + guid: guids.customize, + index: 2, + }, + { + guid: guids.irc, + index: 3, + }, + { + guid: guids.bz, + index: 4, + }, + { + guid: guids.mdn, + index: 5, + }, + ], + }, + { + guid: guids.tb, + index: 2, + }, + ], + "Should use local order as base if remote is older" + ); + } finally { + await engine.wipeClient(); + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_remote_order_newer() { + let engine = await get_engine(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let collection = server.user("foo").collection("bookmarks"); + let serverModified = Date.now() / 1000 + 120; + await resolveConflict( + engine, + collection, + serverModified, + guids => [ + { + guid: guids.tb, + index: 0, + }, + { + guid: guids.res, + index: 1, + children: [ + { + guid: guids.irc, + index: 0, + }, + { + guid: guids.bz, + index: 1, + }, + { + guid: guids.mdn, + index: 2, + }, + { + guid: guids.nightly, + index: 3, + }, + { + guid: guids.support, + index: 4, + }, + { + guid: guids.customize, + index: 5, + }, + ], + }, + { + guid: guids.fx, + index: 2, + }, + ], + "Should use remote order as base if local is older" + ); + } finally { + await engine.wipeClient(); + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bookmark_order() { + let engine = await get_engine(); + let store = engine._store; + _("Starting with a clean slate of no bookmarks"); + await store.wipe(); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + // Index 2 is the tags root. (Root indices depend on the order of the + // `CreateRoot` calls in `Database::CreateBookmarkRoots`). + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "clean slate" + ); + + function bookmark(name, parent) { + let bm = new Bookmark("http://weave.server/my-bookmark"); + bm.id = name; + bm.title = name; + bm.bmkUri = "http://uri/"; + bm.parentid = parent || "unfiled"; + bm.tags = []; + return bm; + } + + function folder(name, parent, children) { + let bmFolder = new BookmarkFolder("http://weave.server/my-bookmark-folder"); + bmFolder.id = name; + bmFolder.title = name; + bmFolder.parentid = parent || "unfiled"; + bmFolder.children = children; + return bmFolder; + } + + async function apply(records) { + for (record of records) { + await store.applyIncoming(record); + } + await engine._apply(); + } + let id10 = "10_aaaaaaaaa"; + _("basic add first bookmark"); + await apply([bookmark(id10, "")]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic add first bookmark" + ); + let id20 = "20_aaaaaaaaa"; + _("basic append behind 10"); + await apply([bookmark(id20, "")]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic append behind 10" + ); + + let id31 = "31_aaaaaaaaa"; + let id30 = "f30_aaaaaaaa"; + _("basic create in folder"); + let b31 = bookmark(id31, id30); + let f30 = folder(id30, "", [id31]); + await apply([b31, f30]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic create in folder" + ); + + let id41 = "41_aaaaaaaaa"; + let id40 = "f40_aaaaaaaa"; + _("insert missing parent -> append to unfiled"); + await apply([bookmark(id41, id40)]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + { + guid: id41, + index: 3, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "insert missing parent -> append to unfiled" + ); + + let id42 = "42_aaaaaaaaa"; + + _("insert another missing parent -> append"); + await apply([bookmark(id42, id40)]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + { + guid: id41, + index: 3, + }, + { + guid: id42, + index: 4, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "insert another missing parent -> append" + ); + + await engine.wipeClient(); + await Service.startOver(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_bookmark_places_query_rewriting.js b/services/sync/tests/unit/test_bookmark_places_query_rewriting.js new file mode 100644 index 0000000000..e8dbbb48b1 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_places_query_rewriting.js @@ -0,0 +1,57 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Rewrite place: URIs."); +const { BookmarkQuery, BookmarkFolder } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function makeTagRecord(id, uri) { + let tagRecord = new BookmarkQuery("bookmarks", id); + tagRecord.queryId = "MagicTags"; + tagRecord.parentName = "Bookmarks Toolbar"; + tagRecord.bmkUri = uri; + tagRecord.title = "tagtag"; + tagRecord.folderName = "bar"; + tagRecord.parentid = PlacesUtils.bookmarks.toolbarGuid; + return tagRecord; +} + +add_bookmark_test(async function run_test(engine) { + let store = engine._store; + + let toolbar = new BookmarkFolder("bookmarks", "toolbar"); + toolbar.parentid = "places"; + toolbar.children = ["abcdefabcdef"]; + + let uri = "place:folder=499&type=7&queryType=1"; + let tagRecord = makeTagRecord("abcdefabcdef", uri); + + _("Type: " + tagRecord.type); + _("Folder name: " + tagRecord.folderName); + await store.applyIncoming(toolbar); + await store.applyIncoming(tagRecord); + await engine._apply(); + + let insertedRecord = await store.createRecord("abcdefabcdef", "bookmarks"); + Assert.equal(insertedRecord.bmkUri, "place:tag=bar"); + + _("... but not if the type is wrong."); + let wrongTypeURI = "place:folder=499&type=2&queryType=1"; + let wrongTypeRecord = makeTagRecord("fedcbafedcba", wrongTypeURI); + await store.applyIncoming(wrongTypeRecord); + toolbar.children = ["fedcbafedcba"]; + await store.applyIncoming(toolbar); + let expected = wrongTypeURI; + await engine._apply(); + // the mirror appends a special param to these. + expected += "&excludeItems=1"; + + insertedRecord = await store.createRecord("fedcbafedcba", "bookmarks"); + Assert.equal(insertedRecord.bmkUri, expected); +}); diff --git a/services/sync/tests/unit/test_bookmark_record.js b/services/sync/tests/unit/test_bookmark_record.js new file mode 100644 index 0000000000..c261027ed9 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_record.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Bookmark, BookmarkQuery, PlacesItem } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function prepareBookmarkItem(collection, id) { + let b = new Bookmark(collection, id); + b.cleartext.stuff = "my payload here"; + return b; +} + +add_task(async function test_bookmark_record() { + await configureIdentity(); + + await generateNewKeys(Service.collectionKeys); + let keyBundle = Service.identity.syncKeyBundle; + + _("Creating a record"); + + let placesItem = new PlacesItem("bookmarks", "foo", "bookmark"); + let bookmarkItem = prepareBookmarkItem("bookmarks", "foo"); + + _("Checking getTypeObject"); + Assert.equal(placesItem.getTypeObject(placesItem.type), Bookmark); + Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark); + + await bookmarkItem.encrypt(keyBundle); + _("Ciphertext is " + bookmarkItem.ciphertext); + Assert.ok(bookmarkItem.ciphertext != null); + + _("Decrypting the record"); + + let payload = await bookmarkItem.decrypt(keyBundle); + Assert.equal(payload.stuff, "my payload here"); + Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark); + Assert.notEqual(payload, bookmarkItem.payload); // wrap.data.payload is the encrypted one +}); + +add_task(async function test_query_foldername() { + // Bug 1443388 + let checks = [ + ["foo", "foo"], + ["", undefined], + ]; + for (let [inVal, outVal] of checks) { + let bmk1 = new BookmarkQuery("bookmarks", Utils.makeGUID()); + bmk1.fromSyncBookmark({ + url: Services.io.newURI("https://example.com"), + folder: inVal, + }); + Assert.strictEqual(bmk1.folderName, outVal); + + // other direction + let bmk2 = new BookmarkQuery("bookmarks", Utils.makeGUID()); + bmk2.folderName = inVal; + let record = bmk2.toSyncBookmark(); + Assert.strictEqual(record.folder, outVal); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_store.js b/services/sync/tests/unit/test_bookmark_store.js new file mode 100644 index 0000000000..2f4330ed2e --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_store.js @@ -0,0 +1,425 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Bookmark, BookmarkFolder, BookmarkQuery, PlacesItem } = + ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const BookmarksToolbarTitle = "toolbar"; + +// apply some test records without going via a test server. +async function apply_records(engine, records) { + for (record of records) { + await engine._store.applyIncoming(record); + } + await engine._apply(); +} + +add_bookmark_test(async function test_ignore_specials(engine) { + _("Ensure that we can't delete bookmark roots."); + let store = engine._store; + + // Belt... + let record = new BookmarkFolder("bookmarks", "toolbar", "folder"); + record.deleted = true; + Assert.notEqual( + null, + await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + + await apply_records(engine, [record]); + + // Ensure that the toolbar exists. + Assert.notEqual( + null, + await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + + await apply_records(engine, [record]); + + Assert.notEqual( + null, + await PlacesTestUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + await store.wipe(); +}); + +add_bookmark_test(async function test_bookmark_create(engine) { + let store = engine._store; + + try { + _("Ensure the record isn't present yet."); + let item = await PlacesUtils.bookmarks.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, item); + + _("Let's create a new record."); + let fxrecord = new Bookmark("bookmarks", "get-firefox1"); + fxrecord.bmkUri = "http://getfirefox.com/"; + fxrecord.title = "Get Firefox!"; + fxrecord.tags = ["firefox", "awesome", "browser"]; + fxrecord.keyword = "awesome"; + fxrecord.parentName = BookmarksToolbarTitle; + fxrecord.parentid = "toolbar"; + await apply_records(engine, [fxrecord]); + + _("Verify it has been created correctly."); + item = await PlacesUtils.bookmarks.fetch(fxrecord.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(item.url.href, "http://getfirefox.com/"); + Assert.equal(item.title, fxrecord.title); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + let keyword = await PlacesUtils.keywords.fetch(fxrecord.keyword); + Assert.equal(keyword.url.href, "http://getfirefox.com/"); + + _( + "Have the store create a new record object. Verify that it has the same data." + ); + let newrecord = await store.createRecord(fxrecord.id); + Assert.ok(newrecord instanceof Bookmark); + for (let property of [ + "type", + "bmkUri", + "title", + "keyword", + "parentName", + "parentid", + ]) { + Assert.equal(newrecord[property], fxrecord[property]); + } + Assert.ok(Utils.deepEquals(newrecord.tags.sort(), fxrecord.tags.sort())); + + _("The calculated sort index is based on frecency data."); + Assert.ok(newrecord.sortindex >= 150); + + _("Create a record with some values missing."); + let tbrecord = new Bookmark("bookmarks", "thunderbird1"); + tbrecord.bmkUri = "http://getthunderbird.com/"; + tbrecord.parentName = BookmarksToolbarTitle; + tbrecord.parentid = "toolbar"; + await apply_records(engine, [tbrecord]); + + _("Verify it has been created correctly."); + item = await PlacesUtils.bookmarks.fetch(tbrecord.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(item.url.href, "http://getthunderbird.com/"); + Assert.equal(item.title, ""); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + keyword = await PlacesUtils.keywords.fetch({ + url: "http://getthunderbird.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_bookmark_update(engine) { + let store = engine._store; + + try { + _("Create a bookmark whose values we'll change."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + await PlacesUtils.keywords.insert({ + url: "http://getfirefox.com/", + keyword: "firefox", + }); + + _("Update the record with some null values."); + let record = await store.createRecord(bmk1.guid); + record.title = null; + record.keyword = null; + record.tags = null; + await apply_records(engine, [record]); + + _("Verify that the values have been cleared."); + let item = await PlacesUtils.bookmarks.fetch(bmk1.guid); + Assert.equal(item.title, ""); + let keyword = await PlacesUtils.keywords.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_bookmark_createRecord(engine) { + let store = engine._store; + + try { + _("Create a bookmark without a title."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + }); + + _("Verify that the record is created accordingly."); + let record = await store.createRecord(bmk1.guid); + Assert.equal(record.title, ""); + Assert.equal(record.keyword, null); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_folder_create(engine) { + let store = engine._store; + + try { + _("Create a folder."); + let folder = new BookmarkFolder("bookmarks", "testfolder-1"); + folder.parentName = BookmarksToolbarTitle; + folder.parentid = "toolbar"; + folder.title = "Test Folder"; + await apply_records(engine, [folder]); + + _("Verify it has been created correctly."); + let item = await PlacesUtils.bookmarks.fetch(folder.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.equal(item.title, folder.title); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + + _( + "Have the store create a new record object. Verify that it has the same data." + ); + let newrecord = await store.createRecord(folder.id); + Assert.ok(newrecord instanceof BookmarkFolder); + for (let property of ["title", "parentName", "parentid"]) { + Assert.equal(newrecord[property], folder[property]); + } + + _("Folders have high sort index to ensure they're synced first."); + Assert.equal(newrecord.sortindex, 1000000); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_folder_createRecord(engine) { + let store = engine._store; + + try { + _("Create a folder."); + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder1", + }); + + _("Create two bookmarks in that folder without assigning them GUIDs."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + _("Create a record for the folder and verify basic properties."); + let record = await store.createRecord(folder1.guid); + Assert.ok(record instanceof BookmarkFolder); + Assert.equal(record.title, "Folder1"); + Assert.equal(record.parentid, "toolbar"); + Assert.equal(record.parentName, BookmarksToolbarTitle); + + _( + "Verify the folder's children. Ensures that the bookmarks were given GUIDs." + ); + Assert.deepEqual(record.children, [bmk1.guid, bmk2.guid]); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_deleted(engine) { + let store = engine._store; + + try { + _("Create a bookmark that will be deleted."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + // The engine needs to think we've previously synced it. + await PlacesTestUtils.markBookmarksAsSynced(); + + _("Delete the bookmark through the store."); + let record = new PlacesItem("bookmarks", bmk1.guid); + record.deleted = true; + await apply_records(engine, [record]); + _("Ensure it has been deleted."); + let item = await PlacesUtils.bookmarks.fetch(bmk1.guid); + let newrec = await store.createRecord(bmk1.guid); + Assert.equal(null, item); + Assert.equal(newrec.deleted, true); + _("Verify that the keyword has been cleared."); + let keyword = await PlacesUtils.keywords.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_move_folder(engine) { + let store = engine._store; + store._childrenToOrder = {}; // *sob* - only needed for legacy. + + try { + _("Create two folders and a bookmark in one of them."); + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder1", + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder2", + }); + let bmk = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + // add records to the store that represent the current state. + await apply_records(engine, [ + await store.createRecord(folder1.guid), + await store.createRecord(folder2.guid), + await store.createRecord(bmk.guid), + ]); + + _("Now simulate incoming records reparenting it."); + let bmkRecord = await store.createRecord(bmk.guid); + Assert.equal(bmkRecord.parentid, folder1.guid); + bmkRecord.parentid = folder2.guid; + + let folder1Record = await store.createRecord(folder1.guid); + Assert.deepEqual(folder1Record.children, [bmk.guid]); + folder1Record.children = []; + let folder2Record = await store.createRecord(folder2.guid); + Assert.deepEqual(folder2Record.children, []); + folder2Record.children = [bmk.guid]; + + await apply_records(engine, [bmkRecord, folder1Record, folder2Record]); + + _("Verify the new parent."); + let movedBmk = await PlacesUtils.bookmarks.fetch(bmk.guid); + Assert.equal(movedBmk.parentGuid, folder2.guid); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_move_order(engine) { + let store = engine._store; + let tracker = engine._tracker; + + // Make sure the tracker is turned on. + tracker.start(); + try { + _("Create two bookmarks"); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + _("Verify order."); + let childIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + "toolbar" + ); + Assert.deepEqual(childIds, [bmk1.guid, bmk2.guid]); + let toolbar = await store.createRecord("toolbar"); + Assert.deepEqual(toolbar.children, [bmk1.guid, bmk2.guid]); + + _("Move bookmarks around."); + store._childrenToOrder = {}; + toolbar.children = [bmk2.guid, bmk1.guid]; + await apply_records(engine, [ + toolbar, + await store.createRecord(bmk1.guid), + await store.createRecord(bmk2.guid), + ]); + delete store._childrenToOrder; + + _("Verify new order."); + let newChildIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + "toolbar" + ); + Assert.deepEqual(newChildIds, [bmk2.guid, bmk1.guid]); + } finally { + await tracker.stop(); + _("Clean up."); + await store.wipe(); + } +}); + +// Tests Bug 806460, in which query records arrive with empty folder +// names and missing bookmark URIs. +add_bookmark_test(async function test_empty_query_doesnt_die(engine) { + let record = new BookmarkQuery("bookmarks", "8xoDGqKrXf1P"); + record.folderName = ""; + record.queryId = ""; + record.parentName = "Toolbar"; + record.parentid = "toolbar"; + + // These should not throw. + await apply_records(engine, [record]); + + delete record.folderName; + await apply_records(engine, [record]); +}); + +add_bookmark_test(async function test_calculateIndex_for_invalid_url(engine) { + let store = engine._store; + + let folderIndex = await store._calculateIndex({ + type: "folder", + }); + equal(folderIndex, 1000000, "Should use high sort index for folders"); + + let toolbarIndex = await store._calculateIndex({ + parentid: "toolbar", + }); + equal(toolbarIndex, 150, "Should bump sort index for toolbar bookmarks"); + + let validURLIndex = await store._calculateIndex({ + bmkUri: "http://example.com/a", + }); + greaterOrEqual(validURLIndex, 0, "Should use frecency for index"); + + let invalidURLIndex = await store._calculateIndex({ + bmkUri: "!@#$%", + }); + equal(invalidURLIndex, 0, "Should not throw for invalid URLs"); +}); diff --git a/services/sync/tests/unit/test_bookmark_tracker.js b/services/sync/tests/unit/test_bookmark_tracker.js new file mode 100644 index 0000000000..9cfbb4de78 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_tracker.js @@ -0,0 +1,1275 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { PlacesTransactions } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesTransactions.sys.mjs" +); + +let engine; +let store; +let tracker; + +const DAY_IN_MS = 24 * 60 * 60 * 1000; + +add_task(async function setup() { + await Service.engineManager.switchAlternatives(); + engine = Service.engineManager.get("bookmarks"); + store = engine._store; + tracker = engine._tracker; +}); + +// Test helpers. +async function verifyTrackerEmpty() { + await PlacesTestUtils.promiseAsyncUpdates(); + let changes = await tracker.getChangedIDs(); + deepEqual(changes, {}); + equal(tracker.score, 0); +} + +async function resetTracker() { + await PlacesTestUtils.markBookmarksAsSynced(); + tracker.resetScore(); +} + +async function cleanup() { + await engine.setLastSync(0); + await store.wipe(); + await resetTracker(); + await tracker.stop(); +} + +// startTracking is a signal that the test wants to notice things that happen +// after this is called (ie, things already tracked should be discarded.) +async function startTracking() { + engine._tracker.start(); + await PlacesTestUtils.markBookmarksAsSynced(); +} + +async function verifyTrackedItems(tracked) { + await PlacesTestUtils.promiseAsyncUpdates(); + let changedIDs = await tracker.getChangedIDs(); + let trackedIDs = new Set(Object.keys(changedIDs)); + for (let guid of tracked) { + ok(guid in changedIDs, `${guid} should be tracked`); + ok(changedIDs[guid].modified > 0, `${guid} should have a modified time`); + ok(changedIDs[guid].counter >= -1, `${guid} should have a change counter`); + trackedIDs.delete(guid); + } + equal( + trackedIDs.size, + 0, + `Unhandled tracked IDs: ${JSON.stringify(Array.from(trackedIDs))}` + ); +} + +async function verifyTrackedCount(expected) { + await PlacesTestUtils.promiseAsyncUpdates(); + let changedIDs = await tracker.getChangedIDs(); + do_check_attribute_count(changedIDs, expected); +} + +// A debugging helper that dumps the full bookmarks tree. +// Currently unused, but might come in handy +// eslint-disable-next-line no-unused-vars +async function dumpBookmarks() { + let columns = [ + "id", + "title", + "guid", + "syncStatus", + "syncChangeCounter", + "position", + ]; + return PlacesUtils.promiseDBConnection().then(connection => { + let all = []; + return connection + .executeCached( + `SELECT ${columns.join(", ")} FROM moz_bookmarks;`, + {}, + row => { + let repr = {}; + for (let column of columns) { + repr[column] = row.getResultByName(column); + } + all.push(repr); + } + ) + .then(() => { + dump("All bookmarks:\n"); + dump(JSON.stringify(all, undefined, 2)); + }); + }); +} + +add_task(async function test_tracking() { + _("Test starting and stopping the tracker"); + + // Remove existing tracking information for roots. + await startTracking(); + + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Test Folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // creating the folder should have made 2 changes - the folder itself and + // the parent of the folder. + await verifyTrackedCount(2); + // Reset the changes as the rest of the test doesn't want to see these. + await resetTracker(); + + function createBmk() { + return PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + } + + try { + _("Tell the tracker to start tracking changes."); + await startTracking(); + await createBmk(); + // We expect two changed items because the containing folder + // changed as well (new child). + await verifyTrackedCount(2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + + _("Notifying twice won't do any harm."); + await createBmk(); + await verifyTrackedCount(3); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_tracker_sql_batching() { + _( + "Test tracker does the correct thing when it is forced to batch SQL queries" + ); + + const SQLITE_MAX_VARIABLE_NUMBER = 999; + let numItems = SQLITE_MAX_VARIABLE_NUMBER * 2 + 10; + + await startTracking(); + + let children = []; + for (let i = 0; i < numItems; i++) { + children.push({ + url: "https://example.org/" + i, + title: "Sync Bookmark " + i, + }); + } + let inserted = await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children, + }, + ], + }); + + Assert.equal(children.length, numItems); + Assert.equal(inserted.length, numItems + 1); + await verifyTrackedCount(numItems + 2); // The parent and grandparent are also tracked. + await resetTracker(); + + await PlacesUtils.bookmarks.remove(inserted[0]); + await verifyTrackedCount(numItems + 2); + + await cleanup(); +}); + +add_task(async function test_bookmarkAdded() { + _("Items inserted via the synchronous bookmarks API should be tracked"); + + try { + await startTracking(); + + _("Insert a folder using the sync API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let syncFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Sync Folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + await verifyTrackedItems(["menu", syncFolder.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a bookmark using the sync API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let syncBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: syncFolder.guid, + url: "https://example.org/sync", + title: "Sync Bookmark", + }); + await verifyTrackedItems([syncFolder.guid, syncBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_bookmarkAdded() { + _("Items inserted via the asynchronous bookmarks API should be tracked"); + + try { + await startTracking(); + + _("Insert a folder using the async API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Async Folder", + }); + await verifyTrackedItems(["menu", asyncFolder.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a bookmark using the async API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: asyncFolder.guid, + url: "https://example.org/async", + title: "Async Bookmark", + }); + await verifyTrackedItems([asyncFolder.guid, asyncBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a separator using the async API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncSep = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: asyncFolder.index, + }); + await verifyTrackedItems(["menu", asyncSep.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemChanged() { + _("Items updated using the asynchronous bookmarks API should be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark"); + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + + await startTracking(); + + _("Update the bookmark using the async API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: fxBmk.guid, + title: "Download Firefox", + url: "https://www.mozilla.org/firefox", + // PlacesUtils.bookmarks.update rejects last modified dates older than + // the added date. + lastModified: new Date(Date.now() + DAY_IN_MS), + }); + + await verifyTrackedItems([fxBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemChanged_itemDates() { + _("Changes to item dates should be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark"); + let fx_bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fx_bm.guid}`); + + await startTracking(); + + _("Reset the bookmark's added date, should not be tracked"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let dateAdded = new Date(Date.now() - DAY_IN_MS); + await PlacesUtils.bookmarks.update({ + guid: fx_bm.guid, + dateAdded, + }); + await verifyTrackedCount(0); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges); + + await resetTracker(); + + _( + "Reset the bookmark's added date and another property, should be tracked" + ); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + dateAdded = new Date(); + await PlacesUtils.bookmarks.update({ + guid: fx_bm.guid, + dateAdded, + title: "test", + }); + await verifyTrackedItems([fx_bm.guid]); + Assert.equal(tracker.score, 2 * SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + + await resetTracker(); + + _("Set the bookmark's last modified date"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let fx_id = await PlacesTestUtils.promiseItemId(fx_bm.guid); + let dateModified = Date.now() * 1000; + PlacesUtils.bookmarks.setItemLastModified(fx_id, dateModified); + await verifyTrackedItems([fx_bm.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemTagged() { + _("Items tagged using the synchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Create a folder"); + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Parent", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _("Folder ID: " + folder); + _("Folder GUID: " + folder.guid); + + _("Track changes to tags"); + let uri = CommonUtils.makeURI("http://getfirefox.com"); + let b = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: uri, + title: "Get Firefox!", + }); + _("New item is " + b); + _("GUID: " + b.guid); + + await startTracking(); + + _("Tag the item"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + PlacesUtils.tagging.tagURI(uri, ["foo"]); + + // bookmark should be tracked, folder should not be. + await verifyTrackedItems([b.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 6); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemUntagged() { + _("Items untagged using the synchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert tagged bookmarks"); + let uri = CommonUtils.makeURI("http://getfirefox.com"); + let fx1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: uri, + title: "Get Firefox!", + }); + // Different parent and title; same URL. + let fx2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: uri, + title: "Download Firefox", + }); + PlacesUtils.tagging.tagURI(uri, ["foo"]); + + await startTracking(); + + _("Remove the tag"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + PlacesUtils.tagging.untagURI(uri, ["foo"]); + + await verifyTrackedItems([fx1.guid, fx2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemUntagged() { + _("Items untagged using the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert tagged bookmarks"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "some tag", + }); + let fxTag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tag.guid, + url: "http://getfirefox.com", + }); + + await startTracking(); + + _("Remove the tag using the async bookmarks API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(fxTag.guid); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemTagged() { + _("Items tagged using the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert untagged bookmarks"); + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Folder 1", + }); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder1.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Folder 2", + }); + // Different parent and title; same URL. + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + + await startTracking(); + + // This will change once tags are moved into a separate table (bug 424160). + // We specifically test this case because Bookmarks.jsm updates tagged + // bookmarks and notifies observers. + _("Insert a tag using the async bookmarks API"); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "some tag", + }); + + _("Tag an item using the async bookmarks API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tag.guid, + url: "http://getfirefox.com", + }); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemKeywordChanged() { + _("Keyword changes via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert two bookmarks with the same URL"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + + await startTracking(); + + _("Add a keyword for both items"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.keywords.insert({ + keyword: "the_keyword", + url: "http://getfirefox.com", + postData: "postData", + }); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemKeywordDeleted() { + _("Keyword deletions via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert two bookmarks with the same URL and keywords"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + await PlacesUtils.keywords.insert({ + keyword: "the_keyword", + url: "http://getfirefox.com", + }); + + await startTracking(); + + _("Remove the keyword"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.keywords.remove("the_keyword"); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_bookmarkAdded_filtered_root() { + _("Items outside the change roots should not be tracked"); + + try { + await startTracking(); + + _("Create a new root"); + let root = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + title: "New root", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _(`New root GUID: ${root.guid}`); + + _("Insert a bookmark underneath the new root"); + let untrackedBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: root.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`New untracked bookmark GUID: ${untrackedBmk.guid}`); + + _("Insert a bookmark underneath the Places root"); + let rootBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`New Places root bookmark GUID: ${rootBmk.guid}`); + + _("New root and bookmark should be ignored"); + await verifyTrackedItems([]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_filtered_root() { + _("Deleted items outside the change roots should not be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark underneath the Places root"); + let rootBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`New Places root bookmark GUID: ${rootBmk.guid}`); + + await startTracking(); + + await PlacesUtils.bookmarks.remove(rootBmk); + + await verifyTrackedItems([]); + // We'll still increment the counter for the removed item. + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onPageAnnoChanged() { + _("Page annotations should not be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark without an annotation"); + let pageURI = "http://getfirefox.com"; + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: pageURI, + title: "Get Firefox!", + }); + + await startTracking(); + + _("Add a page annotation"); + await PlacesUtils.history.update({ + url: pageURI, + annotations: new Map([[PlacesUtils.CHARSET_ANNO, "UTF-16"]]), + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + await resetTracker(); + + _("Remove the page annotation"); + await PlacesUtils.history.update({ + url: pageURI, + annotations: new Map([[PlacesUtils.CHARSET_ANNO, null]]), + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onFaviconChanged() { + _("Favicon changes should not be tracked"); + + try { + await tracker.stop(); + + let pageURI = CommonUtils.makeURI("http://getfirefox.com"); + let iconURI = CommonUtils.makeURI("http://getfirefox.com/icon"); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: pageURI, + title: "Get Firefox!", + }); + + await PlacesTestUtils.addVisits(pageURI); + + await startTracking(); + + _("Favicon annotations should be ignored"); + let iconURL = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAA" + + "AAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg=="; + + PlacesUtils.favicons.replaceFaviconDataFromDataURL( + iconURI, + iconURL, + 0, + Services.scriptSecurityManager.getSystemPrincipal() + ); + + await new Promise(resolve => { + PlacesUtils.favicons.setAndFetchFaviconForPage( + pageURI, + iconURI, + true, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + (uri, dataLen, data, mimeType) => { + resolve(); + }, + Services.scriptSecurityManager.getSystemPrincipal() + ); + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_moveToFolder() { + _("Items moved via `moveToFolder` should be tracked"); + + try { + await tracker.stop(); + + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.menuGuid, + children: [ + { + guid: "bookmarkAAAA", + title: "A", + url: "http://example.com/a", + }, + { + guid: "bookmarkBBBB", + title: "B", + url: "http://example.com/b", + }, + { + guid: "bookmarkCCCC", + title: "C", + url: "http://example.com/c", + }, + { + guid: "bookmarkDDDD", + title: "D", + url: "http://example.com/d", + }, + ], + }); + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.toolbarGuid, + children: [ + { + guid: "bookmarkEEEE", + title: "E", + url: "http://example.com/e", + }, + ], + }); + + await startTracking(); + + _("Move (A B D) to the toolbar"); + await PlacesUtils.bookmarks.moveToFolder( + ["bookmarkAAAA", "bookmarkBBBB", "bookmarkDDDD"], + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + + // Moving multiple bookmarks between two folders should track the old + // folder, new folder, and moved bookmarks. + await verifyTrackedItems([ + "menu", + "toolbar", + "bookmarkAAAA", + "bookmarkBBBB", + "bookmarkDDDD", + ]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Reorder toolbar children: (D A B E)"); + await PlacesUtils.bookmarks.moveToFolder( + ["bookmarkDDDD", "bookmarkAAAA", "bookmarkBBBB"], + PlacesUtils.bookmarks.toolbarGuid, + 0 + ); + + // Reordering bookmarks in a folder should only track the folder, not the + // bookmarks. + await verifyTrackedItems(["toolbar"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_update() { + _("Items moved via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + _("Repositioning a bookmark should track the folder"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: tbBmk.guid, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }); + await verifyTrackedItems(["menu"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + await resetTracker(); + + _("Reparenting a bookmark should track both folders and the bookmark"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: tbBmk.guid, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + await verifyTrackedItems(["menu", "toolbar", tbBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_reorder() { + _("Items reordered via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert out-of-order bookmarks"); + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + + let mozBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://mozilla.org", + title: "Mozilla", + }); + _(`Mozilla GUID: ${mozBmk.guid}`); + + await startTracking(); + + _("Reorder bookmarks"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.reorder(PlacesUtils.bookmarks.menuGuid, [ + mozBmk.guid, + fxBmk.guid, + tbBmk.guid, + ]); + + // We only track the folder if we reorder its children, but we should + // bump the score for every changed item. + await verifyTrackedItems(["menu"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_removeFolderTransaction() { + _("Folders removed in a transaction should be tracked"); + + try { + await tracker.stop(); + + _("Create a folder with two children"); + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _(`Folder GUID: ${folder.guid}`); + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fx.guid}`); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tb.guid}`); + + await startTracking(); + + let txn = PlacesTransactions.Remove({ guid: folder.guid }); + // We haven't executed the transaction yet. + await verifyTrackerEmpty(); + + _("Execute the remove folder transaction"); + await txn.transact(); + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Undo the remove folder transaction"); + await PlacesTransactions.undo(); + + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Redo the transaction"); + await PlacesTransactions.redo(); + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_treeMoved() { + _("Moving an entire tree of bookmarks should track the parents"); + + try { + // Create a couple of parent folders. + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + test: "First test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // A second folder in the first. + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + title: "Second test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // Create a couple of bookmarks in the second folder. + await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Move folder 2 to be a sibling of folder1. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: folder2.guid, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }); + + // the menu and both folders should be tracked, the children should not be. + await verifyTrackedItems(["menu", folder1.guid, folder2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted() { + _("Bookmarks deleted via the synchronous API should be tracked"); + + try { + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Delete the last item - the item and parent should be tracked. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(tb); + + await verifyTrackedItems(["menu", tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemDeleted() { + _("Bookmarks deleted via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + _("Delete the first item"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(fxBmk.guid); + + await verifyTrackedItems(["menu", fxBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemDeleted_eraseEverything() { + _("Erasing everything should track all deleted items"); + + try { + await tracker.stop(); + + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.mobileGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.mobileGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + let mozBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://mozilla.org", + title: "Mozilla", + }); + _(`Mozilla GUID: ${mozBmk.guid}`); + let mdnBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://developer.mozilla.org", + title: "MDN", + }); + _(`MDN GUID: ${mdnBmk.guid}`); + let bugsFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Bugs", + }); + _(`Bugs folder GUID: ${bugsFolder.guid}`); + let bzBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: bugsFolder.guid, + url: "https://bugzilla.mozilla.org", + title: "Bugzilla", + }); + _(`Bugzilla GUID: ${bzBmk.guid}`); + let bugsChildFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: bugsFolder.guid, + title: "Bugs child", + }); + _(`Bugs child GUID: ${bugsChildFolder.guid}`); + let bugsGrandChildBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: bugsChildFolder.guid, + url: "https://example.com", + title: "Bugs grandchild", + }); + _(`Bugs grandchild GUID: ${bugsGrandChildBmk.guid}`); + + await startTracking(); + // Simulate moving a synced item into a new folder. Deleting the folder + // should write a tombstone for the item, but not the folder. + await PlacesTestUtils.setBookmarkSyncFields({ + guid: bugsChildFolder.guid, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, + }); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.eraseEverything(); + + // bugsChildFolder's sync status is still "NEW", so it shouldn't be + // tracked. bugsGrandChildBmk is "NORMAL", so we *should* write a + // tombstone and track it. + await verifyTrackedItems([ + "menu", + mozBmk.guid, + mdnBmk.guid, + "toolbar", + bugsFolder.guid, + "mobile", + fxBmk.guid, + tbBmk.guid, + "unfiled", + bzBmk.guid, + bugsGrandChildBmk.guid, + ]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 8); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 11); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_tree() { + _("Deleting a tree of bookmarks should track all items"); + + try { + // Create a couple of parent folders. + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "First test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // A second folder in the first. + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + title: "Second test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // Create a couple of bookmarks in the second folder. + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Delete folder2 - everything we created should be tracked. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(folder2); + + await verifyTrackedItems([fx.guid, tb.guid, folder1.guid, folder2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 4); + } finally { + _("Clean up."); + await cleanup(); + } +}); diff --git a/services/sync/tests/unit/test_bridged_engine.js b/services/sync/tests/unit/test_bridged_engine.js new file mode 100644 index 0000000000..25a81f8f69 --- /dev/null +++ b/services/sync/tests/unit/test_bridged_engine.js @@ -0,0 +1,248 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { BridgedEngine, BridgeWrapperXPCOM } = ChromeUtils.importESModule( + "resource://services-sync/bridged_engine.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// Wraps an `object` in a proxy so that its methods are bound to it. This +// simulates how XPCOM class instances have all their methods bound. +function withBoundMethods(object) { + return new Proxy(object, { + get(target, key) { + let value = target[key]; + return typeof value == "function" ? value.bind(target) : value; + }, + }); +} + +add_task(async function test_interface() { + class TestBridge { + constructor() { + this.storageVersion = 2; + this.syncID = "syncID111111"; + this.clear(); + } + + clear() { + this.lastSyncMillis = 0; + this.wasSyncStarted = false; + this.incomingEnvelopes = []; + this.uploadedIDs = []; + this.wasSyncFinished = false; + this.wasReset = false; + this.wasWiped = false; + } + + // `mozIBridgedSyncEngine` methods. + + getLastSync(callback) { + CommonUtils.nextTick(() => callback.handleSuccess(this.lastSyncMillis)); + } + + setLastSync(millis, callback) { + this.lastSyncMillis = millis; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + resetSyncId(callback) { + CommonUtils.nextTick(() => callback.handleSuccess(this.syncID)); + } + + ensureCurrentSyncId(newSyncId, callback) { + equal(newSyncId, this.syncID, "Local and new sync IDs should match"); + CommonUtils.nextTick(() => callback.handleSuccess(this.syncID)); + } + + syncStarted(callback) { + this.wasSyncStarted = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + storeIncoming(envelopes, callback) { + this.incomingEnvelopes.push(...envelopes.map(r => JSON.parse(r))); + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + apply(callback) { + let outgoingEnvelopes = [ + { + id: "hanson", + data: { + plants: ["seed", "flower 💐", "rose"], + canYouTell: false, + }, + }, + { + id: "sheryl-crow", + data: { + today: "winding 🛣", + tomorrow: "winding 🛣", + }, + }, + ].map(cleartext => + JSON.stringify({ + id: cleartext.id, + payload: JSON.stringify(cleartext), + }) + ); + CommonUtils.nextTick(() => callback.handleSuccess(outgoingEnvelopes)); + } + + setUploaded(millis, ids, callback) { + this.uploadedIDs.push(...ids); + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + syncFinished(callback) { + this.wasSyncFinished = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + reset(callback) { + this.clear(); + this.wasReset = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + wipe(callback) { + this.clear(); + this.wasWiped = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + } + + let bridge = new TestBridge(); + let engine = new BridgedEngine("Nineties", Service); + engine._bridge = new BridgeWrapperXPCOM(withBoundMethods(bridge)); + engine.enabled = true; + + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + + info("Add server records"); + let foo = server.user("foo"); + let collection = foo.collection("nineties"); + let now = new_timestamp(); + collection.insert( + "backstreet", + encryptPayload({ + id: "backstreet", + data: { + say: "I want it that way", + when: "never", + }, + }), + now + ); + collection.insert( + "tlc", + encryptPayload({ + id: "tlc", + data: { + forbidden: ["scrubs 🚫"], + numberAvailable: false, + }, + }), + now + 5 + ); + + info("Sync the engine"); + // Advance the last sync time to skip the Backstreet Boys... + bridge.lastSyncMillis = 1000 * (now + 2); + await sync_engine_and_validate_telem(engine, false); + + let metaGlobal = foo.collection("meta").wbo("global").get(); + deepEqual( + JSON.parse(metaGlobal.payload).engines.nineties, + { + version: 2, + syncID: "syncID111111", + }, + "Should write storage version and sync ID to m/g" + ); + + greater(bridge.lastSyncMillis, 0, "Should update last sync time"); + ok( + bridge.wasSyncStarted, + "Should have started sync before storing incoming" + ); + deepEqual( + bridge.incomingEnvelopes + .sort((a, b) => a.id.localeCompare(b.id)) + .map(({ payload, ...envelope }) => ({ + cleartextAsObject: JSON.parse(payload), + ...envelope, + })), + [ + { + id: "tlc", + modified: now + 5, + cleartextAsObject: { + id: "tlc", + data: { + forbidden: ["scrubs 🚫"], + numberAvailable: false, + }, + }, + }, + ], + "Should stage incoming records from server" + ); + deepEqual( + bridge.uploadedIDs.sort(), + ["hanson", "sheryl-crow"], + "Should mark new local records as uploaded" + ); + ok(bridge.wasSyncFinished, "Should have finished sync after uploading"); + + deepEqual( + collection.keys().sort(), + ["backstreet", "hanson", "sheryl-crow", "tlc"], + "Should have all records on server" + ); + let expectedRecords = [ + { + id: "sheryl-crow", + data: { + today: "winding 🛣", + tomorrow: "winding 🛣", + }, + }, + { + id: "hanson", + data: { + plants: ["seed", "flower 💐", "rose"], + canYouTell: false, + }, + }, + ]; + for (let expected of expectedRecords) { + let actual = collection.cleartext(expected.id); + deepEqual( + actual, + expected, + `Should upload record ${expected.id} from bridged engine` + ); + } + + await engine.resetClient(); + ok(bridge.wasReset, "Should reset local storage for bridge"); + + await engine.wipeClient(); + ok(bridge.wasWiped, "Should wipe local storage for bridge"); + + await engine.resetSyncID(); + ok( + !foo.collection("nineties"), + "Should delete server collection after resetting sync ID" + ); + } finally { + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_clients_engine.js b/services/sync/tests/unit/test_clients_engine.js new file mode 100644 index 0000000000..d910a67503 --- /dev/null +++ b/services/sync/tests/unit/test_clients_engine.js @@ -0,0 +1,2108 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { ClientEngine, ClientsRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/clients.sys.mjs" +); +const { CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const MORE_THAN_CLIENTS_TTL_REFRESH = 691200; // 8 days +const LESS_THAN_CLIENTS_TTL_REFRESH = 86400; // 1 day + +let engine; + +/** + * Unpack the record with this ID, and verify that it has the same version that + * we should be putting into records. + */ +async function check_record_version(user, id) { + let payload = user.collection("clients").wbo(id).data; + + let rec = new CryptoWrapper(); + rec.id = id; + rec.collection = "clients"; + rec.ciphertext = payload.ciphertext; + rec.hmac = payload.hmac; + rec.IV = payload.IV; + + let cleartext = await rec.decrypt( + Service.collectionKeys.keyForCollection("clients") + ); + + _("Payload is " + JSON.stringify(cleartext)); + equal(Services.appinfo.version, cleartext.version); + equal(1, cleartext.protocols.length); + equal("1.5", cleartext.protocols[0]); +} + +// compare 2 different command arrays, taking into account that a flowID +// attribute must exist, be unique in the commands, but isn't specified in +// "expected" as the value isn't known. +function compareCommands(actual, expected, description) { + let tweakedActual = JSON.parse(JSON.stringify(actual)); + tweakedActual.map(elt => delete elt.flowID); + deepEqual(tweakedActual, expected, description); + // each item must have a unique flowID. + let allIDs = new Set(actual.map(elt => elt.flowID).filter(fid => !!fid)); + equal(allIDs.size, actual.length, "all items have unique IDs"); +} + +async function syncClientsEngine(server) { + engine._lastFxADevicesFetch = 0; + engine.lastModified = server.getCollection("foo", "clients").timestamp; + await engine._sync(); +} + +add_task(async function setup() { + engine = Service.clientsEngine; +}); + +async function cleanup() { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await engine._tracker.clearChangedIDs(); + await engine._resetClient(); + // un-cleanup the logs (the resetBranch will have reset their levels), since + // not all the tests use SyncTestingInfrastructure, and it's cheap. + syncTestLogging(); + // We don't finalize storage at cleanup, since we use the same clients engine + // instance across all tests. +} + +add_task(async function test_bad_hmac() { + _("Ensure that Clients engine deletes corrupt records."); + let deletedCollections = []; + let deletedItems = []; + let callback = { + onItemDeleted(username, coll, wboID) { + deletedItems.push(coll + "/" + wboID); + }, + onCollectionDeleted(username, coll) { + deletedCollections.push(coll); + }, + }; + Object.setPrototypeOf(callback, SyncServerCallback); + let server = await serverForFoo(engine, callback); + let user = server.user("foo"); + + function check_clients_count(expectedCount) { + let coll = user.collection("clients"); + + // Treat a non-existent collection as empty. + equal(expectedCount, coll ? coll.count() : 0); + } + + function check_client_deleted(id) { + let coll = user.collection("clients"); + let wbo = coll.wbo(id); + return !wbo || !wbo.payload; + } + + async function uploadNewKeys() { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + } + + try { + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + await generateNewKeys(Service.collectionKeys); + + _("First sync, client record is uploaded"); + equal(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + check_clients_count(0); + await syncClientsEngine(server); + check_clients_count(1); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + // Our uploaded record has a version. + await check_record_version(user, engine.localID); + + // Initial setup can wipe the server, so clean up. + deletedCollections = []; + deletedItems = []; + + _("Change our keys and our client ID, reupload keys."); + let oldLocalID = engine.localID; // Preserve to test for deletion! + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + + _("Sync."); + await syncClientsEngine(server); + + _("Old record " + oldLocalID + " was deleted, new one uploaded."); + check_clients_count(1); + check_client_deleted(oldLocalID); + + _( + "Now change our keys but don't upload them. " + + "That means we get an HMAC error but redownload keys." + ); + Service.lastHMACEvent = 0; + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + deletedCollections = []; + deletedItems = []; + check_clients_count(1); + await syncClientsEngine(server); + + _("Old record was not deleted, new one uploaded."); + equal(deletedCollections.length, 0); + equal(deletedItems.length, 0); + check_clients_count(2); + + _( + "Now try the scenario where our keys are wrong *and* there's a bad record." + ); + // Clean up and start fresh. + user.collection("clients")._wbos = {}; + Service.lastHMACEvent = 0; + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + deletedCollections = []; + deletedItems = []; + check_clients_count(0); + + await uploadNewKeys(); + + // Sync once to upload a record. + await syncClientsEngine(server); + check_clients_count(1); + + // Generate and upload new keys, so the old client record is wrong. + await uploadNewKeys(); + + // Create a new client record and new keys. Now our keys are wrong, as well + // as the object on the server. We'll download the new keys and also delete + // the bad client record. + oldLocalID = engine.localID; // Preserve to test for deletion! + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + let oldKey = Service.collectionKeys.keyForCollection(); + + equal(deletedCollections.length, 0); + equal(deletedItems.length, 0); + await syncClientsEngine(server); + equal(deletedItems.length, 1); + check_client_deleted(oldLocalID); + check_clients_count(1); + let newKey = Service.collectionKeys.keyForCollection(); + ok(!oldKey.equals(newKey)); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_properties() { + _("Test lastRecordUpload property"); + try { + equal( + Svc.PrefBranch.getPrefType("clients.lastRecordUpload"), + Ci.nsIPrefBranch.PREF_INVALID + ); + equal(engine.lastRecordUpload, 0); + + let now = Date.now(); + engine.lastRecordUpload = now / 1000; + equal(engine.lastRecordUpload, Math.floor(now / 1000)); + } finally { + await cleanup(); + } +}); + +add_task(async function test_full_sync() { + _("Ensure that Clients engine fetches all records for each sync."); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let activeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let deletedID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: deletedID, + name: "Client to delete", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + let store = engine._store; + + _("First sync. 2 records downloaded; our record uploaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + deepEqual( + user.collection("clients").keys().sort(), + [activeID, deletedID, engine.localID].sort(), + "Our record should be uploaded on first sync" + ); + let ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [activeID, deletedID, engine.localID].sort(), + "Other clients should be downloaded on first sync" + ); + + _("Delete a record, then sync again"); + let collection = server.getCollection("foo", "clients"); + collection.remove(deletedID); + // Simulate a timestamp update in info/collections. + await syncClientsEngine(server); + + _("Record should be updated"); + ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [activeID, engine.localID].sort(), + "Deleted client should be removed on next sync" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_sync() { + _("Ensure that Clients engine uploads a new client record once a week."); + + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + function clientWBO() { + return user.collection("clients").wbo(engine.localID); + } + + try { + _("First sync. Client record is uploaded."); + equal(clientWBO(), undefined); + equal(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(!!clientWBO().payload); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + _( + "Let's time travel more than a week back, new record should've been uploaded." + ); + engine.lastRecordUpload -= MORE_THAN_CLIENTS_TTL_REFRESH; + let lastweek = engine.lastRecordUpload; + clientWBO().payload = undefined; + await syncClientsEngine(server); + ok(!!clientWBO().payload); + ok(engine.lastRecordUpload > lastweek); + ok(!engine.isFirstSync); + + _("Remove client record."); + await engine.removeClientData(); + equal(clientWBO().payload, undefined); + + _("Time travel one day back, no record uploaded."); + engine.lastRecordUpload -= LESS_THAN_CLIENTS_TTL_REFRESH; + let yesterday = engine.lastRecordUpload; + await syncClientsEngine(server); + equal(clientWBO().payload, undefined); + equal(engine.lastRecordUpload, yesterday); + ok(!engine.isFirstSync); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_client_name_change() { + _("Ensure client name change incurs a client record update."); + + let tracker = engine._tracker; + + engine.localID; // Needed to increase the tracker changedIDs count. + let initialName = engine.localName; + + tracker.start(); + _("initial name: " + initialName); + + // Tracker already has data, so clear it. + await tracker.clearChangedIDs(); + + let initialScore = tracker.score; + + let changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 0); + + Services.prefs.setStringPref( + "identity.fxaccounts.account.device.name", + "new name" + ); + await tracker.asyncObserver.promiseObserversComplete(); + + _("new name: " + engine.localName); + notEqual(initialName, engine.localName); + changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 1); + ok(engine.localID in changedIDs); + ok(tracker.score > initialScore); + ok(tracker.score >= SCORE_INCREMENT_XLARGE); + + await tracker.stop(); + + await cleanup(); +}); + +add_task(async function test_fxa_device_id_change() { + _("Ensure an FxA device ID change incurs a client record update."); + + let tracker = engine._tracker; + + engine.localID; // Needed to increase the tracker changedIDs count. + + tracker.start(); + + // Tracker already has data, so clear it. + await tracker.clearChangedIDs(); + + let initialScore = tracker.score; + + let changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 0); + + Services.obs.notifyObservers(null, "fxaccounts:new_device_id"); + await tracker.asyncObserver.promiseObserversComplete(); + + changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 1); + ok(engine.localID in changedIDs); + ok(tracker.score > initialScore); + ok(tracker.score >= SINGLE_USER_THRESHOLD); + + await tracker.stop(); + + await cleanup(); +}); + +add_task(async function test_last_modified() { + _("Ensure that remote records have a sane serverLastModified attribute."); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let activeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + let collection = user.collection("clients"); + + _("Sync to download the record"); + await syncClientsEngine(server); + + equal( + engine._store._remoteClients[activeID].serverLastModified, + now - 10, + "last modified in the local record is correctly the server last-modified" + ); + + _("Modify the record and re-upload it"); + // set a new name to make sure we really did upload. + engine._store._remoteClients[activeID].name = "New name"; + engine._modified.set(activeID, 0); + // The sync above also did a POST, so adjust our lastModified. + engine.lastModified = server.getCollection("foo", "clients").timestamp; + await engine._uploadOutgoing(); + + _("Local record should have updated timestamp"); + ok(engine._store._remoteClients[activeID].serverLastModified >= now); + + _("Record on the server should have new name but not serverLastModified"); + let payload = collection.cleartext(activeID); + equal(payload.name, "New name"); + equal(payload.serverLastModified, undefined); + } finally { + await cleanup(); + server.deleteCollections("foo"); + await promiseStopServer(server); + } +}); + +add_task(async function test_send_command() { + _("Verifies _sendCommandToClient puts commands in the outbound queue."); + + let store = engine._store; + let tracker = engine._tracker; + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + let action = "testCommand"; + let args = ["foo", "bar"]; + let extra = { flowID: "flowy" }; + + await engine._sendCommandToClient(action, args, remoteId, extra); + + let newRecord = store._remoteClients[remoteId]; + let clientCommands = (await engine._readCommands())[remoteId]; + notEqual(newRecord, undefined); + equal(clientCommands.length, 1); + + let command = clientCommands[0]; + equal(command.command, action); + equal(command.args.length, 2); + deepEqual(command.args, args); + ok(command.flowID); + + const changes = await tracker.getChangedIDs(); + notEqual(changes[remoteId], undefined); + + await cleanup(); +}); + +// The browser UI might call _addClientCommand indirectly without awaiting on the returned promise. +// We need to make sure this doesn't result on commands not being saved. +add_task(async function test_add_client_command_race() { + let promises = []; + for (let i = 0; i < 100; i++) { + promises.push( + engine._addClientCommand(`client-${i}`, { command: "cmd", args: [] }) + ); + } + await Promise.all(promises); + + let localCommands = await engine._readCommands(); + for (let i = 0; i < 100; i++) { + equal(localCommands[`client-${i}`].length, 1); + } +}); + +add_task(async function test_command_validation() { + _("Verifies that command validation works properly."); + + let store = engine._store; + + let testCommands = [ + ["resetAll", [], true], + ["resetAll", ["foo"], false], + ["resetEngine", ["tabs"], true], + ["resetEngine", [], false], + ["wipeEngine", ["tabs"], true], + ["wipeEngine", [], false], + ["logout", [], true], + ["logout", ["foo"], false], + ["__UNKNOWN__", [], false], + ]; + + for (let [action, args, expectedResult] of testCommands) { + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + await engine.sendCommand(action, args, remoteId); + + let newRecord = store._remoteClients[remoteId]; + notEqual(newRecord, undefined); + + let clientCommands = (await engine._readCommands())[remoteId]; + + if (expectedResult) { + _("Ensuring command is sent: " + action); + equal(clientCommands.length, 1); + + let command = clientCommands[0]; + equal(command.command, action); + deepEqual(command.args, args); + + notEqual(engine._tracker, undefined); + const changes = await engine._tracker.getChangedIDs(); + notEqual(changes[remoteId], undefined); + } else { + _("Ensuring command is scrubbed: " + action); + equal(clientCommands, undefined); + + if (store._tracker) { + equal(engine._tracker[remoteId], undefined); + } + } + } + await cleanup(); +}); + +add_task(async function test_command_duplication() { + _("Ensures duplicate commands are detected and not added"); + + let store = engine._store; + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + let action = "resetAll"; + let args = []; + + await engine.sendCommand(action, args, remoteId); + await engine.sendCommand(action, args, remoteId); + + let clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 1); + + _("Check variant args length"); + await engine._saveCommands({}); + + action = "resetEngine"; + await engine.sendCommand(action, [{ x: "foo" }], remoteId); + await engine.sendCommand(action, [{ x: "bar" }], remoteId); + + _("Make sure we spot a real dupe argument."); + await engine.sendCommand(action, [{ x: "bar" }], remoteId); + + clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 2); + + await cleanup(); +}); + +add_task(async function test_command_invalid_client() { + _("Ensures invalid client IDs are caught"); + + let id = Utils.makeGUID(); + let error; + + try { + await engine.sendCommand("wipeEngine", ["tabs"], id); + } catch (ex) { + error = ex; + } + + equal(error.message.indexOf("Unknown remote client ID: "), 0); + + await cleanup(); +}); + +add_task(async function test_process_incoming_commands() { + _("Ensures local commands are executed"); + + engine.localCommands = [{ command: "logout", args: [] }]; + + let ev = "weave:service:logout:finish"; + + let logoutPromise = new Promise(resolve => { + var handler = function () { + Svc.Obs.remove(ev, handler); + + resolve(); + }; + + Svc.Obs.add(ev, handler); + }); + + // logout command causes processIncomingCommands to return explicit false. + ok(!(await engine.processIncomingCommands())); + + await logoutPromise; + + await cleanup(); +}); + +add_task(async function test_filter_duplicate_names() { + _( + "Ensure that we exclude clients with identical names that haven't synced in a week." + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + // Synced recently. + let recentID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: recentID, + name: "My Phone", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + // Dupe of our client, synced more than 1 week ago. + let dupeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: dupeID, + name: engine.localName, + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 604820 + ); + + // Synced more than 1 week ago, but not a dupe. + let oldID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: oldID, + name: "My old desktop", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 604820 + ); + + try { + let store = engine._store; + + _("First sync"); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + deepEqual( + user.collection("clients").keys().sort(), + [recentID, dupeID, oldID, engine.localID].sort(), + "Our record should be uploaded on first sync" + ); + + let ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [recentID, dupeID, oldID, engine.localID].sort(), + "Duplicate ID should remain in getAllIDs" + ); + ok( + await engine._store.itemExists(dupeID), + "Dupe ID should be considered as existing for Sync methods." + ); + ok( + !engine.remoteClientExists(dupeID), + "Dupe ID should not be considered as existing for external methods." + ); + + // dupe desktop should not appear in .deviceTypes. + equal(engine.deviceTypes.get("desktop"), 2); + equal(engine.deviceTypes.get("mobile"), 1); + + // dupe desktop should not appear in stats + deepEqual(engine.stats, { + hasMobile: 1, + names: [engine.localName, "My Phone", "My old desktop"], + numClients: 3, + }); + + ok(engine.remoteClientExists(oldID), "non-dupe ID should exist."); + ok(!engine.remoteClientExists(dupeID), "dupe ID should not exist"); + equal( + engine.remoteClients.length, + 2, + "dupe should not be in remoteClients" + ); + + // Check that a subsequent Sync doesn't report anything as being processed. + let counts; + Svc.Obs.add("weave:engine:sync:applied", function observe(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", observe); + counts = subject; + }); + + await syncClientsEngine(server); + equal(counts.applied, 0); // We didn't report applying any records. + equal(counts.reconciled, 4); // We reported reconcilliation for all records + equal(counts.succeeded, 0); + equal(counts.failed, 0); + equal(counts.newFailed, 0); + + _("Broadcast logout to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + let collection = server.getCollection("foo", "clients"); + let recentPayload = collection.cleartext(recentID); + compareCommands( + recentPayload.commands, + [{ command: "logout", args: [] }], + "Should send commands to the recent client" + ); + + let oldPayload = collection.cleartext(oldID); + compareCommands( + oldPayload.commands, + [{ command: "logout", args: [] }], + "Should send commands to the week-old client" + ); + + let dupePayload = collection.cleartext(dupeID); + deepEqual( + dupePayload.commands, + [], + "Should not send commands to the dupe client" + ); + + _("Update the dupe client's modified time"); + collection.insertRecord( + { + id: dupeID, + name: engine.localName, + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + _("Second sync."); + await syncClientsEngine(server); + + ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [recentID, oldID, dupeID, engine.localID].sort(), + "Stale client synced, so it should no longer be marked as a dupe" + ); + + ok( + engine.remoteClientExists(dupeID), + "Dupe ID should appear as it synced." + ); + + // Recently synced dupe desktop should appear in .deviceTypes. + equal(engine.deviceTypes.get("desktop"), 3); + + // Recently synced dupe desktop should now appear in stats + deepEqual(engine.stats, { + hasMobile: 1, + names: [engine.localName, "My Phone", engine.localName, "My old desktop"], + numClients: 4, + }); + + ok( + engine.remoteClientExists(dupeID), + "recently synced dupe ID should now exist" + ); + equal( + engine.remoteClients.length, + 3, + "recently synced dupe should now be in remoteClients" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_command_sync() { + _("Ensure that commands are synced across clients."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let user = server.user("foo"); + let remoteId = Utils.makeGUID(); + + function clientWBO(id) { + return user.collection("clients").wbo(id); + } + + _("Create remote client record"); + user.collection("clients").insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + try { + _("Syncing."); + await syncClientsEngine(server); + + _("Checking remote record was downloaded."); + let clientRecord = engine._store._remoteClients[remoteId]; + notEqual(clientRecord, undefined); + equal(clientRecord.commands.length, 0); + + _("Send a command to the remote client."); + await engine.sendCommand("wipeEngine", ["tabs"]); + let clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 1); + await syncClientsEngine(server); + + _("Checking record was uploaded."); + notEqual(clientWBO(engine.localID).payload, undefined); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + notEqual(clientWBO(remoteId).payload, undefined); + + Svc.PrefBranch.setStringPref("client.GUID", remoteId); + await engine._resetClient(); + equal(engine.localID, remoteId); + _("Performing sync on resetted client."); + await syncClientsEngine(server); + notEqual(engine.localCommands, undefined); + equal(engine.localCommands.length, 1); + + let command = engine.localCommands[0]; + equal(command.command, "wipeEngine"); + equal(command.args.length, 1); + equal(command.args[0], "tabs"); + } finally { + await cleanup(); + + try { + let collection = server.getCollection("foo", "clients"); + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_clients_not_in_fxa_list() { + _("Ensure that clients not in the FxA devices list are marked as stale."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId, + protocols: ["1.5"], + }); + + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId2, + protocols: ["1.5"], + }); + + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteId }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + + ok(!engine._store._remoteClients[remoteId].stale); + ok(engine._store._remoteClients[remoteId2].stale); + } finally { + engine.fxAccounts = fxAccounts; + await cleanup(); + + try { + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_dupe_device_ids() { + _( + "Ensure that we mark devices with duplicate fxaDeviceIds but older lastModified as stale." + ); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let remoteDeviceId = Utils.makeGUID(); + + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord( + { + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteDeviceId, + protocols: ["1.5"], + }, + new_timestamp() - 3 + ); + collection.insertRecord({ + id: remoteId2, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteDeviceId, + protocols: ["1.5"], + }); + + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteDeviceId }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + + ok(engine._store._remoteClients[remoteId].stale); + ok(!engine._store._remoteClients[remoteId2].stale); + } finally { + engine.fxAccounts = fxAccounts; + await cleanup(); + + try { + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_refresh_fxa_device_list() { + _("Ensure we refresh the fxa device list when we expect to."); + + await engine._store.wipe(); + engine._lastFxaDeviceRefresh = 0; + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let numRefreshes = 0; + let now = Date.now(); + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [], + refreshDeviceList() { + numRefreshes += 1; + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return now; + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + Assert.equal(numRefreshes, 1, "first sync should refresh"); + now += 1000; // a second later. + await syncClientsEngine(server); + Assert.equal(numRefreshes, 1, "next sync should not refresh"); + now += 60 * 60 * 2 * 1000; // 2 hours later + await syncClientsEngine(server); + Assert.equal(numRefreshes, 2, "2 hours later should refresh"); + now += 1000; // a second later. + Assert.equal(numRefreshes, 2, "next sync should not refresh"); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_optional_client_fields() { + _("Ensure that we produce records with the fields added in Bug 1097222."); + + const SUPPORTED_PROTOCOL_VERSIONS = ["1.5"]; + let local = await engine._store.createRecord(engine.localID, "clients"); + equal(local.name, engine.localName); + equal(local.type, engine.localType); + equal(local.version, Services.appinfo.version); + deepEqual(local.protocols, SUPPORTED_PROTOCOL_VERSIONS); + + // Optional fields. + // Make sure they're what they ought to be... + equal(local.os, Services.appinfo.OS); + equal(local.appPackage, Services.appinfo.ID); + + // ... and also that they're non-empty. + ok(!!local.os); + ok(!!local.appPackage); + ok(!!local.application); + + // We don't currently populate device or formfactor. + // See Bug 1100722, Bug 1100723. + + await cleanup(); +}); + +add_task(async function test_merge_commands() { + _("Verifies local commands for remote clients are merged with the server's"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let desktopID = Utils.makeGUID(); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let mobileID = Utils.makeGUID(); + collection.insertRecord( + { + id: mobileID, + name: "Mobile client", + type: "mobile", + commands: [ + { + command: "logout", + args: [], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Broadcast logout to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + let desktopPayload = collection.cleartext(desktopID); + compareCommands( + desktopPayload.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + { + command: "logout", + args: [], + }, + ], + "Should send the logout command to the desktop client" + ); + + let mobilePayload = collection.cleartext(mobileID); + compareCommands( + mobilePayload.commands, + [{ command: "logout", args: [] }], + "Should not send a duplicate logout to the mobile client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_duplicate_remote_commands() { + _( + "Verifies local commands for remote clients are sent only once (bug 1289287)" + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let desktopID = Utils.makeGUID(); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 1 record downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Send command to client to wipe history engine"); + await engine.sendCommand("wipeEngine", ["history"]); + await syncClientsEngine(server); + + _( + "Simulate the desktop client consuming the command and syncing to the server" + ); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + _("Send another command to the desktop client to wipe tabs engine"); + await engine.sendCommand("wipeEngine", ["tabs"], desktopID); + await syncClientsEngine(server); + + let desktopPayload = collection.cleartext(desktopID); + compareCommands( + desktopPayload.commands, + [ + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should only send the second command to the desktop client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_upload_after_reboot() { + _("Multiple downloads, reboot, then upload (bug 1289287)"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let deviceBID = Utils.makeGUID(); + let deviceCID = Utils.makeGUID(); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceCID, + name: "Device C", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Send command to client to wipe tab engine"); + await engine.sendCommand("wipeEngine", ["tabs"], deviceBID); + + const oldUploadOutgoing = SyncEngine.prototype._uploadOutgoing; + SyncEngine.prototype._uploadOutgoing = async () => + engine._onRecordsWritten([], [deviceBID]); + await syncClientsEngine(server); + + let deviceBPayload = collection.cleartext(deviceBID); + compareCommands( + deviceBPayload.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + ], + "Should be the same because the upload failed" + ); + + _("Simulate the client B consuming the command and syncing to the server"); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + // Simulate reboot + SyncEngine.prototype._uploadOutgoing = oldUploadOutgoing; + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + + await syncClientsEngine(server); + + deviceBPayload = collection.cleartext(deviceBID); + compareCommands( + deviceBPayload.commands, + [ + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should only had written our outgoing command" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_keep_cleared_commands_after_reboot() { + _( + "Download commands, fail upload, reboot, then apply new commands (bug 1289287)" + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let deviceBID = Utils.makeGUID(); + let deviceCID = Utils.makeGUID(); + collection.insertRecord( + { + id: engine.localID, + name: "Device A", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["tabs"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceCID, + name: "Device C", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. Download remote and our record."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + + const oldUploadOutgoing = SyncEngine.prototype._uploadOutgoing; + SyncEngine.prototype._uploadOutgoing = async () => + engine._onRecordsWritten([], [deviceBID]); + let commandsProcessed = 0; + engine.service.wipeClient = _engine => { + commandsProcessed++; + }; + + await syncClientsEngine(server); + await engine.processIncomingCommands(); // Not called by the engine.sync(), gotta call it ourselves + equal(commandsProcessed, 2, "We processed 2 commands"); + + let localRemoteRecord = collection.cleartext(engine.localID); + compareCommands( + localRemoteRecord.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should be the same because the upload failed" + ); + + // Another client sends a wipe command + collection.insertRecord( + { + id: engine.localID, + name: "Device A", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["tabs"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["bookmarks"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 5 + ); + + // Simulate reboot + SyncEngine.prototype._uploadOutgoing = oldUploadOutgoing; + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + + commandsProcessed = 0; + engine.service.wipeClient = _engine => { + commandsProcessed++; + }; + await syncClientsEngine(server); + await engine.processIncomingCommands(); + equal( + commandsProcessed, + 1, + "We processed one command (the other were cleared)" + ); + + localRemoteRecord = collection.cleartext(deviceBID); + deepEqual(localRemoteRecord.commands, [], "Should be empty"); + } finally { + await cleanup(); + + // Reset service (remove mocks) + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + await engine._resetClient(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_deleted_commands() { + _("Verifies commands for a deleted client are discarded"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let activeID = Utils.makeGUID(); + collection.insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let deletedID = Utils.makeGUID(); + collection.insertRecord( + { + id: deletedID, + name: "Client to delete", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + await syncClientsEngine(server); + + _("Delete a record on the server."); + collection.remove(deletedID); + + _("Broadcast a command to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + deepEqual( + collection.keys().sort(), + [activeID, engine.localID].sort(), + "Should not reupload deleted clients" + ); + + let activePayload = collection.cleartext(activeID); + compareCommands( + activePayload.commands, + [{ command: "logout", args: [] }], + "Should send the command to the active client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_command_sync() { + _("Notify other clients when writing their record."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.getCollection("foo", "clients"); + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + try { + equal(collection.count(), 2, "2 remote records written"); + await syncClientsEngine(server); + equal( + collection.count(), + 3, + "3 remote records written (+1 for the synced local record)" + ); + + await engine.sendCommand("wipeEngine", ["tabs"]); + await engine._tracker.addChangedID(engine.localID); + const getClientFxaDeviceId = sinon + .stub(engine, "getClientFxaDeviceId") + .callsFake(id => "fxa-" + id); + const engineMock = sinon.mock(engine); + let _notifyCollectionChanged = engineMock + .expects("_notifyCollectionChanged") + .withArgs(["fxa-" + remoteId, "fxa-" + remoteId2]); + _("Syncing."); + await syncClientsEngine(server); + _notifyCollectionChanged.verify(); + + engineMock.restore(); + getClientFxaDeviceId.restore(); + } finally { + await cleanup(); + await engine._tracker.clearChangedIDs(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function ensureSameFlowIDs() { + let events = []; + let origRecordTelemetryEvent = Service.recordTelemetryEvent; + Service.recordTelemetryEvent = (object, method, value, extra) => { + events.push({ object, method, value, extra }); + }; + + let server = await serverForFoo(engine); + try { + // Setup 2 clients, send them a command, and ensure we get to events + // written, both with the same flowID. + await SyncTestingInfrastructure(server); + let collection = server.getCollection("foo", "clients"); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + await syncClientsEngine(server); + await engine.sendCommand("wipeEngine", ["tabs"]); + await syncClientsEngine(server); + equal(events.length, 2); + // we don't know what the flowID is, but do know it should be the same. + equal(events[0].extra.flowID, events[1].extra.flowID); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + // check it's correctly used when we specify a flow ID + events.length = 0; + let flowID = Utils.makeGUID(); + await engine.sendCommand("wipeEngine", ["tabs"], null, { flowID }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, flowID); + equal(events[1].extra.flowID, flowID); + + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + + // and that it works when something else is in "extra" + events.length = 0; + await engine.sendCommand("wipeEngine", ["tabs"], null, { + reason: "testing", + }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, events[1].extra.flowID); + equal(events[0].extra.reason, "testing"); + equal(events[1].extra.reason, "testing"); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + + // and when both are specified. + events.length = 0; + await engine.sendCommand("wipeEngine", ["tabs"], null, { + reason: "testing", + flowID, + }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, flowID); + equal(events[1].extra.flowID, flowID); + equal(events[0].extra.reason, "testing"); + equal(events[1].extra.reason, "testing"); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + } finally { + Service.recordTelemetryEvent = origRecordTelemetryEvent; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_duplicate_commands_telemetry() { + let events = []; + let origRecordTelemetryEvent = Service.recordTelemetryEvent; + Service.recordTelemetryEvent = (object, method, value, extra) => { + events.push({ object, method, value, extra }); + }; + + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + let collection = server.getCollection("foo", "clients"); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + await syncClientsEngine(server); + // Make sure deduping works before syncing + await engine.sendCommand("wipeEngine", ["history"], remoteId); + await engine.sendCommand("wipeEngine", ["history"], remoteId); + equal(events.length, 1); + await syncClientsEngine(server); + // And after syncing. + await engine.sendCommand("wipeEngine", ["history"], remoteId); + equal(events.length, 1); + // Ensure we aren't deduping commands to different clients + await engine.sendCommand("wipeEngine", ["history"], remoteId2); + equal(events.length, 2); + } finally { + Service.recordTelemetryEvent = origRecordTelemetryEvent; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_other_clients_notified_on_first_sync() { + _( + "Ensure that other clients are notified when we upload our client record for the first time." + ); + + await engine.resetLastSync(); + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const fxAccounts = engine.fxAccounts; + let calls = 0; + engine.fxAccounts = { + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + }, + notifyDevices() { + calls++; + return Promise.resolve(true); + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + engine.lastRecordUpload = 0; + _("First sync, should notify other clients"); + await syncClientsEngine(server); + equal(calls, 1); + + _("Second sync, should not notify other clients"); + await syncClientsEngine(server); + equal(calls, 1); + } finally { + engine.fxAccounts = fxAccounts; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task( + async function device_disconnected_notification_updates_known_stale_clients() { + const spyUpdate = sinon.spy(engine, "updateKnownStaleClients"); + + Services.obs.notifyObservers( + null, + "fxaccounts:device_disconnected", + JSON.stringify({ isLocalDevice: false }) + ); + ok(spyUpdate.calledOnce, "updateKnownStaleClients should be called"); + spyUpdate.resetHistory(); + + Services.obs.notifyObservers( + null, + "fxaccounts:device_disconnected", + JSON.stringify({ isLocalDevice: true }) + ); + ok(spyUpdate.notCalled, "updateKnownStaleClients should not be called"); + + spyUpdate.restore(); + } +); + +add_task(async function update_known_stale_clients() { + const makeFakeClient = id => ({ id, fxaDeviceId: `fxa-${id}` }); + const clients = [ + makeFakeClient("one"), + makeFakeClient("two"), + makeFakeClient("three"), + ]; + const stubRemoteClients = sinon + .stub(engine._store, "_remoteClients") + .get(() => { + return clients; + }); + const stubFetchFxADevices = sinon + .stub(engine, "_fetchFxADevices") + .callsFake(() => { + engine._knownStaleFxADeviceIds = ["fxa-one", "fxa-two"]; + }); + + engine._knownStaleFxADeviceIds = null; + await engine.updateKnownStaleClients(); + ok(clients[0].stale); + ok(clients[1].stale); + ok(!clients[2].stale); + + stubRemoteClients.restore(); + stubFetchFxADevices.restore(); +}); + +add_task(async function test_create_record_command_limit() { + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const fakeLimit = 4 * 1024; + + let maxSizeStub = sinon + .stub(Service, "getMemcacheMaxRecordPayloadSize") + .callsFake(() => fakeLimit); + + let user = server.user("foo"); + let remoteId = Utils.makeGUID(); + + _("Create remote client record"); + user.collection("clients").insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "57", + protocols: ["1.5"], + }); + + try { + _("Initial sync."); + await syncClientsEngine(server); + + _("Send a fairly sane number of commands."); + + for (let i = 0; i < 5; ++i) { + await engine.sendCommand("wipeEngine", [`history: ${i}`], remoteId); + } + + await syncClientsEngine(server); + + _("Make sure they all fit and weren't dropped."); + let parsedServerRecord = user.collection("clients").cleartext(remoteId); + + equal(parsedServerRecord.commands.length, 5); + + await engine.sendCommand("wipeEngine", ["history"], remoteId); + + _("Send a not-sane number of commands."); + // Much higher than the maximum number of commands we could actually fit. + for (let i = 0; i < 500; ++i) { + await engine.sendCommand("wipeEngine", [`tabs: ${i}`], remoteId); + } + + await syncClientsEngine(server); + + _("Ensure we didn't overflow the server limit."); + let wbo = user.collection("clients").wbo(remoteId); + less(wbo.payload.length, fakeLimit); + + _( + "And that the data we uploaded is both sane json and containing some commands." + ); + let remoteCommands = wbo.getCleartext().commands; + greater(remoteCommands.length, 2); + let firstCommand = remoteCommands[0]; + _( + "The first command should still be present, since it had a high priority" + ); + equal(firstCommand.command, "wipeEngine"); + _("And the last command in the list should be the last command we sent."); + let lastCommand = remoteCommands[remoteCommands.length - 1]; + equal(lastCommand.command, "wipeEngine"); + deepEqual(lastCommand.args, ["tabs: 499"]); + } finally { + maxSizeStub.restore(); + await cleanup(); + try { + let collection = server.getCollection("foo", "clients"); + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); diff --git a/services/sync/tests/unit/test_clients_escape.js b/services/sync/tests/unit/test_clients_escape.js new file mode 100644 index 0000000000..53ec7fd7a9 --- /dev/null +++ b/services/sync/tests/unit/test_clients_escape.js @@ -0,0 +1,57 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_clients_escape() { + _("Set up test fixtures."); + + await configureIdentity(); + let keyBundle = Service.identity.syncKeyBundle; + + let engine = Service.clientsEngine; + + try { + _("Test that serializing client records results in uploadable ascii"); + engine.localID = "ascii"; + engine.localName = "wéävê"; + + _("Make sure we have the expected record"); + let record = await engine._createRecord("ascii"); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + + _("Encrypting record..."); + await record.encrypt(keyBundle); + _("Encrypted."); + + let serialized = JSON.stringify(record); + let checkCount = 0; + _("Checking for all ASCII:", serialized); + for (let ch of serialized) { + let code = ch.charCodeAt(0); + _("Checking asciiness of '", ch, "'=", code); + Assert.ok(code < 128); + checkCount++; + } + + _("Processed", checkCount, "characters out of", serialized.length); + Assert.equal(checkCount, serialized.length); + + _("Making sure the record still looks like it did before"); + await record.decrypt(keyBundle); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + + _("Sanity check that creating the record also gives the same"); + record = await engine._createRecord("ascii"); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); diff --git a/services/sync/tests/unit/test_collection_getBatched.js b/services/sync/tests/unit/test_collection_getBatched.js new file mode 100644 index 0000000000..f5425abe92 --- /dev/null +++ b/services/sync/tests/unit/test_collection_getBatched.js @@ -0,0 +1,187 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Collection, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function recordRange(lim, offset, total) { + let res = []; + for (let i = offset; i < Math.min(lim + offset, total); ++i) { + res.push({ id: String(i), payload: "test:" + i }); + } + return res; +} + +function get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + throwAfter = Infinity, + interruptedAfter = Infinity, +}) { + let coll = new Collection("http://example.com/test/", WBORecord, Service); + coll.full = true; + let requests = []; + let responses = []; + coll.get = async function () { + let limit = +this.limit; + let offset = 0; + if (this.offset) { + equal(this.offset.slice(0, 6), "foobar"); + offset = +this.offset.slice(6); + } + requests.push({ + limit, + offset, + spec: this.spec, + headers: Object.assign({}, this.headers), + }); + if (--throwAfter === 0) { + throw new Error("Some Network Error"); + } + let body = recordRange(limit, offset, totalRecords); + let response = { + obj: body, + success: true, + status: 200, + headers: {}, + }; + if (--interruptedAfter === 0) { + response.success = false; + response.status = 412; + response.body = ""; + } else if (offset + limit < totalRecords) { + // Ensure we're treating this as an opaque string, since the docs say + // it might not be numeric. + response.headers["x-weave-next-offset"] = "foobar" + (offset + batchSize); + } + response.headers["x-last-modified"] = lastModified; + responses.push(response); + return response; + }; + return { responses, requests, coll }; +} + +add_task(async function test_success() { + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { responses, requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + }); + let { response, records } = await coll.getBatched(batchSize); + + equal(requests.length, Math.ceil(totalRecords / batchSize)); + + equal(records.length, totalRecords); + checkRecordsOrder(records); + + // ensure we're returning the last response + equal(responses[responses.length - 1], response); + + // check first separately since its a bit of a special case + ok(!requests[0].headers["x-if-unmodified-since"]); + ok(!requests[0].offset); + equal(requests[0].limit, batchSize); + let expectedOffset = 2; + for (let i = 1; i < requests.length; ++i) { + let req = requests[i]; + equal(req.headers["x-if-unmodified-since"], lastModified); + equal(req.limit, batchSize); + if (i !== requests.length - 1) { + equal(req.offset, expectedOffset); + } + + expectedOffset += batchSize; + } + + // ensure we cleaned up anything that would break further + // use of this collection. + ok(!coll._headers["x-if-unmodified-since"]); + ok(!coll.offset); + ok(!coll.limit || coll.limit == Infinity); +}); + +add_task(async function test_total_limit() { + _("getBatched respects the (initial) value of the limit property"); + const totalRecords = 100; + const recordLimit = 11; + const batchSize = 2; + const lastModified = "111111"; + let { requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + }); + coll.limit = recordLimit; + let { records } = await coll.getBatched(batchSize); + checkRecordsOrder(records); + + equal(requests.length, Math.ceil(recordLimit / batchSize)); + equal(records.length, recordLimit); + + for (let i = 0; i < requests.length; ++i) { + let req = requests[i]; + if (i !== requests.length - 1) { + equal(req.limit, batchSize); + } else { + equal(req.limit, recordLimit % batchSize); + } + } + + equal(coll._limit, recordLimit); +}); + +add_task(async function test_412() { + _("We shouldn't record records if we get a 412 in the middle of a batch"); + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { responses, requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + interruptedAfter: 3, + }); + let { response, records } = await coll.getBatched(batchSize); + + equal(requests.length, 3); + equal(records.length, 0); // we should not get any records + + // ensure we're returning the last response + equal(responses[responses.length - 1], response); + + ok(!response.success); + equal(response.status, 412); +}); + +add_task(async function test_get_throws() { + _("getBatched() should throw if a get() throws"); + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + throwAfter: 3, + }); + + await Assert.rejects(coll.getBatched(batchSize), /Some Network Error/); + + equal(requests.length, 3); +}); + +function checkRecordsOrder(records) { + ok(!!records.length); + for (let i = 0; i < records.length; i++) { + equal(records[i].id, String(i)); + equal(records[i].payload, "test:" + i); + } +} diff --git a/services/sync/tests/unit/test_collections_recovery.js b/services/sync/tests/unit/test_collections_recovery.js new file mode 100644 index 0000000000..fd923bc272 --- /dev/null +++ b/services/sync/tests/unit/test_collections_recovery.js @@ -0,0 +1,95 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Verify that we wipe the server if we have to regenerate keys. +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_missing_crypto_collection() { + enableValidationPrefs(); + + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + let johnColls = johnHelper.collections; + + let empty = false; + function maybe_empty(handler) { + return function (request, response) { + if (empty) { + let body = "{}"; + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + } else { + handler(request, response); + } + }; + } + + let handlers = { + "/1.1/johndoe/info/collections": maybe_empty(johnHelper.handler), + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + }; + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = johnU( + coll, + new ServerCollection({}, true).handler() + ); + } + let server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + + try { + let fresh = 0; + let orig = Service._freshStart; + Service._freshStart = async function () { + _("Called _freshStart."); + await orig.call(Service); + fresh++; + }; + + _("Startup, no meta/global: freshStart called once."); + await sync_and_validate_telem(); + Assert.equal(fresh, 1); + fresh = 0; + + _("Regular sync: no need to freshStart."); + await Service.sync(); + Assert.equal(fresh, 0); + + _("Simulate a bad info/collections."); + delete johnColls.crypto; + await sync_and_validate_telem(); + Assert.equal(fresh, 1); + fresh = 0; + + _("Regular sync: no need to freshStart."); + await sync_and_validate_telem(); + Assert.equal(fresh, 0); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_corrupt_keys.js b/services/sync/tests/unit/test_corrupt_keys.js new file mode 100644 index 0000000000..06d7335985 --- /dev/null +++ b/services/sync/tests/unit/test_corrupt_keys.js @@ -0,0 +1,248 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { CryptoWrapper, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_locally_changed_keys() { + enableValidationPrefs(); + + let hmacErrorCount = 0; + function counting(f) { + return async function () { + hmacErrorCount++; + return f.call(this); + }; + } + + Service.handleHMACEvent = counting(Service.handleHMACEvent); + + let server = new SyncServer(); + let johndoe = server.registerUser("johndoe", "password"); + johndoe.createContents({ + meta: {}, + crypto: {}, + clients: {}, + }); + server.start(); + + try { + Svc.PrefBranch.setStringPref("registerEngines", "Tab"); + + await configureIdentity({ username: "johndoe" }, server); + // We aren't doing a .login yet, so fudge the cluster URL. + Service.clusterURL = Service.identity._token.endpoint; + + await Service.engineManager.register(HistoryEngine); + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); + + async function corrupt_local_keys() { + Service.collectionKeys._default.keyPair = [ + await Weave.Crypto.generateRandomKey(), + await Weave.Crypto.generateRandomKey(), + ]; + } + + _("Setting meta."); + + // Bump version on the server. + let m = new WBORecord("meta", "global"); + m.payload = { + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION, + }; + await m.upload(Service.resource(Service.metaURL)); + + _( + "New meta/global: " + + JSON.stringify(johndoe.collection("meta").wbo("global")) + ); + + // Upload keys. + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + Assert.ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + + // Check that login works. + Assert.ok(await Service.login()); + Assert.ok(Service.isLoggedIn); + + // Sync should upload records. + await sync_and_validate_telem(); + + // Tabs exist. + _("Tabs modified: " + johndoe.modified("tabs")); + Assert.ok(johndoe.modified("tabs") > 0); + + // Let's create some server side history records. + let liveKeys = Service.collectionKeys.keyForCollection("history"); + _("Keys now: " + liveKeys.keyPair); + let visitType = Ci.nsINavHistoryService.TRANSITION_LINK; + let history = johndoe.createCollection("history"); + for (let i = 0; i < 5; i++) { + let id = "record-no--" + i; + let modified = Date.now() / 1000 - 60 * (i + 10); + + let w = new CryptoWrapper("history", "id"); + w.cleartext = { + id, + histUri: "http://foo/bar?" + id, + title: id, + sortindex: i, + visits: [{ date: (modified - 5) * 1000000, type: visitType }], + deleted: false, + }; + await w.encrypt(liveKeys); + + let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac }; + history.insert(id, payload, modified); + } + + history.timestamp = Date.now() / 1000; + let old_key_time = johndoe.modified("crypto"); + _("Old key time: " + old_key_time); + + // Check that we can decrypt one. + let rec = new CryptoWrapper("history", "record-no--0"); + await rec.fetch( + Service.resource(Service.storageURL + "history/record-no--0") + ); + _(JSON.stringify(rec)); + Assert.ok(!!(await rec.decrypt(liveKeys))); + + Assert.equal(hmacErrorCount, 0); + + // Fill local key cache with bad data. + await corrupt_local_keys(); + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + + Assert.equal(hmacErrorCount, 0); + + _("HMAC error count: " + hmacErrorCount); + // Now syncing should succeed, after one HMAC error. + await sync_and_validate_telem(ping => { + Assert.equal( + ping.engines.find(e => e.name == "history").incoming.applied, + 5 + ); + }); + + Assert.equal(hmacErrorCount, 1); + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + + // And look! We downloaded history! + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--0") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--1") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--2") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--3") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--4") + ); + Assert.equal(hmacErrorCount, 1); + + _("Busting some new server values."); + // Now what happens if we corrupt the HMAC on the server? + for (let i = 5; i < 10; i++) { + let id = "record-no--" + i; + let modified = 1 + Date.now() / 1000; + + let w = new CryptoWrapper("history", "id"); + w.cleartext = { + id, + histUri: "http://foo/bar?" + id, + title: id, + sortindex: i, + visits: [{ date: (modified - 5) * 1000000, type: visitType }], + deleted: false, + }; + await w.encrypt(Service.collectionKeys.keyForCollection("history")); + w.hmac = w.hmac.toUpperCase(); + + let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac }; + history.insert(id, payload, modified); + } + history.timestamp = Date.now() / 1000; + + _("Server key time hasn't changed."); + Assert.equal(johndoe.modified("crypto"), old_key_time); + + _("Resetting HMAC error timer."); + Service.lastHMACEvent = 0; + + _("Syncing..."); + await sync_and_validate_telem(ping => { + Assert.equal( + ping.engines.find(e => e.name == "history").incoming.failed, + 5 + ); + }); + + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + _( + "Server keys have been updated, and we skipped over 5 more HMAC errors without adjusting history." + ); + Assert.ok(johndoe.modified("crypto") > old_key_time); + Assert.equal(hmacErrorCount, 6); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--5") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--6") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--7") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--8") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--9") + ); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + validate_all_future_pings(); + + run_next_test(); +} diff --git a/services/sync/tests/unit/test_declined.js b/services/sync/tests/unit/test_declined.js new file mode 100644 index 0000000000..af7f8eb8c5 --- /dev/null +++ b/services/sync/tests/unit/test_declined.js @@ -0,0 +1,194 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { DeclinedEngines } = ChromeUtils.importESModule( + "resource://services-sync/stages/declined.sys.mjs" +); +const { EngineSynchronizer } = ChromeUtils.importESModule( + "resource://services-sync/stages/enginesync.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); + +function PetrolEngine() {} +PetrolEngine.prototype.name = "petrol"; + +function DieselEngine() {} +DieselEngine.prototype.name = "diesel"; + +function DummyEngine() {} +DummyEngine.prototype.name = "dummy"; + +function ActualEngine() {} +ActualEngine.prototype.name = "actual"; +Object.setPrototypeOf(ActualEngine.prototype, SyncEngine.prototype); + +function getEngineManager() { + let manager = new EngineManager(Service); + Service.engineManager = manager; + manager._engines = { + petrol: new PetrolEngine(), + diesel: new DieselEngine(), + dummy: new DummyEngine(), + actual: new ActualEngine(), + }; + return manager; +} + +/** + * 'Fetch' a meta/global record that doesn't mention declined. + * + * Push it into the EngineSynchronizer to set enabled; verify that those are + * correct. + * + * Then push it into DeclinedEngines to set declined; verify that none are + * declined, and a notification is sent for our locally disabled-but-not- + * declined engines. + */ +add_task(async function testOldMeta() { + let meta = { + payload: { + engines: { + petrol: 1, + diesel: 2, + nonlocal: 3, // Enabled but not supported. + }, + }, + }; + + _("Record: " + JSON.stringify(meta)); + + let manager = getEngineManager(); + + // Update enabled from meta/global. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledFromMeta(meta, 3, manager); + + Assert.ok(manager._engines.petrol.enabled, "'petrol' locally enabled."); + Assert.ok(manager._engines.diesel.enabled, "'diesel' locally enabled."); + Assert.ok( + !("nonlocal" in manager._engines), + "We don't know anything about the 'nonlocal' engine." + ); + Assert.ok(!manager._engines.actual.enabled, "'actual' not locally enabled."); + Assert.ok(!manager.isDeclined("actual"), "'actual' not declined, though."); + + let declinedEngines = new DeclinedEngines(Service); + + function onNotDeclined(subject, topic, data) { + Observers.remove("weave:engines:notdeclined", onNotDeclined); + Assert.ok( + subject.undecided.has("actual"), + "EngineManager observed that 'actual' was undecided." + ); + + let declined = manager.getDeclined(); + _("Declined: " + JSON.stringify(declined)); + + Assert.ok(!meta.changed, "No need to upload a new meta/global."); + run_next_test(); + } + + Observers.add("weave:engines:notdeclined", onNotDeclined); + + declinedEngines.updateDeclined(meta, manager); +}); + +/** + * 'Fetch' a meta/global that declines an engine we don't + * recognize. Ensure that we track that declined engine along + * with any we locally declined, and that the meta/global + * record is marked as changed and includes all declined + * engines. + */ +add_task(async function testDeclinedMeta() { + let meta = { + payload: { + engines: { + petrol: 1, + diesel: 2, + nonlocal: 3, // Enabled but not supported. + }, + declined: ["nonexistent"], // Declined and not supported. + }, + }; + + _("Record: " + JSON.stringify(meta)); + + let manager = getEngineManager(); + manager._engines.petrol.enabled = true; + manager._engines.diesel.enabled = true; + manager._engines.dummy.enabled = true; + manager._engines.actual.enabled = false; // Disabled but not declined. + + manager.decline(["localdecline"]); // Declined and not supported. + + let declinedEngines = new DeclinedEngines(Service); + + function onNotDeclined(subject, topic, data) { + Observers.remove("weave:engines:notdeclined", onNotDeclined); + Assert.ok( + subject.undecided.has("actual"), + "EngineManager observed that 'actual' was undecided." + ); + + let declined = manager.getDeclined(); + _("Declined: " + JSON.stringify(declined)); + + Assert.equal( + declined.indexOf("actual"), + -1, + "'actual' is locally disabled, but not marked as declined." + ); + + Assert.equal( + declined.indexOf("clients"), + -1, + "'clients' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("petrol"), + -1, + "'petrol' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("diesel"), + -1, + "'diesel' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("dummy"), + -1, + "'dummy' is enabled and not remotely declined." + ); + + Assert.ok( + 0 <= declined.indexOf("nonexistent"), + "'nonexistent' was declined on the server." + ); + + Assert.ok( + 0 <= declined.indexOf("localdecline"), + "'localdecline' was declined locally." + ); + + // The meta/global is modified, too. + Assert.ok( + 0 <= meta.payload.declined.indexOf("nonexistent"), + "meta/global's declined contains 'nonexistent'." + ); + Assert.ok( + 0 <= meta.payload.declined.indexOf("localdecline"), + "meta/global's declined contains 'localdecline'." + ); + Assert.strictEqual(true, meta.changed, "meta/global was changed."); + } + + Observers.add("weave:engines:notdeclined", onNotDeclined); + + declinedEngines.updateDeclined(meta, manager); +}); diff --git a/services/sync/tests/unit/test_disconnect_shutdown.js b/services/sync/tests/unit/test_disconnect_shutdown.js new file mode 100644 index 0000000000..0606c93a7d --- /dev/null +++ b/services/sync/tests/unit/test_disconnect_shutdown.js @@ -0,0 +1,101 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { SyncDisconnect, SyncDisconnectInternal } = ChromeUtils.importESModule( + "resource://services-sync/SyncDisconnect.sys.mjs" +); +const { AsyncShutdown } = ChromeUtils.importESModule( + "resource://gre/modules/AsyncShutdown.sys.mjs" +); +const { PREF_LAST_FXA_USER } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsCommon.sys.mjs" +); + +add_task(async function test_shutdown_blocker() { + let spySignout = sinon.stub( + SyncDisconnectInternal, + "doSyncAndAccountDisconnect" + ); + + // We don't need to check for the lock regularly as we end up aborting the wait. + SyncDisconnectInternal.lockRetryInterval = 1000; + // Force the retry count to a very large value - this test should never + // abort due to the retry count and we want the test to fail (aka timeout) + // should our abort code not work. + SyncDisconnectInternal.lockRetryCount = 10000; + // mock the "browser" sanitize function - it should not be called by + // this test. + let spyBrowser = sinon.stub(SyncDisconnectInternal, "doSanitizeBrowserData"); + // mock Sync + let mockEngine1 = { + enabled: true, + name: "Test Engine 1", + wipeClient: sinon.spy(), + }; + let mockEngine2 = { + enabled: false, + name: "Test Engine 2", + wipeClient: sinon.spy(), + }; + + // This weave mock never gives up the lock. + let Weave = { + Service: { + enabled: true, + lock: () => false, // so we never get the lock. + unlock: sinon.spy(), + + engineManager: { + getAll: sinon.stub().returns([mockEngine1, mockEngine2]), + }, + errorHandler: { + resetFileLog: sinon.spy(), + }, + }, + }; + let weaveStub = sinon.stub(SyncDisconnectInternal, "getWeave"); + weaveStub.returns(Weave); + + Services.prefs.setStringPref(PREF_LAST_FXA_USER, "dGVzdEBleGFtcGxlLmNvbQ=="); + + let promiseDisconnected = SyncDisconnect.disconnect(true); + + // Pretend we hit the shutdown blocker. + info("simulating quitApplicationGranted"); + Services.prefs.setBoolPref("toolkit.asyncshutdown.testing", true); + AsyncShutdown.quitApplicationGranted._trigger(); + Services.prefs.clearUserPref("toolkit.asyncshutdown.testing"); + + info("waiting for disconnect to complete"); + await promiseDisconnected; + + Assert.ok( + !Services.prefs.prefHasUserValue(PREF_LAST_FXA_USER), + "Should have reset different user warning pref" + ); + Assert.equal( + Weave.Service.unlock.callCount, + 0, + "should not have unlocked at the end" + ); + Assert.ok(!Weave.Service.enabled, "Weave should be and remain disabled"); + Assert.equal( + Weave.Service.errorHandler.resetFileLog.callCount, + 1, + "should have reset the log" + ); + Assert.equal( + mockEngine1.wipeClient.callCount, + 1, + "enabled engine should have been wiped" + ); + Assert.equal( + mockEngine2.wipeClient.callCount, + 0, + "disabled engine should not have been wiped" + ); + Assert.equal(spyBrowser.callCount, 1, "should not sanitize the browser"); + Assert.equal(spySignout.callCount, 1, "should have signed out of FxA"); +}); diff --git a/services/sync/tests/unit/test_engine.js b/services/sync/tests/unit/test_engine.js new file mode 100644 index 0000000000..31a08d5bc9 --- /dev/null +++ b/services/sync/tests/unit/test_engine.js @@ -0,0 +1,246 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function SteamStore(engine) { + Store.call(this, "Steam", engine); + this.wasWiped = false; +} +SteamStore.prototype = { + async wipe() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(SteamStore.prototype, Store.prototype); + +function SteamTracker(name, engine) { + LegacyTracker.call(this, name || "Steam", engine); +} +Object.setPrototypeOf(SteamTracker.prototype, LegacyTracker.prototype); + +function SteamEngine(name, service) { + SyncEngine.call(this, name, service); + this.wasReset = false; + this.wasSynced = false; +} +SteamEngine.prototype = { + _storeObj: SteamStore, + _trackerObj: SteamTracker, + + async _resetClient() { + this.wasReset = true; + }, + + async _sync() { + this.wasSynced = true; + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +var engineObserver = { + topics: [], + + observe(subject, topic, data) { + Assert.equal(data, "steam"); + this.topics.push(topic); + }, + + reset() { + this.topics = []; + }, +}; +Observers.add("weave:engine:reset-client:start", engineObserver); +Observers.add("weave:engine:reset-client:finish", engineObserver); +Observers.add("weave:engine:wipe-client:start", engineObserver); +Observers.add("weave:engine:wipe-client:finish", engineObserver); +Observers.add("weave:engine:sync:start", engineObserver); +Observers.add("weave:engine:sync:finish", engineObserver); + +async function cleanup(engine) { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + engine.wasReset = false; + engine.wasSynced = false; + engineObserver.reset(); + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +} + +add_task(async function test_members() { + _("Engine object members"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.equal(engine.Name, "Steam"); + Assert.equal(engine.prefName, "steam"); + Assert.ok(engine._store instanceof SteamStore); + Assert.ok(engine._tracker instanceof SteamTracker); +}); + +add_task(async function test_score() { + _("Engine.score corresponds to tracker.score and is readonly"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.equal(engine.score, 0); + engine._tracker.score += 5; + Assert.equal(engine.score, 5); + + try { + engine.score = 10; + } catch (ex) { + // Setting an attribute that has a getter produces an error in + // Firefox <= 3.6 and is ignored in later versions. Either way, + // the attribute's value won't change. + } + Assert.equal(engine.score, 5); +}); + +add_task(async function test_resetClient() { + _("Engine.resetClient calls _resetClient"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.ok(!engine.wasReset); + + await engine.resetClient(); + Assert.ok(engine.wasReset); + Assert.equal(engineObserver.topics[0], "weave:engine:reset-client:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:finish"); + + await cleanup(engine); +}); + +add_task(async function test_invalidChangedIDs() { + _("Test that invalid changed IDs on disk don't end up live."); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + let tracker = engine._tracker; + + await tracker._beforeSave(); + await IOUtils.writeUTF8(tracker._storage.path, "5", { + tmpPath: tracker._storage.path + ".tmp", + }); + + ok(!tracker._storage.dataReady); + const changes = await tracker.getChangedIDs(); + changes.placeholder = true; + deepEqual( + changes, + { placeholder: true }, + "Accessing changed IDs should load changes from disk as a side effect" + ); + ok(tracker._storage.dataReady); + + Assert.ok(changes.placeholder); + await cleanup(engine); +}); + +add_task(async function test_wipeClient() { + _("Engine.wipeClient calls resetClient, wipes store, clears changed IDs"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.ok(!engine.wasReset); + Assert.ok(!engine._store.wasWiped); + Assert.ok(await engine._tracker.addChangedID("a-changed-id")); + let changes = await engine._tracker.getChangedIDs(); + Assert.ok("a-changed-id" in changes); + + await engine.wipeClient(); + Assert.ok(engine.wasReset); + Assert.ok(engine._store.wasWiped); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(JSON.stringify(changes), "{}"); + Assert.equal(engineObserver.topics[0], "weave:engine:wipe-client:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:start"); + Assert.equal(engineObserver.topics[2], "weave:engine:reset-client:finish"); + Assert.equal(engineObserver.topics[3], "weave:engine:wipe-client:finish"); + + await cleanup(engine); +}); + +add_task(async function test_enabled() { + _("Engine.enabled corresponds to preference"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + try { + Assert.ok(!engine.enabled); + Svc.PrefBranch.setBoolPref("engine.steam", true); + Assert.ok(engine.enabled); + + engine.enabled = false; + Assert.ok(!Svc.PrefBranch.getBoolPref("engine.steam")); + } finally { + await cleanup(engine); + } +}); + +add_task(async function test_sync() { + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + try { + _("Engine.sync doesn't call _sync if it's not enabled"); + Assert.ok(!engine.enabled); + Assert.ok(!engine.wasSynced); + await engine.sync(); + + Assert.ok(!engine.wasSynced); + + _("Engine.sync calls _sync if it's enabled"); + engine.enabled = true; + + await engine.sync(); + Assert.ok(engine.wasSynced); + Assert.equal(engineObserver.topics[0], "weave:engine:sync:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:sync:finish"); + } finally { + await cleanup(engine); + } +}); + +add_task(async function test_disabled_no_track() { + _("When an engine is disabled, its tracker is not tracking."); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + let tracker = engine._tracker; + Assert.equal(engine, tracker.engine); + + Assert.ok(!engine.enabled); + Assert.ok(!tracker._isTracking); + let changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + Assert.ok(!tracker.engineIsEnabled()); + Assert.ok(!tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + let promisePrefChangeHandled = Promise.withResolvers(); + const origMethod = tracker.onEngineEnabledChanged; + tracker.onEngineEnabledChanged = async (...args) => { + await origMethod.apply(tracker, args); + promisePrefChangeHandled.resolve(); + }; + + engine.enabled = true; // Also enables the tracker automatically. + await promisePrefChangeHandled.promise; + Assert.ok(tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + await tracker.addChangedID("abcdefghijkl"); + changes = await tracker.getChangedIDs(); + Assert.ok(0 < changes.abcdefghijkl); + promisePrefChangeHandled = Promise.withResolvers(); + Svc.PrefBranch.setBoolPref("engine." + engine.prefName, false); + await promisePrefChangeHandled.promise; + Assert.ok(!tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + await cleanup(engine); +}); diff --git a/services/sync/tests/unit/test_engine_abort.js b/services/sync/tests/unit/test_engine_abort.js new file mode 100644 index 0000000000..f9bbf9d338 --- /dev/null +++ b/services/sync/tests/unit/test_engine_abort.js @@ -0,0 +1,79 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +add_task(async function test_processIncoming_abort() { + _( + "An abort exception, raised in applyIncoming, will abort _processIncoming." + ); + let engine = new RotaryEngine(Service); + + let collection = new ServerCollection(); + let id = Utils.makeGUID(); + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + _("Create some server data."); + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + _("Fake applyIncoming to abort."); + engine._store.applyIncoming = async function (record) { + let ex = { + code: SyncEngine.prototype.eEngineAbortApplyIncoming, + cause: "Nooo", + }; + _("Throwing: " + JSON.stringify(ex)); + throw ex; + }; + + _("Trying _processIncoming. It will throw after aborting."); + let err; + try { + await engine._syncStartup(); + await engine._processIncoming(); + } catch (ex) { + err = ex; + } + + Assert.equal(err, "Nooo"); + err = undefined; + + _("Trying engine.sync(). It will abort without error."); + try { + // This will quietly fail. + await engine.sync(); + } catch (ex) { + err = ex; + } + + Assert.equal(err, undefined); + + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_engine_changes_during_sync.js b/services/sync/tests/unit/test_engine_changes_during_sync.js new file mode 100644 index 0000000000..891bea41ec --- /dev/null +++ b/services/sync/tests/unit/test_engine_changes_during_sync.js @@ -0,0 +1,611 @@ +const { FormHistory } = ChromeUtils.importESModule( + "resource://gre/modules/FormHistory.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Bookmark, BookmarkFolder, BookmarkQuery } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { HistoryRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { FormRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { PrefRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); + +const LoginInfo = Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +/** + * We don't test the clients or tabs engines because neither has conflict + * resolution logic. The clients engine syncs twice per global sync, and + * custom conflict resolution logic for commands that doesn't use + * timestamps. Tabs doesn't have conflict resolution at all, since it's + * read-only. + */ + +async function assertChildGuids(folderGuid, expectedChildGuids, message) { + let tree = await PlacesUtils.promiseBookmarksTree(folderGuid); + let childGuids = tree.children.map(child => child.guid); + deepEqual(childGuids, expectedChildGuids, message); +} + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine._store.wipe(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function test_history_change_during_sync() { + _("Ensure that we don't bump the score when applying history records."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("history"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("history"); + + // Override `uploadOutgoing` to insert a record while we're applying + // changes. The tracker should ignore this change. + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local history visit"); + await addVisit("during_sync"); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + let remoteRec = new HistoryRec("history", "UrOOuzE5QM-e"); + remoteRec.histUri = "http://getfirefox.com/"; + remoteRec.title = "Get Firefox!"; + remoteRec.visits = [ + { + date: PlacesUtils.toPRTime(Date.now()), + type: PlacesUtils.history.TRANSITION_TYPED, + }, + ]; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for visits added during sync" + ); + + equal( + collection.count(), + 1, + "New local visit should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second history sync" + ); + + equal( + collection.count(), + 2, + "New local visit should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_passwords_change_during_sync() { + _("Ensure that we don't bump the score when applying passwords."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("passwords"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local password"); + let login = new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ); + await Services.logins.addLoginAsync(login); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + let remoteRec = new LoginRec( + "passwords", + "{765e3d6e-071d-d640-a83d-81a7eb62d3ed}" + ); + remoteRec.formSubmitURL = ""; + remoteRec.httpRealm = ""; + remoteRec.hostname = "https://mozilla.org"; + remoteRec.username = "username"; + remoteRec.password = "sekrit"; + remoteRec.timeCreated = Date.now(); + remoteRec.timePasswordChanged = Date.now(); + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for passwords added during first sync" + ); + + equal( + collection.count(), + 1, + "New local password should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second passwords sync" + ); + + equal( + collection.count(), + 2, + "New local password should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_prefs_change_during_sync() { + _("Ensure that we don't bump the score when applying prefs."); + + const TEST_PREF = "test.duringSync"; + // create a "control pref" for the pref we sync. + Services.prefs.setBoolPref("services.sync.prefs.sync.test.duringSync", true); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("prefs"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("prefs"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Updating local pref value"); + // Change the value of a synced pref. + Services.prefs.setStringPref(TEST_PREF, "hello"); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + // All synced prefs are stored in a single record, so we'll only ever + // have one record on the server. This test just checks that we don't + // track or upload prefs changed during the sync. + let guid = CommonUtils.encodeBase64URL(Services.appinfo.ID); + let remoteRec = new PrefRec("prefs", guid); + remoteRec.value = { + [TEST_PREF]: "world", + }; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for prefs added during first sync" + ); + let payloads = collection.payloads(); + equal( + payloads.length, + 1, + "Should not upload multiple prefs records after first sync" + ); + equal( + payloads[0].value[TEST_PREF], + "world", + "Should not upload pref value changed during first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second prefs sync" + ); + payloads = collection.payloads(); + equal( + payloads.length, + 1, + "Should not upload multiple prefs records after second sync" + ); + equal( + payloads[0].value[TEST_PREF], + "hello", + "Should upload changed pref value during second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + Services.prefs.clearUserPref(TEST_PREF); + } +}); + +add_task(async function test_forms_change_during_sync() { + _("Ensure that we don't bump the score when applying form records."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("forms"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("forms"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local form history entry"); + await FormHistory.update([ + { + op: "add", + fieldname: "favoriteDrink", + value: "cocoa", + }, + ]); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + // Add an existing remote form history entry. We shouldn't bump the score when + // we apply this record. + let remoteRec = new FormRec("forms", "Tl9dHgmJSR6FkyxS"); + remoteRec.name = "name"; + remoteRec.value = "alice"; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for forms added during first sync" + ); + + equal( + collection.count(), + 1, + "New local form should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second forms sync" + ); + + equal( + collection.count(), + 2, + "New local form should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_bookmark_change_during_sync() { + _("Ensure that we track bookmark changes made during a sync."); + + enableValidationPrefs(); + let schedulerProto = Object.getPrototypeOf(Service.scheduler); + let syncThresholdDescriptor = Object.getOwnPropertyDescriptor( + schedulerProto, + "syncThreshold" + ); + Object.defineProperty(Service.scheduler, "syncThreshold", { + // Trigger resync if any changes exist, rather than deciding based on the + // normal sync threshold. + get: () => 0, + }); + + let engine = Service.engineManager.get("bookmarks"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + + // Already-tracked bookmarks that shouldn't be uploaded during the first sync. + let bzBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }); + _(`Bugzilla GUID: ${bzBmk.guid}`); + + await PlacesTestUtils.setBookmarkSyncFields({ + guid: bzBmk.guid, + syncChangeCounter: 0, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + + let collection = server.user("foo").collection("bookmarks"); + + let bmk3; // New child of Folder 1, created locally during sync. + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting bookmark into local store"); + bmk3 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "https://mozilla.org/", + title: "Mozilla", + }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + // New bookmarks that should be uploaded during the first sync. + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Folder 1", + }); + _(`Folder GUID: ${folder1.guid}`); + + let tbBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + + engine._tracker.start(); + + try { + let bmk2_guid = "get-firefox1"; // New child of Folder 1, created remotely. + let folder2_guid = "folder2-1111"; // New folder, created remotely. + let tagQuery_guid = "tag-query111"; // New tag query child of Folder 2, created remotely. + let bmk4_guid = "example-org1"; // New tagged child of Folder 2, created remotely. + { + // An existing record changed on the server that should not trigger + // another sync when applied. + let remoteBzBmk = new Bookmark("bookmarks", bzBmk.guid); + remoteBzBmk.bmkUri = "https://bugzilla.mozilla.org/"; + remoteBzBmk.description = "New description"; + remoteBzBmk.title = "Bugzilla"; + remoteBzBmk.tags = ["new", "tags"]; + remoteBzBmk.parentName = "Bookmarks Menu"; + remoteBzBmk.parentid = "menu"; + collection.insert(bzBmk.guid, encryptPayload(remoteBzBmk.cleartext)); + + let remoteFolder = new BookmarkFolder("bookmarks", folder2_guid); + remoteFolder.title = "Folder 2"; + remoteFolder.children = [bmk4_guid, tagQuery_guid]; + remoteFolder.parentName = "Bookmarks Menu"; + remoteFolder.parentid = "menu"; + collection.insert(folder2_guid, encryptPayload(remoteFolder.cleartext)); + + let remoteFxBmk = new Bookmark("bookmarks", bmk2_guid); + remoteFxBmk.bmkUri = "http://getfirefox.com/"; + remoteFxBmk.description = "Firefox is awesome."; + remoteFxBmk.title = "Get Firefox!"; + remoteFxBmk.tags = ["firefox", "awesome", "browser"]; + remoteFxBmk.keyword = "awesome"; + remoteFxBmk.parentName = "Folder 1"; + remoteFxBmk.parentid = folder1.guid; + collection.insert(bmk2_guid, encryptPayload(remoteFxBmk.cleartext)); + + // A tag query referencing a nonexistent tag folder, which we should + // create locally when applying the record. + let remoteTagQuery = new BookmarkQuery("bookmarks", tagQuery_guid); + remoteTagQuery.bmkUri = "place:type=7&folder=999"; + remoteTagQuery.title = "Taggy tags"; + remoteTagQuery.folderName = "taggy"; + remoteTagQuery.parentName = "Folder 2"; + remoteTagQuery.parentid = folder2_guid; + collection.insert( + tagQuery_guid, + encryptPayload(remoteTagQuery.cleartext) + ); + + // A bookmark that should appear in the results for the tag query. + let remoteTaggedBmk = new Bookmark("bookmarks", bmk4_guid); + remoteTaggedBmk.bmkUri = "https://example.org/"; + remoteTaggedBmk.title = "Tagged bookmark"; + remoteTaggedBmk.tags = ["taggy"]; + remoteTaggedBmk.parentName = "Folder 2"; + remoteTaggedBmk.parentid = folder2_guid; + collection.insert(bmk4_guid, encryptPayload(remoteTaggedBmk.cleartext)); + + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + title: "toolbar", + children: [folder1.guid], + parentName: "places", + parentid: "places", + }) + ); + + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + title: "menu", + children: [bzBmk.guid, folder2_guid], + parentName: "places", + parentid: "places", + }) + ); + + collection.insert( + folder1.guid, + encryptPayload({ + id: folder1.guid, + type: "folder", + title: "Folder 1", + children: [bmk2_guid], + parentName: "toolbar", + parentid: "toolbar", + }) + ); + } + + await assertChildGuids( + folder1.guid, + [tbBmk.guid], + "Folder should have 1 child before first sync" + ); + + let pingsPromise = wait_for_pings(2); + + let changes = await PlacesSyncUtils.bookmarks.pullChanges(); + deepEqual( + Object.keys(changes).sort(), + [folder1.guid, tbBmk.guid, "menu", "mobile", "toolbar", "unfiled"].sort(), + "Should track bookmark and folder created before first sync" + ); + + // Unlike the tests above, we can't use `sync_engine_and_validate_telem` + // because the bookmarks engine will automatically schedule a follow-up + // sync for us. + _("Perform first sync and immediate follow-up sync"); + Service.sync({ engines: ["bookmarks"] }); + + let pings = await pingsPromise; + equal(pings.length, 2, "Should submit two pings"); + ok( + pings.every(p => { + assert_success_ping(p); + return p.syncs.length == 1; + }), + "Should submit 1 sync per ping" + ); + + strictEqual( + Service.scheduler.globalScore, + 0, + "Should reset global score after follow-up sync" + ); + ok(bmk3, "Should insert bookmark during first sync to simulate change"); + ok( + collection.wbo(bmk3.guid), + "Changed bookmark should be uploaded after follow-up sync" + ); + + let bmk2 = await PlacesUtils.bookmarks.fetch({ + guid: bmk2_guid, + }); + ok(bmk2, "Remote bookmark should be applied during first sync"); + { + // We only check child GUIDs, and not their order, because the exact + // order is an implementation detail. + let folder1Children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + folder1.guid + ); + deepEqual( + folder1Children.sort(), + [bmk2_guid, tbBmk.guid, bmk3.guid].sort(), + "Folder 1 should have 3 children after first sync" + ); + } + await assertChildGuids( + folder2_guid, + [bmk4_guid, tagQuery_guid], + "Folder 2 should have 2 children after first sync" + ); + let taggedURIs = []; + await PlacesUtils.bookmarks.fetch({ tags: ["taggy"] }, b => + taggedURIs.push(b.url) + ); + equal(taggedURIs.length, 1, "Should have 1 tagged URI"); + equal( + taggedURIs[0].href, + "https://example.org/", + "Synced tagged bookmark should appear in tagged URI list" + ); + + changes = await PlacesSyncUtils.bookmarks.pullChanges(); + deepEqual( + changes, + {}, + "Should have already uploaded changes in follow-up sync" + ); + + // First ping won't include validation data, since we've changed bookmarks + // and `canValidate` will indicate it can't proceed. + let engineData = pings.map(p => { + return p.syncs[0].engines.find(e => e.name == "bookmarks-buffered"); + }); + ok(engineData[0].validation, "Engine should validate after first sync"); + ok(engineData[1].validation, "Engine should validate after second sync"); + } finally { + Object.defineProperty( + schedulerProto, + "syncThreshold", + syncThresholdDescriptor + ); + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_enginemanager.js b/services/sync/tests/unit/test_enginemanager.js new file mode 100644 index 0000000000..3e366be54f --- /dev/null +++ b/services/sync/tests/unit/test_enginemanager.js @@ -0,0 +1,232 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function PetrolEngine() {} +PetrolEngine.prototype.name = "petrol"; +PetrolEngine.prototype.finalize = async function () {}; + +function DieselEngine() {} +DieselEngine.prototype.name = "diesel"; +DieselEngine.prototype.finalize = async function () {}; + +function DummyEngine() {} +DummyEngine.prototype.name = "dummy"; +DummyEngine.prototype.finalize = async function () {}; + +class ActualEngine extends SyncEngine { + constructor(service) { + super("Actual", service); + } +} + +add_task(async function test_basics() { + _("We start out with a clean slate"); + + let manager = new EngineManager(Service); + + let engines = await manager.getAll(); + Assert.equal(engines.length, 0); + Assert.equal(await manager.get("dummy"), undefined); + + _("Register an engine"); + await manager.register(DummyEngine); + let dummy = await manager.get("dummy"); + Assert.ok(dummy instanceof DummyEngine); + + engines = await manager.getAll(); + Assert.equal(engines.length, 1); + Assert.equal(engines[0], dummy); + + _("Register an already registered engine is ignored"); + await manager.register(DummyEngine); + Assert.equal(await manager.get("dummy"), dummy); + + _("Register multiple engines in one go"); + await manager.register([PetrolEngine, DieselEngine]); + let petrol = await manager.get("petrol"); + let diesel = await manager.get("diesel"); + Assert.ok(petrol instanceof PetrolEngine); + Assert.ok(diesel instanceof DieselEngine); + + engines = await manager.getAll(); + Assert.equal(engines.length, 3); + Assert.notEqual(engines.indexOf(petrol), -1); + Assert.notEqual(engines.indexOf(diesel), -1); + + _("Retrieve multiple engines in one go"); + engines = await manager.get(["dummy", "diesel"]); + Assert.equal(engines.length, 2); + Assert.notEqual(engines.indexOf(dummy), -1); + Assert.notEqual(engines.indexOf(diesel), -1); + + _("getEnabled() only returns enabled engines"); + engines = await manager.getEnabled(); + Assert.equal(engines.length, 0); + + petrol.enabled = true; + engines = await manager.getEnabled(); + Assert.equal(engines.length, 1); + Assert.equal(engines[0], petrol); + + dummy.enabled = true; + diesel.enabled = true; + engines = await manager.getEnabled(); + Assert.equal(engines.length, 3); + + _("getEnabled() returns enabled engines in sorted order"); + petrol.syncPriority = 1; + dummy.syncPriority = 2; + diesel.syncPriority = 3; + + engines = await manager.getEnabled(); + + Assert.deepEqual(engines, [petrol, dummy, diesel]); + + _("Changing the priorities should change the order in getEnabled()"); + + dummy.syncPriority = 4; + + engines = await manager.getEnabled(); + + Assert.deepEqual(engines, [petrol, diesel, dummy]); + + _("Unregister an engine by name"); + await manager.unregister("dummy"); + Assert.equal(await manager.get("dummy"), undefined); + engines = await manager.getAll(); + Assert.equal(engines.length, 2); + Assert.equal(engines.indexOf(dummy), -1); + + _("Unregister an engine by value"); + // manager.unregister() checks for instanceof Engine, so let's make one: + await manager.register(ActualEngine); + let actual = await manager.get("actual"); + Assert.ok(actual instanceof ActualEngine); + Assert.ok(actual instanceof SyncEngine); + + await manager.unregister(actual); + Assert.equal(await manager.get("actual"), undefined); +}); + +class AutoEngine { + constructor(type) { + this.name = "automobile"; + this.type = type; + this.initializeCalled = false; + this.finalizeCalled = false; + this.isActive = false; + } + + async initialize() { + Assert.ok(!this.initializeCalled); + Assert.equal(AutoEngine.current, undefined); + this.initializeCalled = true; + this.isActive = true; + AutoEngine.current = this; + } + + async finalize() { + Assert.equal(AutoEngine.current, this); + Assert.ok(!this.finalizeCalled); + Assert.ok(this.isActive); + this.finalizeCalled = true; + this.isActive = false; + AutoEngine.current = undefined; + } +} + +class GasolineEngine extends AutoEngine { + constructor() { + super("gasoline"); + } +} + +class ElectricEngine extends AutoEngine { + constructor() { + super("electric"); + } +} + +add_task(async function test_alternates() { + let manager = new EngineManager(Service); + let engines = await manager.getAll(); + Assert.equal(engines.length, 0); + + const prefName = "services.sync.engines.automobile.electric"; + Services.prefs.clearUserPref(prefName); + + await manager.registerAlternatives( + "automobile", + prefName, + ElectricEngine, + GasolineEngine + ); + + let gasEngine = manager.get("automobile"); + Assert.equal(gasEngine.type, "gasoline"); + + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + Assert.equal(AutoEngine.current, gasEngine); + + _("Check that setting the controlling pref to false makes no difference"); + Services.prefs.setBoolPref(prefName, false); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Even after the call to switchAlternatives"); + await manager.switchAlternatives(); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Set the pref to true, we still shouldn't switch yet"); + Services.prefs.setBoolPref(prefName, true); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Now we expect to switch from gas to electric"); + await manager.switchAlternatives(); + let elecEngine = manager.get("automobile"); + Assert.equal(elecEngine.type, "electric"); + Assert.ok(elecEngine.isActive); + Assert.ok(elecEngine.initializeCalled); + Assert.ok(!elecEngine.finalizeCalled); + Assert.equal(AutoEngine.current, elecEngine); + + Assert.ok(!gasEngine.isActive); + Assert.ok(gasEngine.finalizeCalled); + + _("Switch back, and ensure we get a new instance that got initialized again"); + Services.prefs.setBoolPref(prefName, false); + await manager.switchAlternatives(); + + // First make sure we deactivated the electric engine as we should + Assert.ok(!elecEngine.isActive); + Assert.ok(elecEngine.initializeCalled); + Assert.ok(elecEngine.finalizeCalled); + + let newGasEngine = manager.get("automobile"); + Assert.notEqual(newGasEngine, gasEngine); + Assert.equal(newGasEngine.type, "gasoline"); + + Assert.ok(newGasEngine.isActive); + Assert.ok(newGasEngine.initializeCalled); + Assert.ok(!newGasEngine.finalizeCalled); + + _("Make sure unregister removes the alt info too"); + await manager.unregister("automobile"); + Assert.equal(manager.get("automobile"), null); + Assert.ok(newGasEngine.finalizeCalled); + Assert.deepEqual(Object.keys(manager._altEngineInfo), []); +}); diff --git a/services/sync/tests/unit/test_errorhandler_1.js b/services/sync/tests/unit/test_errorhandler_1.js new file mode 100644 index 0000000000..2d52b93a02 --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_1.js @@ -0,0 +1,341 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +const fakeServer = new SyncServer(); +fakeServer.start(); +const fakeServerUrl = "http://localhost:" + fakeServer.port; + +registerCleanupFunction(function () { + return promiseStopServer(fakeServer).finally(() => { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + }); +}); + +let engine; +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(EHTestsCommon.CatapultEngine); + engine = Service.engineManager.get("catapult"); +}); + +async function clean() { + let promiseLogReset = promiseOneObserver("weave:service:reset-file-log"); + await Service.startOver(); + await promiseLogReset; + Status.resetSync(); + Status.resetBackoff(); + // Move log levels back to trace (startOver will have reversed this), sicne + syncTestLogging(); +} + +add_task(async function test_401_logout() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await sync_and_validate_telem(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + let promiseErrors = new Promise(res => { + Svc.Obs.add("weave:service:sync:error", onSyncError); + function onSyncError() { + _("Got weave:service:sync:error in first sync."); + Svc.Obs.remove("weave:service:sync:error", onSyncError); + + // Wait for the automatic next sync. + Svc.Obs.add("weave:service:login:error", onLoginError); + function onLoginError() { + _("Got weave:service:login:error in second sync."); + Svc.Obs.remove("weave:service:login:error", onLoginError); + res(); + } + } + }); + + // Make sync fail due to login rejected. + await configureIdentity({ username: "janedoe" }, server); + Service._updateCachedURLs(); + + _("Starting first sync."); + await sync_and_validate_telem(ping => { + deepEqual(ping.failureReason, { name: "httperror", code: 401 }); + }); + _("First sync done."); + + await promiseErrors; + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + Assert.ok(!Service.isLoggedIn); + + // Clean up. + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_credentials_changed_logout() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await sync_and_validate_telem(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, CREDENTIALS_CHANGED); + deepEqual(ping.failureReason, { + name: "unexpectederror", + error: "Error: Aborting sync, remote setup failed", + }); + }); + + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + Assert.ok(!Service.isLoggedIn); + + // Clean up. + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_login_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + // when calling sync + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + Service.identity._syncKeyBundle = null; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_sync_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + // when calling sync + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, CREDENTIALS_CHANGED); + deepEqual(ping.failureReason, { + name: "unexpectederror", + error: "Error: Aborting sync, remote setup failed", + }); + }); + + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + // If we clean this tick, telemetry won't get the right error + await Async.promiseYield(); + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_login_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are reported when calling sync. + await configureIdentity({ username: "broken.wipe" }); + Service.clusterURL = fakeServerUrl; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + await clean(); +}); + +add_task(async function test_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are reported when calling sync. + Services.io.offline = true; + + await Service.sync(); + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_login_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + Service.identity._syncKeyBundle = null; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_sync_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await Service.sync(); + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_login_network_error() { + enableValidationPrefs(); + + await configureIdentity({ username: "johndoe" }); + Service.clusterURL = fakeServerUrl; + + // Test network errors are not reported. + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are not reported. + Services.io.offline = true; + + await Service.sync(); + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_sync_server_maintenance_error() { + enableValidationPrefs(); + + // Test server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + const BACKOFF = 42; + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + Assert.equal(Status.service, STATUS_OK); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, SERVER_MAINTENANCE); + deepEqual(ping.engines.find(e => e.failureReason).failureReason, { + name: "httperror", + code: 503, + }); + }); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_info_collections_login_server_maintenance_error() { + enableValidationPrefs(); + + // Test info/collections server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.info" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_meta_global_login_server_maintenance_error() { + enableValidationPrefs(); + + // Test meta/global server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.meta" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_errorhandler_2.js b/services/sync/tests/unit/test_errorhandler_2.js new file mode 100644 index 0000000000..5cab4d832d --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_2.js @@ -0,0 +1,550 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const fakeServer = new SyncServer(); +fakeServer.start(); + +registerCleanupFunction(function () { + return promiseStopServer(fakeServer).finally(() => { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + }); +}); + +const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"]); +logsdir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); + +function removeLogFiles() { + let entries = logsdir.directoryEntries; + while (entries.hasMoreElements()) { + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + logfile.remove(false); + } +} + +function getLogFiles() { + let result = []; + let entries = logsdir.directoryEntries; + while (entries.hasMoreElements()) { + result.push(entries.getNext().QueryInterface(Ci.nsIFile)); + } + return result; +} + +let engine; +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(EHTestsCommon.CatapultEngine); + engine = Service.engineManager.get("catapult"); +}); + +async function clean() { + let promiseLogReset = promiseOneObserver("weave:service:reset-file-log"); + await Service.startOver(); + await promiseLogReset; + Status.resetSync(); + Status.resetBackoff(); + removeLogFiles(); + // Move log levels back to trace (startOver will have reversed this), sicne + syncTestLogging(); +} + +add_task(async function test_crypto_keys_login_server_maintenance_error() { + enableValidationPrefs(); + + Status.resetSync(); + // Test crypto/keys server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.keys" }, server); + + // Force re-download of keys + Service.collectionKeys.clear(); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_lastSync_not_updated_on_complete_failure() { + enableValidationPrefs(); + + // Test info/collections prolonged server maintenance errors are reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "johndoe" }, server); + + // Do an initial sync that we expect to be successful. + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await sync_and_validate_telem(); + await promiseObserved; + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + let lastSync = Svc.PrefBranch.getStringPref("lastSync"); + + Assert.ok(lastSync); + + // Report server maintenance on info/collections requests + server.registerPathHandler( + "/1.1/johndoe/info/collections", + EHTestsCommon.service_unavailable + ); + + promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await sync_and_validate_telem(() => {}); + await promiseObserved; + + Assert.equal(Status.sync, SERVER_MAINTENANCE); + Assert.equal(Status.service, SYNC_FAILED); + + // We shouldn't update lastSync on complete failure. + Assert.equal(lastSync, Svc.PrefBranch.getStringPref("lastSync")); + + await clean(); + await promiseStopServer(server); +}); + +add_task( + async function test_sync_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + const BACKOFF = 42; + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_info_collections_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test info/collections server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.info" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_meta_global_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test meta/global server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.meta" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.keys" }, server); + // Force re-download of keys + Service.collectionKeys.clear(); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + + // Start off with an empty account, do not upload a key. + await configureIdentity({ username: "broken.keys" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + + // Start off with an empty account, do not upload a key. + await configureIdentity({ username: "broken.wipe" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_wipeRemote_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test that we report prolonged server maintenance errors that occur whilst + // wiping all remote devices. + let server = await EHTestsCommon.sync_httpd_setup(); + + await configureIdentity({ username: "broken.wipe" }, server); + await EHTestsCommon.generateAndUploadKeys(); + + engine.exception = null; + engine.enabled = true; + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + Svc.PrefBranch.setStringPref("firstSync", "wipeRemote"); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, SYNC_FAILED); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + Assert.equal(Svc.PrefBranch.getStringPref("firstSync"), "wipeRemote"); + + await clean(); + await promiseStopServer(server); + } +); + +add_task(async function test_sync_engine_generic_fail() { + enableValidationPrefs(); + + equal(getLogFiles().length, 0); + + let server = await EHTestsCommon.sync_httpd_setup(); + engine.enabled = true; + engine.sync = async function sync() { + Svc.Obs.notify("weave:engine:sync:error", ENGINE_UNKNOWN_FAIL, "catapult"); + }; + let lastSync = Svc.PrefBranch.getStringPref("lastSync", null); + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + Assert.equal(Status.engines.catapult, undefined); + + let promiseObserved = new Promise(res => { + Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() { + Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish); + + log.info("Adding reset-file-log observer."); + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + res(); + }); + }); + }); + + Assert.ok(await EHTestsCommon.setUp(server)); + await sync_and_validate_telem(ping => { + deepEqual(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(e => e.status).status, ENGINE_UNKNOWN_FAIL); + }); + + await promiseObserved; + + _("Status.engines: " + JSON.stringify(Status.engines)); + Assert.equal(Status.engines.catapult, ENGINE_UNKNOWN_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // lastSync should update on partial failure. + Assert.notEqual(lastSync, Svc.PrefBranch.getStringPref("lastSync")); + + // Test Error log was written on SYNC_FAILED_PARTIAL. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); + + await promiseStopServer(server); +}); + +add_task(async function test_logs_on_sync_error() { + enableValidationPrefs(); + + _( + "Ensure that an error is still logged when weave:service:sync:error " + + "is notified, despite shouldReportError returning false." + ); + + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + log.info("TESTING"); + + // Ensure that we report no error. + Status.login = MASTER_PASSWORD_LOCKED; + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + Svc.Obs.notify("weave:service:sync:error", {}); + await promiseObserved; + + // Test that error log was written. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); +}); + +add_task(async function test_logs_on_login_error() { + enableValidationPrefs(); + + _( + "Ensure that an error is still logged when weave:service:login:error " + + "is notified, despite shouldReportError returning false." + ); + + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + log.info("TESTING"); + + // Ensure that we report no error. + Status.login = MASTER_PASSWORD_LOCKED; + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + Svc.Obs.notify("weave:service:login:error", {}); + await promiseObserved; + + // Test that error log was written. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); +}); + +// This test should be the last one since it monkeypatches the engine object +// and we should only have one engine object throughout the file (bug 629664). +add_task(async function test_engine_applyFailed() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + + engine.enabled = true; + delete engine.exception; + engine.sync = async function sync() { + Svc.Obs.notify("weave:engine:sync:applied", { newFailed: 1 }, "catapult"); + }; + + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + + Assert.equal(Status.engines.catapult, undefined); + Assert.ok(await EHTestsCommon.setUp(server)); + await Service.sync(); + await promiseObserved; + + Assert.equal(Status.engines.catapult, ENGINE_APPLY_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // Test Error log was written on SYNC_FAILED_PARTIAL. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_errorhandler_filelog.js b/services/sync/tests/unit/test_errorhandler_filelog.js new file mode 100644 index 0000000000..66260b3f59 --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_filelog.js @@ -0,0 +1,473 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { logManager } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsCommon.sys.mjs" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"]); +logsdir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); + +// Delay to wait before cleanup, to allow files to age. +// This is so large because the file timestamp granularity is per-second, and +// so otherwise we can end up with all of our files -- the ones we want to +// keep, and the ones we want to clean up -- having the same modified time. +const CLEANUP_DELAY = 2000; +const DELAY_BUFFER = 500; // Buffer for timers on different OS platforms. + +function run_test() { + validate_all_future_pings(); + run_next_test(); +} + +add_test(function test_noOutput() { + // Ensure that the log appender won't print anything. + logManager._fileAppender.level = Log.Level.Fatal + 1; + + // Clear log output from startup. + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false); + Svc.Obs.notify("weave:service:sync:finish"); + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogOuter() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogOuter); + // Clear again without having issued any output. + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", true); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogInner() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogInner); + + logManager._fileAppender.level = Log.Level.Trace; + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); + }); +}); + +add_test(function test_logOnSuccess_false() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); +}); + +function readFile(file, callback) { + NetUtil.asyncFetch( + { + uri: NetUtil.newURI(file), + loadUsingSystemPrincipal: true, + }, + function (inputStream, statusCode, request) { + let data = NetUtil.readInputStreamToString( + inputStream, + inputStream.available() + ); + callback(statusCode, data); + } + ); +} + +add_test(function test_logOnSuccess_true() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("success-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_sync_error_logOnError_false() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +add_test(function test_sync_error_logOnError_true() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +add_test(function test_login_error_logOnError_false() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + + // Fake an unsuccessful login. + Svc.Obs.notify("weave:service:login:error"); +}); + +add_test(function test_login_error_logOnError_true() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + }); + + // Fake an unsuccessful login. + Svc.Obs.notify("weave:service:login:error"); +}); + +add_test(function test_noNewFailed_noErrorLog() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + // failed is nonzero and newFailed is zero -- shouldn't write a log. + let count = { + applied: 8, + succeeded: 4, + failed: 5, + newFailed: 0, + reconciled: 4, + }; + Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine"); + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_newFailed_errorLog() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnSuccess", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up 2"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + }); + // newFailed is nonzero -- should write a log. + let count = { + applied: 8, + succeeded: 4, + failed: 5, + newFailed: 4, + reconciled: 4, + }; + + Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine"); + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_errorLog_dumpAddons() { + Svc.PrefBranch.setStringPref("log.logger", "Trace"); + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure we logged some addon list (which is probably empty) + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf("Addons installed"), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + }); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +// Check that error log files are deleted above an age threshold. +add_test(async function test_logErrorCleanup_age() { + _("Beginning test_logErrorCleanup_age."); + let maxAge = CLEANUP_DELAY / 1000; + let oldLogs = []; + let numLogs = 10; + let errString = "some error log\n"; + + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + Svc.PrefBranch.setIntPref("log.appender.file.maxErrorAge", maxAge); + + _("Making some files."); + const logsDir = PathUtils.join(PathUtils.profileDir, "weave", "logs"); + await IOUtils.makeDirectory(logsDir); + for (let i = 0; i < numLogs; i++) { + let now = Date.now(); + let filename = "error-sync-" + now + "" + i + ".txt"; + let newLog = new FileUtils.File(PathUtils.join(logsDir, filename)); + let foStream = FileUtils.openFileOutputStream(newLog); + foStream.write(errString, errString.length); + foStream.close(); + _(" > Created " + filename); + oldLogs.push(newLog.leafName); + } + + Svc.Obs.add( + "services-tests:common:log-manager:cleanup-logs", + function onCleanupLogs() { + Svc.Obs.remove( + "services-tests:common:log-manager:cleanup-logs", + onCleanupLogs + ); + + // Only the newest created log file remains. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.ok( + oldLogs.every(function (e) { + return e != logfile.leafName; + }) + ); + Assert.ok(!entries.hasMoreElements()); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + run_next_test(); + } + ); + + let delay = CLEANUP_DELAY + DELAY_BUFFER; + + _("Cleaning up logs after " + delay + "msec."); + CommonUtils.namedTimer( + function onTimer() { + Svc.Obs.notify("weave:service:sync:error"); + }, + delay, + this, + "cleanup-timer" + ); +}); + +add_task(async function test_remove_log_on_startOver() { + Svc.PrefBranch.setBoolPref("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + let promiseLogWritten = promiseOneObserver("weave:service:reset-file-log"); + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); + + await promiseLogWritten; + // Should have at least 1 log file. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + + // Fake a reset. + let promiseRemoved = promiseOneObserver("weave:service:remove-file-log"); + Svc.Obs.notify("weave:service:start-over:finish"); + await promiseRemoved; + + // should be no files left. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); +}); diff --git a/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js b/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js new file mode 100644 index 0000000000..d73d548cc7 --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js @@ -0,0 +1,294 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { FakeCryptoService } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/fakeservices.sys.mjs" +); + +var engineManager = Service.engineManager; + +function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); +} +CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + throw this.exception; + }, +}; +Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + +async function sync_httpd_setup() { + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let catapultEngine = engineManager.get("catapult"); + let syncID = await catapultEngine.resetLocalSyncID(); + let engines = { catapult: { version: catapultEngine.version, syncID } }; + + // Track these using the collections helper, which keeps modified times + // up-to-date. + let clientsColl = new ServerCollection({}, true); + let keysWBO = new ServerWBO("keys"); + let globalWBO = new ServerWBO("global", { + storageVersion: STORAGE_VERSION, + syncID: Utils.makeGUID(), + engines, + }); + + let handlers = { + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/meta/global": upd("meta", globalWBO.handler()), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + }; + return httpd_setup(handlers); +} + +async function setUp(server) { + await configureIdentity({ username: "johndoe" }, server); + new FakeCryptoService(); + syncTestLogging(); +} + +async function generateAndUploadKeys(server) { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let res = Service.resource( + server.baseURI + "/1.1/johndoe/storage/crypto/keys" + ); + return (await serverKeys.upload(res)).success; +} + +add_task(async function setup() { + await engineManager.clear(); + validate_all_future_pings(); + await engineManager.register(CatapultEngine); +}); + +add_task(async function test_backoff500() { + enableValidationPrefs(); + + _("Test: HTTP 500 sets backoff status."); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 500 }; + + try { + Assert.ok(!Status.enforceBackoff); + + // Forcibly create and upload keys here -- otherwise we don't get to the 500! + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + Assert.ok(Status.enforceBackoff); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetBackoff(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_backoff503() { + enableValidationPrefs(); + + _( + "Test: HTTP 503 with Retry-After header leads to backoff notification and sets backoff status." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + const BACKOFF = 42; + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + let backoffInterval; + Svc.Obs.add("weave:service:backoff:interval", function (subject) { + backoffInterval = subject; + }); + + try { + Assert.ok(!Status.enforceBackoff); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, BACKOFF); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + } finally { + Status.resetBackoff(); + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_overQuota() { + enableValidationPrefs(); + + _("Test: HTTP 400 with body error code 14 means over quota."); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { + status: 400, + toString() { + return "14"; + }, + }; + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, OVER_QUOTA); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_service_networkError() { + enableValidationPrefs(); + + _( + "Test: Connection refused error from Service.sync() leads to the right status code." + ); + let server = await sync_httpd_setup(); + await setUp(server); + await promiseStopServer(server); + // Provoke connection refused. + Service.clusterURL = "http://localhost:12345/"; + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Service._loggedIn = true; + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED); + } finally { + Status.resetSync(); + await Service.startOver(); + } +}); + +add_task(async function test_service_offline() { + enableValidationPrefs(); + + _( + "Test: Wanting to sync in offline mode leads to the right status code but does not increment the ignorable error count." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + await promiseStopServer(server); + Services.io.offline = true; + Services.prefs.setBoolPref("network.dns.offline-localhost", false); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Service._loggedIn = true; + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED); + } finally { + Status.resetSync(); + await Service.startOver(); + } + Services.io.offline = false; + Services.prefs.clearUserPref("network.dns.offline-localhost"); +}); + +add_task(async function test_engine_networkError() { + enableValidationPrefs(); + + _( + "Test: Network related exceptions from engine.sync() lead to the right status code." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = Components.Exception( + "NS_ERROR_UNKNOWN_HOST", + Cr.NS_ERROR_UNKNOWN_HOST + ); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_resource_timeout() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + // Resource throws this when it encounters a timeout. + engine.exception = Components.Exception( + "Aborting due to channel inactivity.", + Cr.NS_ERROR_NET_TIMEOUT + ); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_extension_storage_engine.js b/services/sync/tests/unit/test_extension_storage_engine.js new file mode 100644 index 0000000000..a061812aca --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_engine.js @@ -0,0 +1,275 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + Service: "resource://services-sync/service.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", +}); + +const { ExtensionStorageEngineBridge, ExtensionStorageEngineKinto } = + ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" + ); + +const { BridgeWrapperXPCOM } = ChromeUtils.importESModule( + "resource://services-sync/bridged_engine.sys.mjs" +); + +Services.prefs.setStringPref("webextensions.storage.sync.log.level", "debug"); + +add_task(async function test_switching_between_kinto_and_bridged() { + function assertUsingKinto(message) { + let kintoEngine = Service.engineManager.get("extension-storage"); + Assert.ok(kintoEngine instanceof ExtensionStorageEngineKinto, message); + } + function assertUsingBridged(message) { + let bridgedEngine = Service.engineManager.get("extension-storage"); + Assert.ok(bridgedEngine instanceof ExtensionStorageEngineBridge, message); + } + + let isUsingKinto = Services.prefs.getBoolPref( + "webextensions.storage.sync.kinto", + false + ); + if (isUsingKinto) { + assertUsingKinto("Should use Kinto engine before flipping pref"); + } else { + assertUsingBridged("Should use bridged engine before flipping pref"); + } + + _("Flip pref"); + Services.prefs.setBoolPref("webextensions.storage.sync.kinto", !isUsingKinto); + await Service.engineManager.switchAlternatives(); + + if (isUsingKinto) { + assertUsingBridged("Should use bridged engine after flipping pref"); + } else { + assertUsingKinto("Should use Kinto engine after flipping pref"); + } + + _("Clean up"); + Services.prefs.clearUserPref("webextensions.storage.sync.kinto"); + await Service.engineManager.switchAlternatives(); +}); + +add_task(async function test_enable() { + const PREF = "services.sync.engine.extension-storage.force"; + + let addonsEngine = Service.engineManager.get("addons"); + let extensionStorageEngine = Service.engineManager.get("extension-storage"); + + try { + Assert.ok( + addonsEngine.enabled, + "Add-ons engine should be enabled by default" + ); + Assert.ok( + extensionStorageEngine.enabled, + "Extension storage engine should be enabled by default" + ); + + addonsEngine.enabled = false; + Assert.ok( + !extensionStorageEngine.enabled, + "Disabling add-ons should disable extension storage" + ); + + extensionStorageEngine.enabled = true; + Assert.ok( + !extensionStorageEngine.enabled, + "Enabling extension storage without override pref shouldn't work" + ); + + Services.prefs.setBoolPref(PREF, true); + Assert.ok( + extensionStorageEngine.enabled, + "Setting override pref should enable extension storage" + ); + + extensionStorageEngine.enabled = false; + Assert.ok( + !extensionStorageEngine.enabled, + "Disabling extension storage engine with override pref should work" + ); + + extensionStorageEngine.enabled = true; + Assert.ok( + extensionStorageEngine.enabled, + "Enabling extension storage with override pref should work" + ); + } finally { + addonsEngine.enabled = true; + Services.prefs.clearUserPref(PREF); + } +}); + +add_task(async function test_notifyPendingChanges() { + let engine = new ExtensionStorageEngineBridge(Service); + + let extension = { id: "ext-1" }; + let expectedChange = { + a: "b", + c: "d", + }; + + let lastSync = 0; + let syncID = Utils.makeGUID(); + let error = null; + engine.component = { + QueryInterface: ChromeUtils.generateQI([ + "mozIBridgedSyncEngine", + "mozIExtensionStorageArea", + "mozISyncedExtensionStorageArea", + ]), + ensureCurrentSyncId(id, callback) { + if (syncID != id) { + syncID = id; + lastSync = 0; + } + callback.handleSuccess(id); + }, + resetSyncId(callback) { + callback.handleSuccess(syncID); + }, + syncStarted(callback) { + callback.handleSuccess(); + }, + getLastSync(callback) { + callback.handleSuccess(lastSync); + }, + setLastSync(lastSyncMillis, callback) { + lastSync = lastSyncMillis; + callback.handleSuccess(); + }, + apply(callback) { + callback.handleSuccess([]); + }, + fetchPendingSyncChanges(callback) { + if (error) { + callback.handleError(Cr.NS_ERROR_FAILURE, error.message); + } else { + callback.onChanged(extension.id, JSON.stringify(expectedChange)); + callback.handleSuccess(); + } + }, + setUploaded(modified, ids, callback) { + callback.handleSuccess(); + }, + syncFinished(callback) { + callback.handleSuccess(); + }, + takeMigrationInfo(callback) { + callback.handleSuccess(null); + }, + }; + + engine._bridge = new BridgeWrapperXPCOM(engine.component); + + let server = await serverForFoo(engine); + + let actualChanges = []; + let listener = changes => actualChanges.push(changes); + extensionStorageSync.addOnChangedListener(extension, listener); + + try { + await SyncTestingInfrastructure(server); + + info("Sync engine; notify about changes"); + await sync_engine_and_validate_telem(engine, false); + deepEqual( + actualChanges, + [expectedChange], + "Should notify about changes during sync" + ); + + error = new Error("oops!"); + actualChanges = []; + await sync_engine_and_validate_telem(engine, false); + deepEqual( + actualChanges, + [], + "Should finish syncing even if notifying about changes fails" + ); + } finally { + extensionStorageSync.removeOnChangedListener(extension, listener); + await promiseStopServer(server); + await engine.finalize(); + } +}); + +// It's difficult to know what to test - there's already tests for the bridged +// engine etc - so we just try and check that this engine conforms to the +// mozIBridgedSyncEngine interface guarantees. +add_task(async function test_engine() { + // Forcibly set the bridged engine in the engine manager. the reason we do + // this, unlike the other tests where we just create the engine, is so that + // telemetry can get at the engine's `overrideTelemetryName`, which it gets + // through the engine manager. + await Service.engineManager.unregister("extension-storage"); + await Service.engineManager.register(ExtensionStorageEngineBridge); + let engine = Service.engineManager.get("extension-storage"); + Assert.equal(engine.version, 1); + + Assert.deepEqual(await engine.getSyncID(), null); + await engine.resetLocalSyncID(); + Assert.notEqual(await engine.getSyncID(), null); + + Assert.equal(await engine.getLastSync(), 0); + // lastSync is seconds on this side of the world, but milli-seconds on the other. + await engine.setLastSync(1234.567); + // should have 2 digit precision. + Assert.equal(await engine.getLastSync(), 1234.57); + await engine.setLastSync(0); + + // Set some data. + await extensionStorageSync.set({ id: "ext-2" }, { ext_2_key: "ext_2_value" }); + // Now do a sync with out regular test server. + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + + info("Add server records"); + let foo = server.user("foo"); + let collection = foo.collection("extension-storage"); + let now = new_timestamp(); + + collection.insert( + "fakeguid0000", + encryptPayload({ + id: "fakeguid0000", + extId: "ext-1", + data: JSON.stringify({ foo: "bar" }), + }), + now + ); + + info("Sync the engine"); + + let ping = await sync_engine_and_validate_telem(engine, false); + Assert.ok(ping.engines.find(e => e.name == "rust-webext-storage")); + Assert.equal( + ping.engines.find(e => e.name == "extension-storage"), + null + ); + + // We should have applied the data from the existing collection record. + Assert.deepEqual(await extensionStorageSync.get({ id: "ext-1" }, null), { + foo: "bar", + }); + + // should now be 2 records on the server. + let payloads = collection.payloads(); + Assert.equal(payloads.length, 2); + // find the new one we wrote. + let newPayload = + payloads[0].id == "fakeguid0000" ? payloads[1] : payloads[0]; + Assert.equal(newPayload.data, `{"ext_2_key":"ext_2_value"}`); + // should have updated the timestamp. + greater(await engine.getLastSync(), 0, "Should update last sync time"); + } finally { + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_extension_storage_engine_kinto.js b/services/sync/tests/unit/test_extension_storage_engine_kinto.js new file mode 100644 index 0000000000..b074fe376c --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_engine_kinto.js @@ -0,0 +1,136 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true); + +const { ExtensionStorageEngineKinto: ExtensionStorageEngine } = + ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" + ); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { extensionStorageSyncKinto: extensionStorageSync } = + ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" + ); + +let engine; + +function mock(options) { + let calls = []; + let ret = function () { + calls.push(arguments); + return options.returns; + }; + let proto = { + get calls() { + return calls; + }, + }; + Object.setPrototypeOf(proto, Function.prototype); + Object.setPrototypeOf(ret, proto); + return ret; +} + +function setSkipChance(v) { + Services.prefs.setIntPref( + "services.sync.extension-storage.skipPercentageChance", + v + ); +} + +add_task(async function setup() { + await Service.engineManager.register(ExtensionStorageEngine); + engine = Service.engineManager.get("extension-storage"); + do_get_profile(); // so we can use FxAccounts + loadWebExtensionTestFunctions(); + setSkipChance(0); +}); + +add_task(async function test_calling_sync_calls__sync() { + let oldSync = ExtensionStorageEngine.prototype._sync; + let syncMock = (ExtensionStorageEngine.prototype._sync = mock({ + returns: true, + })); + try { + // I wanted to call the main sync entry point for the entire + // package, but that fails because it tries to sync ClientEngine + // first, which fails. + await engine.sync(); + } finally { + ExtensionStorageEngine.prototype._sync = oldSync; + } + equal(syncMock.calls.length, 1); +}); + +add_task(async function test_sync_skip() { + try { + // Do a few times to ensure we aren't getting "lucky" WRT Math.random() + for (let i = 0; i < 10; ++i) { + setSkipChance(100); + engine._tracker._score = 0; + ok( + !engine.shouldSkipSync("user"), + "Should allow explicitly requested syncs" + ); + ok(!engine.shouldSkipSync("startup"), "Should allow startup syncs"); + ok( + engine.shouldSkipSync("schedule"), + "Should skip scheduled syncs if skipProbability is 100" + ); + engine._tracker._score = MULTI_DEVICE_THRESHOLD; + ok( + !engine.shouldSkipSync("schedule"), + "should allow scheduled syncs if tracker score is high" + ); + engine._tracker._score = 0; + setSkipChance(0); + ok( + !engine.shouldSkipSync("schedule"), + "Should allow scheduled syncs if probability is 0" + ); + } + } finally { + engine._tracker._score = 0; + setSkipChance(0); + } +}); + +add_task(async function test_calling_wipeClient_calls_clearAll() { + let oldClearAll = extensionStorageSync.clearAll; + let clearMock = (extensionStorageSync.clearAll = mock({ + returns: Promise.resolve(), + })); + try { + await engine.wipeClient(); + } finally { + extensionStorageSync.clearAll = oldClearAll; + } + equal(clearMock.calls.length, 1); +}); + +add_task(async function test_calling_sync_calls_ext_storage_sync() { + const extension = { id: "my-extension" }; + let oldSync = extensionStorageSync.syncAll; + let syncMock = (extensionStorageSync.syncAll = mock({ + returns: Promise.resolve(), + })); + try { + await withSyncContext(async function (context) { + // Set something so that everyone knows that we're using storage.sync + await extensionStorageSync.set(extension, { a: "b" }, context); + let ping = await sync_engine_and_validate_telem(engine, false); + Assert.ok(ping.engines.find(e => e.name == "extension-storage")); + Assert.equal( + ping.engines.find(e => e.name == "rust-webext-storage"), + null + ); + }); + } finally { + extensionStorageSync.syncAll = oldSync; + } + Assert.ok(syncMock.calls.length >= 1); +}); diff --git a/services/sync/tests/unit/test_extension_storage_migration_telem.js b/services/sync/tests/unit/test_extension_storage_migration_telem.js new file mode 100644 index 0000000000..a4b4c95f55 --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_migration_telem.js @@ -0,0 +1,81 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Import the rust-based and kinto-based implementations. Not great to grab +// these as they're somewhat private, but we want to run the pings through our +// validation machinery which is here in the sync test code. +const { extensionStorageSync: rustImpl } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSync.sys.mjs" +); +const { extensionStorageSyncKinto: kintoImpl } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" +); + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { ExtensionStorageEngineBridge } = ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" +); + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", false); +Services.prefs.setStringPref("webextensions.storage.sync.log.level", "debug"); + +// It's tricky to force error cases here (the databases are opened with +// exclusive locks) and that part of the code has coverage in the vendored +// application-services webext-storage crate. So this just tests that the +// migration data ends up in the ping, and exactly once. +add_task(async function test_sync_migration_telem() { + // Set some stuff using the kinto-based impl prior to fully setting up sync. + let e1 = { id: "test@mozilla.com" }; + let c1 = { extension: e1, callOnClose() {} }; + + let e2 = { id: "test-2@mozilla.com" }; + let c2 = { extension: e2, callOnClose() {} }; + await kintoImpl.set(e1, { foo: "bar" }, c1); + await kintoImpl.set(e1, { baz: "quux" }, c1); + await kintoImpl.set(e2, { second: "2nd" }, c2); + + Assert.deepEqual(await rustImpl.get(e1, "foo", c1), { foo: "bar" }); + Assert.deepEqual(await rustImpl.get(e1, "baz", c1), { baz: "quux" }); + Assert.deepEqual(await rustImpl.get(e2, null, c2), { second: "2nd" }); + + // Explicitly unregister first. It's very possible this isn't needed for this + // case, however it's fairly harmless, we hope to uplift this patch to beta, + // and earlier today we had beta-only problems caused by this (bug 1629116) + await Service.engineManager.unregister("extension-storage"); + await Service.engineManager.register(ExtensionStorageEngineBridge); + let engine = Service.engineManager.get("extension-storage"); + let server = await serverForFoo(engine, undefined); + try { + await SyncTestingInfrastructure(server); + await Service.engineManager.switchAlternatives(); + + _("First sync"); + let ping = await sync_engine_and_validate_telem(engine, false, null, true); + Assert.deepEqual(ping.migrations, [ + { + type: "webext-storage", + entries: 3, + entriesSuccessful: 3, + extensions: 2, + extensionsSuccessful: 2, + openFailure: false, + }, + ]); + + // force another sync + await engine.setLastSync(0); + _("Second sync"); + + ping = await sync_engine_and_validate_telem(engine, false, null, true); + Assert.deepEqual(ping.migrations, undefined); + } finally { + await kintoImpl.clear(e1, c1); + await kintoImpl.clear(e2, c2); + await rustImpl.clear(e1, c1); + await rustImpl.clear(e2, c2); + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_extension_storage_tracker_kinto.js b/services/sync/tests/unit/test_extension_storage_tracker_kinto.js new file mode 100644 index 0000000000..2de56ae400 --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_tracker_kinto.js @@ -0,0 +1,44 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true); + +const { ExtensionStorageEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { extensionStorageSyncKinto: extensionStorageSync } = + ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" + ); + +let engine; + +add_task(async function setup() { + await Service.engineManager.register(ExtensionStorageEngine); + engine = Service.engineManager.get("extension-storage"); + do_get_profile(); // so we can use FxAccounts + loadWebExtensionTestFunctions(); +}); + +add_task(async function test_changing_extension_storage_changes_score() { + const tracker = engine._tracker; + const extension = { id: "my-extension-id" }; + tracker.start(); + await withSyncContext(async function (context) { + await extensionStorageSync.set(extension, { a: "b" }, context); + }); + Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM); + + tracker.resetScore(); + await withSyncContext(async function (context) { + await extensionStorageSync.remove(extension, "a", context); + }); + Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM); + + await tracker.stop(); +}); diff --git a/services/sync/tests/unit/test_form_validator.js b/services/sync/tests/unit/test_form_validator.js new file mode 100644 index 0000000000..58ea8b855b --- /dev/null +++ b/services/sync/tests/unit/test_form_validator.js @@ -0,0 +1,86 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FormValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "11111", + guid: "11111", + name: "foo", + fieldname: "foo", + value: "bar", + }, + { + id: "22222", + guid: "22222", + name: "foo2", + fieldname: "foo2", + value: "bar2", + }, + { + id: "33333", + guid: "33333", + name: "foo3", + fieldname: "foo3", + value: "bar3", + }, + ], + client: [ + { + id: "11111", + guid: "11111", + name: "foo", + fieldname: "foo", + value: "bar", + }, + { + id: "22222", + guid: "22222", + name: "foo2", + fieldname: "foo2", + value: "bar2", + }, + { + id: "33333", + guid: "33333", + name: "foo3", + fieldname: "foo3", + value: "bar3", + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new FormValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 3); + equal(records.length, 3); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); + +add_task(async function test_formValidatorIgnoresMissingClients() { + // Since history form records are not deleted from the server, the + // |FormValidator| shouldn't set the |missingClient| flag in |problemData|. + let { server, client } = getDummyServerAndClient(); + client.pop(); + + let validator = new FormValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 2); + equal(records.length, 3); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + deepEqual(problemData, expected); +}); diff --git a/services/sync/tests/unit/test_forms_store.js b/services/sync/tests/unit/test_forms_store.js new file mode 100644 index 0000000000..716487865f --- /dev/null +++ b/services/sync/tests/unit/test_forms_store.js @@ -0,0 +1,176 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Make sure the form store follows the Store api and correctly accesses the backend form storage" +); +const { FormEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +add_task(async function run_test() { + let engine = new FormEngine(Service); + await engine.initialize(); + let store = engine._store; + + async function applyEnsureNoFailures(records) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch(records, countTelemetry)).length, + 0 + ); + } + + _("Remove any existing entries"); + await store.wipe(); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add a form entry"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "name!!", + value: "value??", + }, + ]); + + _("Should have 1 entry now"); + let id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + Assert.ok(store.itemExists(id)); + + _("Should be able to find this entry as a dupe"); + Assert.equal( + await engine._findDupe({ name: "name!!", value: "value??" }), + id + ); + + let rec = await store.createRecord(id); + _("Got record for id", id, rec); + Assert.equal(rec.name, "name!!"); + Assert.equal(rec.value, "value??"); + + _("Create a non-existent id for delete"); + Assert.ok((await store.createRecord("deleted!!")).deleted); + + _("Try updating.. doesn't do anything yet"); + await store.update({}); + + _("Remove all entries"); + await store.wipe(); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add another entry"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "another", + value: "entry", + }, + ]); + id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + + _("Change the id of the new entry to something else"); + await store.changeItemID(id, "newid"); + + _("Make sure it's there"); + Assert.ok(store.itemExists("newid")); + + _("Remove the entry"); + await store.remove({ + id: "newid", + }); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Removing the entry again shouldn't matter"); + await store.remove({ + id: "newid", + }); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add another entry to delete using applyIncomingBatch"); + let toDelete = { + id: Utils.makeGUID(), + name: "todelete", + value: "entry", + }; + await applyEnsureNoFailures([toDelete]); + id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + Assert.ok(store.itemExists(id)); + // mark entry as deleted + toDelete.id = id; + toDelete.deleted = true; + await applyEnsureNoFailures([toDelete]); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add an entry to wipe"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "towipe", + value: "entry", + }, + ]); + + await store.wipe(); + + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Ensure we work if formfill is disabled."); + Services.prefs.setBoolPref("browser.formfill.enable", false); + try { + // a search + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + // an update. + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "some", + value: "entry", + }, + ]); + } finally { + Services.prefs.clearUserPref("browser.formfill.enable"); + await store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_forms_tracker.js b/services/sync/tests/unit/test_forms_tracker.js new file mode 100644 index 0000000000..aee74381ad --- /dev/null +++ b/services/sync/tests/unit/test_forms_tracker.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FormEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + _("Verify we've got an empty tracker to work with."); + let engine = new FormEngine(Service); + await engine.initialize(); + let tracker = engine._tracker; + + let changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + async function addEntry(name, value) { + await engine._store.create({ name, value }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + async function removeEntry(name, value) { + let guid = await engine._findDupe({ name, value }); + await engine._store.remove({ id: guid }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + + try { + _("Create an entry. Won't show because we haven't started tracking yet"); + await addEntry("name", "John Doe"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + await removeEntry("name", "John Doe"); + await addEntry("email", "john@doe.com"); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 2); + + _("Notifying twice won't do any harm."); + tracker.start(); + await addEntry("address", "Memory Lane"); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 3); + + _("Check that ignoreAll is respected"); + await tracker.clearChangedIDs(); + tracker.score = 0; + tracker.ignoreAll = true; + await addEntry("username", "johndoe123"); + await addEntry("favoritecolor", "green"); + await removeEntry("name", "John Doe"); + tracker.ignoreAll = false; + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + equal(tracker.score, 0); + + _("Let's stop tracking again."); + await tracker.clearChangedIDs(); + await tracker.stop(); + await removeEntry("address", "Memory Lane"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + _("Notifying twice won't do any harm."); + await tracker.stop(); + await removeEntry("email", "john@doe.com"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + } finally { + _("Clean up."); + await engine._store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_fxa_node_reassignment.js b/services/sync/tests/unit/test_fxa_node_reassignment.js new file mode 100644 index 0000000000..0b25df0183 --- /dev/null +++ b/services/sync/tests/unit/test_fxa_node_reassignment.js @@ -0,0 +1,399 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Test that node reassignment happens correctly using the FxA identity mgr."); +// The node-reassignment logic is quite different for FxA than for the legacy +// provider. In particular, there's no special request necessary for +// reassignment - it comes from the token server - so we need to ensure the +// Fxa cluster manager grabs a new token. + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); + +add_task(async function setup() { + // Disables all built-in engines. Important for avoiding errors thrown by the + // add-ons engine. + await Service.engineManager.clear(); + + // Setup the sync auth manager. + Status.__authManager = Service.identity = new SyncAuthManager(); +}); + +// API-compatible with SyncServer handler. Bind `handler` to something to use +// as a ServerCollection handler. +function handleReassign(handler, req, resp) { + resp.setStatusLine(req.httpVersion, 401, "Node reassignment"); + resp.setHeader("Content-Type", "application/json"); + let reassignBody = JSON.stringify({ error: "401inator in place" }); + resp.bodyOutputStream.write(reassignBody, reassignBody.length); +} + +var numTokenRequests = 0; + +function prepareServer(cbAfterTokenFetch) { + syncTestLogging(); + let config = makeIdentityConfig({ username: "johndoe" }); + // A server callback to ensure we don't accidentally hit the wrong endpoint + // after a node reassignment. + let callback = { + onRequest(req, resp) { + let full = `${req.scheme}://${req.host}:${req.port}${req.path}`; + let expected = config.fxaccount.token.endpoint; + Assert.ok( + full.startsWith(expected), + `request made to ${full}, expected ${expected}` + ); + }, + }; + Object.setPrototypeOf(callback, SyncServerCallback); + let server = new SyncServer(callback); + server.registerUser("johndoe"); + server.start(); + + // Set the token endpoint for the initial token request that's done implicitly + // via configureIdentity. + config.fxaccount.token.endpoint = server.baseURI + "1.1/johndoe/"; + // And future token fetches will do magic around numReassigns. + let numReassigns = 0; + return configureIdentity(config).then(() => { + Service.identity._tokenServerClient = { + getTokenUsingOAuth() { + return new Promise(res => { + // Build a new URL with trailing zeros for the SYNC_VERSION part - this + // will still be seen as equivalent by the test server, but different + // by sync itself. + numReassigns += 1; + let trailingZeros = new Array(numReassigns + 1).join("0"); + let token = config.fxaccount.token; + token.endpoint = server.baseURI + "1.1" + trailingZeros + "/johndoe"; + token.uid = config.username; + _(`test server saw token fetch - endpoint now ${token.endpoint}`); + numTokenRequests += 1; + res(token); + if (cbAfterTokenFetch) { + cbAfterTokenFetch(); + } + }); + }, + }; + return server; + }); +} + +function getReassigned() { + try { + return Services.prefs.getBoolPref("services.sync.lastSyncReassigned"); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_UNEXPECTED) { + do_throw( + "Got exception retrieving lastSyncReassigned: " + Log.exceptionStr(ex) + ); + } + } + return false; +} + +/** + * Make a test request to `url`, then watch the result of two syncs + * to ensure that a node request was made. + * Runs `between` between the two. This can be used to undo deliberate failure + * setup, detach observers, etc. + */ +async function syncAndExpectNodeReassignment( + server, + firstNotification, + between, + secondNotification, + url +) { + _("Starting syncAndExpectNodeReassignment\n"); + let deferred = Promise.withResolvers(); + async function onwards() { + let numTokenRequestsBefore; + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // Track whether we fetched a new token. + numTokenRequestsBefore = numTokenRequests; + + // Allow for tests to clean up error conditions. + between(); + } + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Second sync nextTick."); + Assert.equal( + numTokenRequests, + numTokenRequestsBefore + 1, + "fetched a new token" + ); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + await Service.sync(); + } + + // Make sure that we really do get a 401 (but we can only do that if we are + // already logged in, as the login process is what sets up the URLs) + if (Service.isLoggedIn) { + _("Making request to " + url + " which should 401"); + let request = new RESTRequest(url); + await request.get(); + Assert.equal(request.response.status, 401); + CommonUtils.nextTick(onwards); + } else { + _("Skipping preliminary validation check for a 401 as we aren't logged in"); + CommonUtils.nextTick(onwards); + } + await deferred.promise; +} + +// Check that when we sync we don't request a new token by default - our +// test setup has configured the client with a valid token, and that token +// should be used to form the cluster URL. +add_task(async function test_single_token_fetch() { + enableValidationPrefs(); + + _("Test a normal sync only fetches 1 token"); + + let numTokenFetches = 0; + + function afterTokenFetch() { + numTokenFetches++; + } + + // Set the cluster URL to an "old" version - this is to ensure we don't + // use that old cached version for the first sync but prefer the value + // we got from the token (and as above, we are also checking we don't grab + // a new token). If the test actually attempts to connect to this URL + // it will crash. + Service.clusterURL = "http://example.com/"; + + let server = await prepareServer(afterTokenFetch); + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded"); + Assert.equal(numTokenFetches, 0, "didn't fetch a new token"); + // A bit hacky, but given we know how prepareServer works we can deduce + // that clusterURL we expect. + let expectedClusterURL = server.baseURI + "1.1/johndoe/"; + Assert.equal(Service.clusterURL, expectedClusterURL); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_momentary_401_engine() { + enableValidationPrefs(); + + _("Test a failure for engine URLs that's resolved by reassignment."); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // We want to verify that the clusterURL pref has been cleared after a 401 + // inside a sync. Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function between() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Assert.ok(getReassigned()); + } + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:finish", + between, + "weave:service:sync:finish", + Service.storageURL + "rotary" + ); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +// This test ends up being a failing info fetch *after we're already logged in*. +add_task(async function test_momentary_401_info_collections_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for info/collections after login that's resolved by reassignment." + ); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Arrange for info/collections to return a 401."); + let oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.info = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.infoURL + ); +}); + +// This test ends up being a failing info fetch *before we're logged in*. +// In this case we expect to recover during the login phase - so the first +// sync succeeds. +add_task(async function test_momentary_401_info_collections_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for info/collections before login that's resolved by reassignment." + ); + + let oldHandler; + let sawTokenFetch = false; + + function afterTokenFetch() { + // After a single token fetch, we undo our evil handleReassign hack, so + // the next /info request returns the collection instead of a 401 + server.toplevelHandlers.info = oldHandler; + sawTokenFetch = true; + } + + let server = await prepareServer(afterTokenFetch); + + // Return a 401 for the next /info request - it will be reset immediately + // after a new token is fetched. + oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded"); + // sync was successful - check we grabbed a new token. + Assert.ok(sawTokenFetch, "a new token was fetched by this test."); + // and we are done. + await Service.startOver(); + await promiseStopServer(server); +}); + +// This test ends up being a failing meta/global fetch *after we're already logged in*. +add_task(async function test_momentary_401_storage_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL after login that's resolved by" + + "reassignment." + ); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Arrange for meta/global to return a 401."); + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +// This test ends up being a failing meta/global fetch *before we've logged in*. +add_task(async function test_momentary_401_storage_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL before login, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(!Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:login:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); diff --git a/services/sync/tests/unit/test_fxa_service_cluster.js b/services/sync/tests/unit/test_fxa_service_cluster.js new file mode 100644 index 0000000000..0203d01ef5 --- /dev/null +++ b/services/sync/tests/unit/test_fxa_service_cluster.js @@ -0,0 +1,58 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { initializeIdentityWithTokenServerResponse } = + ChromeUtils.importESModule( + "resource://testing-common/services/sync/fxa_utils.sys.mjs" + ); + +add_task(async function test_findCluster() { + _("Test FxA _findCluster()"); + + _("_findCluster() throws on 500 errors."); + initializeIdentityWithTokenServerResponse({ + status: 500, + headers: [], + body: "", + }); + + await Assert.rejects( + Service.identity._findCluster(), + /TokenServerClientServerError/ + ); + + _("_findCluster() returns null on authentication errors."); + initializeIdentityWithTokenServerResponse({ + status: 401, + headers: { "content-type": "application/json" }, + body: "{}", + }); + + let cluster = await Service.identity._findCluster(); + Assert.strictEqual(cluster, null); + + _("_findCluster() works with correct tokenserver response."); + let endpoint = "http://example.com/something"; + initializeIdentityWithTokenServerResponse({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + api_endpoint: endpoint, + duration: 300, + id: "id", + key: "key", + uid: "uid", + }), + }); + + cluster = await Service.identity._findCluster(); + // The cluster manager ensures a trailing "/" + Assert.strictEqual(cluster, endpoint + "/"); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } +}); diff --git a/services/sync/tests/unit/test_history_engine.js b/services/sync/tests/unit/test_history_engine.js new file mode 100644 index 0000000000..9cca379b0b --- /dev/null +++ b/services/sync/tests/unit/test_history_engine.js @@ -0,0 +1,429 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); + +// Use only for rawAddVisit. +XPCOMUtils.defineLazyServiceGetter( + this, + "asyncHistory", + "@mozilla.org/browser/history;1", + "mozIAsyncHistory" +); +async function rawAddVisit(id, uri, visitPRTime, transitionType) { + return new Promise((resolve, reject) => { + let results = []; + let handler = { + handleResult(result) { + results.push(result); + }, + handleError(resultCode, placeInfo) { + do_throw(`updatePlaces gave error ${resultCode}!`); + }, + handleCompletion(count) { + resolve({ results, count }); + }, + }; + asyncHistory.updatePlaces( + [ + { + guid: id, + uri: typeof uri == "string" ? CommonUtils.makeURI(uri) : uri, + visits: [{ visitDate: visitPRTime, transitionType }], + }, + ], + handler + ); + }); +} + +add_task(async function test_history_download_limit() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let lastSync = new_timestamp(); + + let collection = server.user("foo").collection("history"); + for (let i = 0; i < 15; i++) { + let id = "place" + i.toString(10).padStart(7, "0"); + let wbo = new ServerWBO( + id, + encryptPayload({ + id, + histUri: "http://example.com/" + i, + title: "Page " + i, + visits: [ + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.TYPED, + }, + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + ], + }), + lastSync + 1 + i + ); + wbo.sortindex = 15 - i; + collection.insertWBO(wbo); + } + + // We have 15 records on the server since the last sync, but our download + // limit is 5 records at a time. We should eventually fetch all 15. + await engine.setLastSync(lastSync); + engine.downloadBatchSize = 4; + engine.downloadLimit = 5; + + // Don't actually fetch any backlogged records, so that we can inspect + // the backlog between syncs. + engine.guidFetchBatchSize = 0; + + let ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + let backlogAfterFirstSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterFirstSync, [ + "place0000000", + "place0000001", + "place0000002", + "place0000003", + "place0000004", + "place0000005", + "place0000006", + "place0000007", + "place0000008", + "place0000009", + ]); + + // We should have fast-forwarded the last sync time. + equal(await engine.getLastSync(), lastSync + 15); + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + ok(!ping.engines[0].incoming); + + // After the second sync, our backlog still contains the same GUIDs: we + // weren't able to make progress on fetching them, since our + // `guidFetchBatchSize` is 0. + let backlogAfterSecondSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterFirstSync, backlogAfterSecondSync); + + // Now add a newer record to the server. + let newWBO = new ServerWBO( + "placeAAAAAAA", + encryptPayload({ + id: "placeAAAAAAA", + histUri: "http://example.com/a", + title: "New Page A", + visits: [ + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.TYPED, + }, + ], + }), + lastSync + 20 + ); + newWBO.sortindex = -1; + collection.insertWBO(newWBO); + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 1 }); + + // Our backlog should remain the same. + let backlogAfterThirdSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterSecondSync, backlogAfterThirdSync); + + equal(await engine.getLastSync(), lastSync + 20); + + // Bump the fetch batch size to let the backlog make progress. We should + // make 3 requests to fetch 5 backlogged GUIDs. + engine.guidFetchBatchSize = 2; + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + deepEqual(Array.from(engine.toFetch).sort(), [ + "place0000005", + "place0000006", + "place0000007", + "place0000008", + "place0000009", + ]); + + // Sync again to clear out the backlog. + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + deepEqual(Array.from(engine.toFetch), []); + + await engine.wipeClient(); + await engine.finalize(); +}); + +add_task(async function test_history_visit_roundtrip() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + engine._tracker.start(); + + let id = "aaaaaaaaaaaa"; + let oneHourMS = 60 * 60 * 1000; + // Insert a visit with a non-round microsecond timestamp (e.g. it's not evenly + // divisible by 1000). This will typically be the case for visits that occur + // during normal navigation. + let time = (Date.now() - oneHourMS) * 1000 + 555; + // We use the low level history api since it lets us provide microseconds + let { count } = await rawAddVisit( + id, + "https://www.example.com", + time, + PlacesUtils.history.TRANSITIONS.TYPED + ); + equal(count, 1); + // Check that it was inserted and that we didn't round on the insert. + let visits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(visits.length, 1); + equal(visits[0].date, time); + + let collection = server.user("foo").collection("history"); + + // Sync the visit up to the server. + await sync_engine_and_validate_telem(engine, false); + + collection.updateRecord( + id, + cleartext => { + // Double-check that we didn't round the visit's timestamp to the nearest + // millisecond when uploading. + equal(cleartext.visits[0].date, time); + // Add a remote visit so that we get past the deepEquals check in reconcile + // (otherwise the history engine will skip applying this record). The + // contents of this visit don't matter, beyond the fact that it needs to + // exist. + cleartext.visits.push({ + date: (Date.now() - oneHourMS / 2) * 1000, + type: PlacesUtils.history.TRANSITIONS.LINK, + }); + }, + new_timestamp() + 10 + ); + + // Force a remote sync. + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + // Make sure that we didn't duplicate the visit when inserting. (Prior to bug + // 1423395, we would insert a duplicate visit, where the timestamp was + // effectively `Math.round(microsecondTimestamp / 1000) * 1000`.) + visits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(visits.length, 2); + + await engine.wipeClient(); + await engine.finalize(); +}); + +add_task(async function test_history_visit_dedupe_old() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let initialVisits = Array.from({ length: 25 }, (_, index) => ({ + transition: PlacesUtils.history.TRANSITION_LINK, + date: new Date(Date.UTC(2017, 10, 1 + index)), + })); + initialVisits.push({ + transition: PlacesUtils.history.TRANSITION_LINK, + date: new Date(), + }); + await PlacesUtils.history.insert({ + url: "https://www.example.com", + visits: initialVisits, + }); + + let recentVisits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(recentVisits.length, 20); + let { visits: allVisits, guid } = await PlacesUtils.history.fetch( + "https://www.example.com", + { + includeVisits: true, + } + ); + equal(allVisits.length, 26); + + let collection = server.user("foo").collection("history"); + + await sync_engine_and_validate_telem(engine, false); + + collection.updateRecord( + guid, + data => { + data.visits.push( + // Add a couple remote visit equivalent to some old visits we have already + { + date: Date.UTC(2017, 10, 1) * 1000, // Nov 1, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + { + date: Date.UTC(2017, 10, 2) * 1000, // Nov 2, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + // Add a couple new visits to make sure we are still applying them. + { + date: Date.UTC(2017, 11, 4) * 1000, // Dec 4, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + { + date: Date.UTC(2017, 11, 5) * 1000, // Dec 5, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + } + ); + }, + new_timestamp() + 10 + ); + + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + allVisits = ( + await PlacesUtils.history.fetch("https://www.example.com", { + includeVisits: true, + }) + ).visits; + + equal(allVisits.length, 28); + ok( + allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 4)), + "Should contain the Dec. 4th visit" + ); + ok( + allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 5)), + "Should contain the Dec. 5th visit" + ); + + await engine.wipeClient(); + await engine.finalize(); +}); + +add_task(async function test_history_unknown_fields() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + engine._tracker.start(); + + let id = "aaaaaaaaaaaa"; + let oneHourMS = 60 * 60 * 1000; + // Insert a visit with a non-round microsecond timestamp (e.g. it's not evenly + // divisible by 1000). This will typically be the case for visits that occur + // during normal navigation. + let time = (Date.now() - oneHourMS) * 1000 + 555; + // We use the low level history api since it lets us provide microseconds + let { count } = await rawAddVisit( + id, + "https://www.example.com", + time, + PlacesUtils.history.TRANSITIONS.TYPED + ); + equal(count, 1); + + let collection = server.user("foo").collection("history"); + + // Sync the visit up to the server. + await sync_engine_and_validate_telem(engine, false); + + collection.updateRecord( + id, + cleartext => { + equal(cleartext.visits[0].date, time); + + // Add unknown fields to an instance of a visit + cleartext.visits.push({ + date: (Date.now() - oneHourMS / 2) * 1000, + type: PlacesUtils.history.TRANSITIONS.LINK, + unknownVisitField: "an unknown field could show up in a visit!", + }); + cleartext.title = "A page title"; + // Add unknown fields to the payload for this URL + cleartext.unknownStrField = "an unknown str field"; + cleartext.unknownObjField = { newField: "a field within an object" }; + }, + new_timestamp() + 10 + ); + + // Force a remote sync. + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + // Add a new visit to ensure we're actually putting things back on the server + let newTime = (Date.now() - oneHourMS) * 1000 + 555; + await rawAddVisit( + id, + "https://www.example.com", + newTime, + PlacesUtils.history.TRANSITIONS.LINK + ); + + // Sync again + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + let placeInfo = await PlacesSyncUtils.history.fetchURLInfoForGuid(id); + + // Found the place we're looking for + Assert.equal(placeInfo.title, "A page title"); + Assert.equal(placeInfo.url, "https://www.example.com/"); + + // It correctly returns any unknownFields that might've been + // stored in the moz_places_extra table + deepEqual(JSON.parse(placeInfo.unknownFields), { + unknownStrField: "an unknown str field", + unknownObjField: { newField: "a field within an object" }, + }); + + // Getting visits via SyncUtils also will return unknownFields + // via the moz_historyvisits_extra table + let visits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(visits.length, 3); + + // fetchVisitsForURL is a sync method that gets called during upload + // so unknown field should already be at the top-level + deepEqual( + visits[0].unknownVisitField, + "an unknown field could show up in a visit!" + ); + + // Remote history record should have the fields back at the top level + let remotePlace = collection.payloads().find(rec => rec.id === id); + deepEqual(remotePlace.unknownStrField, "an unknown str field"); + deepEqual(remotePlace.unknownObjField, { + newField: "a field within an object", + }); + + await engine.wipeClient(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_history_store.js b/services/sync/tests/unit/test_history_store.js new file mode 100644 index 0000000000..07aee0dd01 --- /dev/null +++ b/services/sync/tests/unit/test_history_store.js @@ -0,0 +1,570 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +const TIMESTAMP1 = (Date.now() - 103406528) * 1000; +const TIMESTAMP2 = (Date.now() - 6592903) * 1000; +const TIMESTAMP3 = (Date.now() - 123894) * 1000; + +function promiseOnVisitObserved() { + return new Promise(res => { + let listener = new PlacesWeakCallbackWrapper(events => { + PlacesObservers.removeListener(["page-visited"], listener); + res(); + }); + PlacesObservers.addListener(["page-visited"], listener); + }); +} + +function isDateApproximately(actual, expected, skewMillis = 1000) { + let lowerBound = expected - skewMillis; + let upperBound = expected + skewMillis; + return actual >= lowerBound && actual <= upperBound; +} + +let engine, store, fxuri, fxguid, tburi, tbguid; + +async function applyEnsureNoFailures(records) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch(records, countTelemetry)).length, + 0 + ); +} + +add_task(async function setup() { + engine = new HistoryEngine(Service); + await engine.initialize(); + store = engine._store; +}); + +add_task(async function test_store() { + _("Verify that we've got an empty store to work with."); + do_check_empty(await store.getAllIDs()); + + _("Let's create an entry in the database."); + fxuri = CommonUtils.makeURI("http://getfirefox.com/"); + + await PlacesTestUtils.addVisits({ + uri: fxuri, + title: "Get Firefox!", + visitDate: TIMESTAMP1, + }); + _("Verify that the entry exists."); + let ids = Object.keys(await store.getAllIDs()); + Assert.equal(ids.length, 1); + fxguid = ids[0]; + Assert.ok(await store.itemExists(fxguid)); + + _("If we query a non-existent record, it's marked as deleted."); + let record = await store.createRecord("non-existent"); + Assert.ok(record.deleted); + + _("Verify createRecord() returns a complete record."); + record = await store.createRecord(fxguid); + Assert.equal(record.histUri, fxuri.spec); + Assert.equal(record.title, "Get Firefox!"); + Assert.equal(record.visits.length, 1); + Assert.equal(record.visits[0].date, TIMESTAMP1); + Assert.equal(record.visits[0].type, Ci.nsINavHistoryService.TRANSITION_LINK); + + _("Let's modify the record and have the store update the database."); + let secondvisit = { + date: TIMESTAMP2, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }; + let onVisitObserved = promiseOnVisitObserved(); + let updatedRec = await store.createRecord(fxguid); + updatedRec.cleartext.title = "Hol Dir Firefox!"; + updatedRec.cleartext.visits.push(secondvisit); + await applyEnsureNoFailures([updatedRec]); + await onVisitObserved; + let queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, "Hol Dir Firefox!"); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP2 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: new Date(TIMESTAMP1 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_LINK, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_store_create() { + _("Create a brand new record through the store."); + tbguid = Utils.makeGUID(); + tburi = CommonUtils.makeURI("http://getthunderbird.com"); + let onVisitObserved = promiseOnVisitObserved(); + let record = await store.createRecord(tbguid); + record.cleartext = { + id: tbguid, + histUri: tburi.spec, + title: "The bird is the word!", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }; + await applyEnsureNoFailures([record]); + await onVisitObserved; + Assert.ok(await store.itemExists(tbguid)); + do_check_attribute_count(await store.getAllIDs(), 1); + let queryres = await PlacesUtils.history.fetch(tburi.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, "The bird is the word!"); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP3 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_null_title() { + _( + "Make sure we handle a null title gracefully (it can happen in some cases, e.g. for resource:// URLs)" + ); + let resguid = Utils.makeGUID(); + let resuri = CommonUtils.makeURI("unknown://title"); + let record = await store.createRecord(resguid); + record.cleartext = { + id: resguid, + histUri: resuri.spec, + title: null, + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }; + await applyEnsureNoFailures([record]); + do_check_attribute_count(await store.getAllIDs(), 1); + + let queryres = await PlacesUtils.history.fetch(resuri.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, ""); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP3 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_invalid_records() { + _("Make sure we handle invalid URLs in places databases gracefully."); + await PlacesUtils.withConnectionWrapper( + "test_invalid_record", + async function (db) { + await db.execute( + "INSERT INTO moz_places " + + "(url, url_hash, title, rev_host, visit_count, last_visit_date) " + + "VALUES ('invalid-uri', hash('invalid-uri'), 'Invalid URI', '.', 1, " + + TIMESTAMP3 + + ")" + ); + // Add the corresponding visit to retain database coherence. + await db.execute( + "INSERT INTO moz_historyvisits " + + "(place_id, visit_date, visit_type, session) " + + "VALUES ((SELECT id FROM moz_places WHERE url_hash = hash('invalid-uri') AND url = 'invalid-uri'), " + + TIMESTAMP3 + + ", " + + Ci.nsINavHistoryService.TRANSITION_TYPED + + ", 1)" + ); + } + ); + do_check_attribute_count(await store.getAllIDs(), 1); + + _("Make sure we report records with invalid URIs."); + let invalid_uri_guid = Utils.makeGUID(); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch( + [ + { + id: invalid_uri_guid, + histUri: ":::::::::::::::", + title: "Doesn't have a valid URI", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + Assert.equal(failed.length, 1); + Assert.equal(failed[0], invalid_uri_guid); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "<URL> is not a valid URL." + ); + Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1); + + _("Make sure we handle records with invalid GUIDs gracefully (ignore)."); + await applyEnsureNoFailures([ + { + id: "invalid", + histUri: "http://invalid.guid/", + title: "Doesn't have a valid GUID", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ]); + + _( + "Make sure we handle records with invalid visit codes or visit dates, gracefully ignoring those visits." + ); + let no_date_visit_guid = Utils.makeGUID(); + let no_type_visit_guid = Utils.makeGUID(); + let invalid_type_visit_guid = Utils.makeGUID(); + let non_integer_visit_guid = Utils.makeGUID(); + countTelemetry = new SyncedRecordsTelemetry(); + failed = await store.applyIncomingBatch( + [ + { + id: no_date_visit_guid, + histUri: "http://no.date.visit/", + title: "Visit has no date", + visits: [{ type: Ci.nsINavHistoryService.TRANSITION_EMBED }], + }, + { + id: no_type_visit_guid, + histUri: "http://no.type.visit/", + title: "Visit has no type", + visits: [{ date: TIMESTAMP3 }], + }, + { + id: invalid_type_visit_guid, + histUri: "http://invalid.type.visit/", + title: "Visit has invalid type", + visits: [ + { + date: TIMESTAMP3, + type: Ci.nsINavHistoryService.TRANSITION_LINK - 1, + }, + ], + }, + { + id: non_integer_visit_guid, + histUri: "http://non.integer.visit/", + title: "Visit has non-integer date", + visits: [ + { date: 1234.567, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + Assert.equal(failed.length, 0); + + // Make sure we can apply tombstones (both valid and invalid) + countTelemetry = new SyncedRecordsTelemetry(); + failed = await store.applyIncomingBatch( + [ + { id: no_date_visit_guid, deleted: true }, + { id: "not-a-valid-guid", deleted: true }, + ], + countTelemetry + ); + Assert.deepEqual(failed, ["not-a-valid-guid"]); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "<URL> is not a valid URL." + ); + + _("Make sure we handle records with javascript: URLs gracefully."); + await applyEnsureNoFailures( + [ + { + id: Utils.makeGUID(), + histUri: "javascript:''", + title: "javascript:''", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + + _("Make sure we handle records without any visits gracefully."); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + histUri: "http://getfirebug.com", + title: "Get Firebug!", + visits: [], + }, + ]); +}); + +add_task(async function test_unknowingly_invalid_records() { + _("Make sure we handle rejection of records by places gracefully."); + let oldCAU = store._canAddURI; + store._canAddURI = () => true; + try { + _("Make sure that when places rejects this record we record it as failed"); + let guid = Utils.makeGUID(); + let countTelemetry = new SyncedRecordsTelemetry(); + let invalidRecord = await store.createRecord(guid); + invalidRecord.cleartext = { + id: guid, + histUri: "javascript:''", + title: "javascript:''", + visits: [ + { + date: TIMESTAMP3, + type: Ci.nsINavHistoryService.TRANSITION_EMBED, + }, + ], + }; + let result = await store.applyIncomingBatch( + [invalidRecord], + countTelemetry + ); + deepEqual(result, [guid]); + } finally { + store._canAddURI = oldCAU; + } +}); + +add_task(async function test_clamp_visit_dates() { + let futureVisitTime = Date.now() + 5 * 60 * 1000; + let recentVisitTime = Date.now() - 5 * 60 * 1000; + + let recordA = await store.createRecord("visitAAAAAAA"); + recordA.cleartext = { + id: "visitAAAAAAA", + histUri: "http://example.com/a", + title: "A", + visits: [ + { + date: "invalidDate", + type: Ci.nsINavHistoryService.TRANSITION_LINK, + }, + ], + }; + let recordB = await store.createRecord("visitBBBBBBB"); + recordB.cleartext = { + id: "visitBBBBBBB", + histUri: "http://example.com/b", + title: "B", + visits: [ + { + date: 100, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: 250, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ], + }; + let recordC = await store.createRecord("visitCCCCCCC"); + recordC.cleartext = { + id: "visitCCCCCCC", + histUri: "http://example.com/c", + title: "D", + visits: [ + { + date: futureVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_BOOKMARK, + }, + ], + }; + let recordD = await store.createRecord("visitDDDDDDD"); + recordD.cleartext = { + id: "visitDDDDDDD", + histUri: "http://example.com/d", + title: "D", + visits: [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + }, + ], + }; + await applyEnsureNoFailures([recordA, recordB, recordC, recordD]); + + let visitsForA = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/a" + ); + deepEqual(visitsForA, [], "Should ignore visits with invalid dates"); + + let visitsForB = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/b" + ); + deepEqual( + visitsForB, + [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + // We should clamp visit dates older than original Mosaic release. + date: PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ], + "Should record clamped visit and valid visit for B" + ); + + let visitsForC = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/c" + ); + equal(visitsForC.length, 1, "Should record clamped future visit for C"); + let visitDateForC = PlacesUtils.toDate(visitsForC[0].date); + ok( + isDateApproximately(visitDateForC, Date.now()), + "Should clamp future visit date for C to now" + ); + + let visitsForD = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/d" + ); + deepEqual( + visitsForD, + [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + }, + ], + "Should not clamp valid visit dates" + ); +}); + +add_task(async function test_remove() { + _("Remove an existent record and a non-existent from the store."); + await applyEnsureNoFailures([ + { id: fxguid, deleted: true }, + { id: Utils.makeGUID(), deleted: true }, + ]); + Assert.equal(false, await store.itemExists(fxguid)); + let queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); + + _("Make sure wipe works."); + await store.wipe(); + do_check_empty(await store.getAllIDs()); + queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); + queryres = await PlacesUtils.history.fetch(tburi.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); +}); + +add_task(async function test_chunking() { + let mvpi = store.MAX_VISITS_PER_INSERT; + store.MAX_VISITS_PER_INSERT = 3; + let checkChunks = function (input, expected) { + let chunks = Array.from(store._generateChunks(input)); + deepEqual(chunks, expected); + }; + try { + checkChunks([{ visits: ["x"] }], [[{ visits: ["x"] }]]); + + // 3 should still be one chunk. + checkChunks([{ visits: ["x", "x", "x"] }], [[{ visits: ["x", "x", "x"] }]]); + + // 4 should still be one chunk as we don't split individual records. + checkChunks( + [{ visits: ["x", "x", "x", "x"] }], + [[{ visits: ["x", "x", "x", "x"] }]] + ); + + // 4 in the first and 1 in the second should be 2 chunks. + checkChunks( + [{ visits: ["x", "x", "x", "x"] }, { visits: ["x"] }], + // expected + [[{ visits: ["x", "x", "x", "x"] }], [{ visits: ["x"] }]] + ); + + // we put multiple records into chunks + checkChunks( + [ + { visits: ["x", "x"] }, + { visits: ["x"] }, + { visits: ["x"] }, + { visits: ["x", "x"] }, + { visits: ["x", "x", "x", "x"] }, + ], + // expected + [ + [{ visits: ["x", "x"] }, { visits: ["x"] }], + [{ visits: ["x"] }, { visits: ["x", "x"] }], + [{ visits: ["x", "x", "x", "x"] }], + ] + ); + } finally { + store.MAX_VISITS_PER_INSERT = mvpi; + } +}); + +add_task(async function test_getAllIDs_filters_file_uris() { + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition: PlacesUtils.history.TRANSITION_LINK, + }); + await visitAddedPromise; + + do_check_attribute_count(await store.getAllIDs(), 0); + + await PlacesUtils.history.clear(); +}); + +add_task(async function test_applyIncomingBatch_filters_file_uris() { + const guid = Utils.makeGUID(); + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + await applyEnsureNoFailures([ + { + id: guid, + histUri: uri.spec, + title: "TPS CONFIG", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }, + ]); + Assert.equal(false, await store.itemExists(guid)); + let queryres = await PlacesUtils.history.fetch(uri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); +}); + +add_task(async function cleanup() { + _("Clean up."); + await PlacesUtils.history.clear(); +}); diff --git a/services/sync/tests/unit/test_history_tracker.js b/services/sync/tests/unit/test_history_tracker.js new file mode 100644 index 0000000000..6f351d6984 --- /dev/null +++ b/services/sync/tests/unit/test_history_tracker.js @@ -0,0 +1,251 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PlacesDBUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesDBUtils.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +let tracker; + +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(HistoryEngine); + engine = Service.engineManager.get("history"); + tracker = engine._tracker; +}); + +async function verifyTrackerEmpty() { + let changes = await engine.pullNewChanges(); + do_check_empty(changes); + equal(tracker.score, 0); +} + +async function verifyTrackedCount(expected) { + let changes = await engine.pullNewChanges(); + do_check_attribute_count(changes, expected); +} + +async function verifyTrackedItems(tracked) { + let changes = await engine.pullNewChanges(); + let trackedIDs = new Set(Object.keys(changes)); + for (let guid of tracked) { + ok(guid in changes, `${guid} should be tracked`); + ok(changes[guid] > 0, `${guid} should have a modified time`); + trackedIDs.delete(guid); + } + equal( + trackedIDs.size, + 0, + `Unhandled tracked IDs: ${JSON.stringify(Array.from(trackedIDs))}` + ); +} + +async function resetTracker() { + await tracker.clearChangedIDs(); + tracker.resetScore(); +} + +async function cleanup() { + await PlacesUtils.history.clear(); + await resetTracker(); + await tracker.stop(); +} + +add_task(async function test_empty() { + _("Verify we've got an empty, disabled tracker to work with."); + await verifyTrackerEmpty(); + Assert.ok(!tracker._isTracking); + + await cleanup(); +}); + +add_task(async function test_not_tracking() { + _("Create history item. Won't show because we haven't started tracking yet"); + await addVisit("not_tracking"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_start_tracking() { + _("Add hook for save completion."); + let savePromise = new Promise((resolve, reject) => { + let save = tracker._storage._save; + tracker._storage._save = async function () { + try { + await save.call(this); + resolve(); + } catch (ex) { + reject(ex); + } finally { + tracker._storage._save = save; + } + }; + }); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await addVisit("start_tracking"); + await scorePromise; + + _("Score updated in test_start_tracking."); + await verifyTrackedCount(1); + Assert.equal(tracker.score, SCORE_INCREMENT_SMALL); + + await savePromise; + + _("changedIDs written to disk. Proceeding."); + await cleanup(); +}); + +add_task(async function test_start_tracking_twice() { + _("Verifying preconditions."); + tracker.start(); + await addVisit("start_tracking_twice1"); + await verifyTrackedCount(1); + Assert.equal(tracker.score, SCORE_INCREMENT_SMALL); + + _("Notifying twice won't do any harm."); + tracker.start(); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await addVisit("start_tracking_twice2"); + await scorePromise; + + _("Score updated in test_start_tracking_twice."); + await verifyTrackedCount(2); + Assert.equal(tracker.score, 2 * SCORE_INCREMENT_SMALL); + + await cleanup(); +}); + +add_task(async function test_track_delete() { + _("Deletions are tracked."); + + // This isn't present because we weren't tracking when it was visited. + await addVisit("track_delete"); + let uri = CommonUtils.makeURI("http://getfirefox.com/track_delete"); + let guid = await engine._store.GUIDForUri(uri.spec); + await verifyTrackerEmpty(); + + tracker.start(); + let visitRemovedPromise = promiseVisit("removed", uri); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await PlacesUtils.history.remove(uri); + await Promise.all([scorePromise, visitRemovedPromise]); + + await verifyTrackedItems([guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + + await cleanup(); +}); + +add_task(async function test_dont_track_expiration() { + _("Expirations are not tracked."); + let uriToRemove = await addVisit("to_remove"); + let guidToRemove = await engine._store.GUIDForUri(uriToRemove.spec); + + await resetTracker(); + await verifyTrackerEmpty(); + + tracker.start(); + let visitRemovedPromise = promiseVisit("removed", uriToRemove); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + + // Observe expiration. + Services.obs.addObserver(function onExpiration(aSubject, aTopic, aData) { + Services.obs.removeObserver(onExpiration, aTopic); + // Remove the remaining page to update its score. + PlacesUtils.history.remove(uriToRemove); + }, PlacesUtils.TOPIC_EXPIRATION_FINISHED); + + // Force expiration of 1 entry. + Services.prefs.setIntPref("places.history.expiration.max_pages", 0); + Cc["@mozilla.org/places/expiration;1"] + .getService(Ci.nsIObserver) + .observe(null, "places-debug-start-expiration", 1); + + await Promise.all([scorePromise, visitRemovedPromise]); + await verifyTrackedItems([guidToRemove]); + + await cleanup(); +}); + +add_task(async function test_stop_tracking() { + _("Let's stop tracking again."); + await tracker.stop(); + await addVisit("stop_tracking"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_stop_tracking_twice() { + await tracker.stop(); + await addVisit("stop_tracking_twice1"); + + _("Notifying twice won't do any harm."); + await tracker.stop(); + await addVisit("stop_tracking_twice2"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_filter_file_uris() { + tracker.start(); + + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition: PlacesUtils.history.TRANSITION_LINK, + }); + await visitAddedPromise; + + await verifyTrackerEmpty(); + await tracker.stop(); + await cleanup(); +}); + +add_task(async function test_filter_hidden() { + tracker.start(); + + _("Add visit; should be hidden by the redirect"); + let hiddenURI = await addVisit("hidden"); + let hiddenGUID = await engine._store.GUIDForUri(hiddenURI.spec); + _(`Hidden visit GUID: ${hiddenGUID}`); + + _("Add redirect visit; should be tracked"); + let trackedURI = await addVisit( + "redirect", + hiddenURI.spec, + PlacesUtils.history.TRANSITION_REDIRECT_PERMANENT + ); + let trackedGUID = await engine._store.GUIDForUri(trackedURI.spec); + _(`Tracked visit GUID: ${trackedGUID}`); + + _("Add visit for framed link; should be ignored"); + let embedURI = await addVisit( + "framed_link", + null, + PlacesUtils.history.TRANSITION_FRAMED_LINK + ); + let embedGUID = await engine._store.GUIDForUri(embedURI.spec); + _(`Framed link visit GUID: ${embedGUID}`); + + _("Run Places maintenance to mark redirect visit as hidden"); + await PlacesDBUtils.maintenanceOnIdle(); + + await verifyTrackedItems([trackedGUID]); + + await cleanup(); +}); diff --git a/services/sync/tests/unit/test_hmac_error.js b/services/sync/tests/unit/test_hmac_error.js new file mode 100644 index 0000000000..26dbc12dea --- /dev/null +++ b/services/sync/tests/unit/test_hmac_error.js @@ -0,0 +1,250 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// Track HMAC error counts. +var hmacErrorCount = 0; +(function () { + let hHE = Service.handleHMACEvent; + Service.handleHMACEvent = async function () { + hmacErrorCount++; + return hHE.call(Service); + }; +})(); + +async function shared_setup() { + enableValidationPrefs(); + syncTestLogging(); + + hmacErrorCount = 0; + + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + + // Make sure RotaryEngine is the only one we sync. + let { engine, syncID, tracker } = await registerRotaryEngine(); + await engine.setLastSync(123); // Needs to be non-zero so that tracker is queried. + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + await tracker.addChangedID("scotsman", 0); + Assert.equal(1, Service.engineManager.getEnabled().length); + + let engines = { + rotary: { version: engine.version, syncID }, + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }; + + // Common server objects. + let global = new ServerWBO("global", { engines }); + let keysWBO = new ServerWBO("keys"); + let rotaryColl = new ServerCollection({}, true); + let clientsColl = new ServerCollection({}, true); + + return [engine, rotaryColl, clientsColl, keysWBO, global, tracker]; +} + +add_task(async function hmac_error_during_404() { + _("Attempt to replicate the HMAC error setup."); + let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] = + await shared_setup(); + + // Hand out 404s for crypto/keys. + let keysHandler = keysWBO.handler(); + let key404Counter = 0; + let keys404Handler = function (request, response) { + if (key404Counter > 0) { + let body = "Not Found"; + response.setStatusLine(request.httpVersion, 404, body); + response.bodyOutputStream.write(body, body.length); + key404Counter--; + return; + } + keysHandler(request, response); + }; + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let handlers = { + "/1.1/foo/info/collections": collectionsHelper.handler, + "/1.1/foo/storage/meta/global": upd("meta", global.handler()), + "/1.1/foo/storage/crypto/keys": upd("crypto", keys404Handler), + "/1.1/foo/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()), + }; + + let server = sync_httpd_setup(handlers); + // Do not instantiate SyncTestingInfrastructure; we need real crypto. + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + try { + _("Syncing."); + await sync_and_validate_telem(); + + _( + "Partially resetting client, as if after a restart, and forcing redownload." + ); + Service.collectionKeys.clear(); + await engine.setLastSync(0); // So that we redownload records. + key404Counter = 1; + _("---------------------------"); + await sync_and_validate_telem(); + _("---------------------------"); + + // Two rotary items, one client record... no errors. + Assert.equal(hmacErrorCount, 0); + } finally { + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + await promiseStopServer(server); + } +}); + +add_task(async function hmac_error_during_node_reassignment() { + _("Attempt to replicate an HMAC error during node reassignment."); + let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] = + await shared_setup(); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + // We'll provide a 401 mid-way through the sync. This function + // simulates shifting to a node which has no data. + function on401() { + _("Deleting server data..."); + global.delete(); + rotaryColl.delete(); + keysWBO.delete(); + clientsColl.delete(); + delete collectionsHelper.collections.rotary; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.clients; + _("Deleted server data."); + } + + let should401 = false; + function upd401(coll, handler) { + return function (request, response) { + if (should401 && request.method != "DELETE") { + on401(); + should401 = false; + let body = '"reassigned!"'; + response.setStatusLine(request.httpVersion, 401, "Node reassignment."); + response.bodyOutputStream.write(body, body.length); + return; + } + handler(request, response); + }; + } + + let handlers = { + "/1.1/foo/info/collections": collectionsHelper.handler, + "/1.1/foo/storage/meta/global": upd("meta", global.handler()), + "/1.1/foo/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/foo/storage/clients": upd401("clients", clientsColl.handler()), + "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()), + }; + + let server = sync_httpd_setup(handlers); + // Do not instantiate SyncTestingInfrastructure; we need real crypto. + await configureIdentity({ username: "foo" }, server); + + _("Syncing."); + // First hit of clients will 401. This will happen after meta/global and + // keys -- i.e., in the middle of the sync, but before RotaryEngine. + should401 = true; + + // Use observers to perform actions when our sync finishes. + // This allows us to observe the automatic next-tick sync that occurs after + // an abort. + function onSyncError() { + do_throw("Should not get a sync error!"); + } + let onSyncFinished = function () {}; + let obs = { + observe: function observe(subject, topic, data) { + switch (topic) { + case "weave:service:sync:error": + onSyncError(); + break; + case "weave:service:sync:finish": + onSyncFinished(); + break; + } + }, + }; + + Svc.Obs.add("weave:service:sync:finish", obs); + Svc.Obs.add("weave:service:sync:error", obs); + + // This kicks off the actual test. Split into a function here to allow this + // source file to broadly follow actual execution order. + async function onwards() { + _("== Invoking first sync."); + await Service.sync(); + _("We should not simultaneously have data but no keys on the server."); + let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman"); + let hasKeys = keysWBO.modified; + + _("We correctly handle 401s by aborting the sync and starting again."); + Assert.ok(!hasData == !hasKeys); + + _("Be prepared for the second (automatic) sync..."); + } + + _("Make sure that syncing again causes recovery."); + let callbacksPromise = new Promise(resolve => { + onSyncFinished = function () { + _("== First sync done."); + _("---------------------------"); + onSyncFinished = function () { + _("== Second (automatic) sync done."); + let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman"); + let hasKeys = keysWBO.modified; + Assert.ok(!hasData == !hasKeys); + + // Kick off another sync. Can't just call it, because we're inside the + // lock... + (async () => { + await Async.promiseYield(); + _("Now a fresh sync will get no HMAC errors."); + _( + "Partially resetting client, as if after a restart, and forcing redownload." + ); + Service.collectionKeys.clear(); + await engine.setLastSync(0); + hmacErrorCount = 0; + + onSyncFinished = async function () { + // Two rotary items, one client record... no errors. + Assert.equal(hmacErrorCount, 0); + + Svc.Obs.remove("weave:service:sync:finish", obs); + Svc.Obs.remove("weave:service:sync:error", obs); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + server.stop(resolve); + }; + + Service.sync(); + })().catch(console.error); + }; + }; + }); + await onwards(); + await callbacksPromise; +}); diff --git a/services/sync/tests/unit/test_httpd_sync_server.js b/services/sync/tests/unit/test_httpd_sync_server.js new file mode 100644 index 0000000000..6ac8ff5e04 --- /dev/null +++ b/services/sync/tests/unit/test_httpd_sync_server.js @@ -0,0 +1,250 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_test(function test_creation() { + // Explicit callback for this one. + let server = new SyncServer(Object.create(SyncServerCallback)); + Assert.ok(!!server); // Just so we have a check. + server.start(null, function () { + _("Started on " + server.port); + server.stop(run_next_test); + }); +}); + +add_test(function test_url_parsing() { + let server = new SyncServer(); + + // Check that we can parse a WBO URI. + let parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto/keys"); + let [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage/crypto/keys"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, "crypto/keys"); + Assert.equal(null, server.pathRE.exec("/nothing/else")); + + // Check that we can parse a collection URI. + parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto"); + [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage/crypto"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, "crypto"); + + // We don't allow trailing slash on storage URI. + parts = server.pathRE.exec("/1.1/johnsmith/storage/"); + Assert.equal(parts, undefined); + + // storage alone is a valid request. + parts = server.pathRE.exec("/1.1/johnsmith/storage"); + [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, undefined); + + parts = server.storageRE.exec("storage"); + let collection; + [all, , collection] = parts; + Assert.equal(all, "storage"); + Assert.equal(collection, undefined); + + run_next_test(); +}); + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +function localRequest(server, path) { + _("localRequest: " + path); + let url = server.baseURI.substr(0, server.baseURI.length - 1) + path; + _("url: " + url); + return new RESTRequest(url); +} + +add_task(async function test_basic_http() { + let server = new SyncServer(); + server.registerUser("john", "password"); + Assert.ok(server.userExists("john")); + server.start(); + _("Started on " + server.port); + + let req = localRequest(server, "/1.1/john/storage/crypto/keys"); + _("req is " + req); + // Shouldn't reject, beyond that we don't care. + await req.get(); + + await promiseStopServer(server); +}); + +add_task(async function test_info_collections() { + let server = new SyncServer(Object.create(SyncServerCallback)); + function responseHasCorrectHeaders(r) { + Assert.equal(r.status, 200); + Assert.equal(r.headers["content-type"], "application/json"); + Assert.ok("x-weave-timestamp" in r.headers); + } + + server.registerUser("john", "password"); + server.start(); + + let req = localRequest(server, "/1.1/john/info/collections"); + await req.get(); + responseHasCorrectHeaders(req.response); + Assert.equal(req.response.body, "{}"); + + let putReq = localRequest(server, "/1.1/john/storage/crypto/keys"); + let payload = JSON.stringify({ foo: "bar" }); + let putResp = await putReq.put(payload); + + responseHasCorrectHeaders(putResp); + + let putResponseBody = putResp.body; + _("PUT response body: " + JSON.stringify(putResponseBody)); + + // When we PUT something to crypto/keys, "crypto" appears in the response. + req = localRequest(server, "/1.1/john/info/collections"); + + await req.get(); + responseHasCorrectHeaders(req.response); + let expectedColl = server.getCollection("john", "crypto"); + Assert.ok(!!expectedColl); + let modified = expectedColl.timestamp; + Assert.ok(modified > 0); + Assert.equal(putResponseBody, modified); + Assert.equal(JSON.parse(req.response.body).crypto, modified); + + await promiseStopServer(server); +}); + +add_task(async function test_storage_request() { + let keysURL = "/1.1/john/storage/crypto/keys?foo=bar"; + let foosURL = "/1.1/john/storage/crypto/foos"; + let storageURL = "/1.1/john/storage"; + + let server = new SyncServer(); + let creation = server.timestamp(); + server.registerUser("john", "password"); + + server.createContents("john", { + crypto: { foos: { foo: "bar" } }, + }); + let coll = server.user("john").collection("crypto"); + Assert.ok(!!coll); + + _("We're tracking timestamps."); + Assert.ok(coll.timestamp >= creation); + + async function retrieveWBONotExists() { + let req = localRequest(server, keysURL); + let response = await req.get(); + _("Body is " + response.body); + _("Modified is " + response.newModified); + Assert.equal(response.status, 404); + Assert.equal(response.body, "Not found"); + } + + async function retrieveWBOExists() { + let req = localRequest(server, foosURL); + let response = await req.get(); + _("Body is " + response.body); + _("Modified is " + response.newModified); + let parsedBody = JSON.parse(response.body); + Assert.equal(parsedBody.id, "foos"); + Assert.equal(parsedBody.modified, coll.wbo("foos").modified); + Assert.equal(JSON.parse(parsedBody.payload).foo, "bar"); + } + + async function deleteWBONotExists() { + let req = localRequest(server, keysURL); + server.callback.onItemDeleted = function (username, collection, wboID) { + do_throw("onItemDeleted should not have been called."); + }; + + let response = await req.delete(); + + _("Body is " + response.body); + _("Modified is " + response.newModified); + Assert.equal(response.status, 200); + delete server.callback.onItemDeleted; + } + + async function deleteWBOExists() { + let req = localRequest(server, foosURL); + server.callback.onItemDeleted = function (username, collection, wboID) { + _("onItemDeleted called for " + collection + "/" + wboID); + delete server.callback.onItemDeleted; + Assert.equal(username, "john"); + Assert.equal(collection, "crypto"); + Assert.equal(wboID, "foos"); + }; + await req.delete(); + _("Body is " + req.response.body); + _("Modified is " + req.response.newModified); + Assert.equal(req.response.status, 200); + } + + async function deleteStorage() { + _("Testing DELETE on /storage."); + let now = server.timestamp(); + _("Timestamp: " + now); + let req = localRequest(server, storageURL); + await req.delete(); + + _("Body is " + req.response.body); + _("Modified is " + req.response.newModified); + let parsedBody = JSON.parse(req.response.body); + Assert.ok(parsedBody >= now); + do_check_empty(server.users.john.collections); + } + + async function getStorageFails() { + _("Testing that GET on /storage fails."); + let req = localRequest(server, storageURL); + await req.get(); + Assert.equal(req.response.status, 405); + Assert.equal(req.response.headers.allow, "DELETE"); + } + + async function getMissingCollectionWBO() { + _("Testing that fetching a WBO from an on-existent collection 404s."); + let req = localRequest(server, storageURL + "/foobar/baz"); + await req.get(); + Assert.equal(req.response.status, 404); + } + + server.start(null); + + await retrieveWBONotExists(); + await retrieveWBOExists(); + await deleteWBOExists(); + await deleteWBONotExists(); + await getStorageFails(); + await getMissingCollectionWBO(); + await deleteStorage(); + + await promiseStopServer(server); +}); + +add_task(async function test_x_weave_records() { + let server = new SyncServer(); + server.registerUser("john", "password"); + + server.createContents("john", { + crypto: { foos: { foo: "bar" }, bars: { foo: "baz" } }, + }); + server.start(); + + let wbo = localRequest(server, "/1.1/john/storage/crypto/foos"); + await wbo.get(); + Assert.equal(false, "x-weave-records" in wbo.response.headers); + let col = localRequest(server, "/1.1/john/storage/crypto"); + await col.get(); + // Collection fetches do. + Assert.equal(col.response.headers["x-weave-records"], "2"); + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_interval_triggers.js b/services/sync/tests/unit/test_interval_triggers.js new file mode 100644 index 0000000000..6f2821ec45 --- /dev/null +++ b/services/sync/tests/unit/test_interval_triggers.js @@ -0,0 +1,472 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Svc.PrefBranch.setStringPref("registerEngines", ""); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let scheduler; +let clientsEngine; + +async function sync_httpd_setup() { + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + return httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + }); +} + +async function setUp(server) { + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)); +} + +add_task(async function setup() { + scheduler = Service.scheduler; + clientsEngine = Service.clientsEngine; + + // Don't remove stale clients when syncing. This is a test-only workaround + // that lets us add clients directly to the store, without losing them on + // the next sync. + clientsEngine._removeRemoteClient = async id => {}; +}); + +add_task(async function test_successful_sync_adjustSyncInterval() { + enableValidationPrefs(); + + _("Test successful sync calling adjustSyncInterval"); + let syncSuccesses = 0; + function onSyncFinish() { + _("Sync success."); + syncSuccesses++; + } + Svc.Obs.add("weave:service:sync:finish", onSyncFinish); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.hasIncomingItems); + + _("Test as long as numClients <= 1 our sync interval is SINGLE_USER."); + // idle == true && numClients <= 1 && hasIncomingItems == false + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncSuccesses, 1); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncSuccesses, 2); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncSuccesses, 3); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == true && numClients <= 1 && hasIncomingItems == true + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncSuccesses, 4); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _( + "Test as long as idle && numClients > 1 our sync interval is idleInterval." + ); + // idle == true && numClients > 1 && hasIncomingItems == true + await Service.clientsEngine._store.create({ + id: "foo", + cleartext: { name: "bar", type: "mobile" }, + }); + await Service.sync(); + Assert.equal(syncSuccesses, 5); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + // idle == true && numClients > 1 && hasIncomingItems == false + scheduler.hasIncomingItems = false; + await Service.sync(); + Assert.equal(syncSuccesses, 6); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + _("Test non-idle, numClients > 1, no incoming items => activeInterval."); + // idle == false && numClients > 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncSuccesses, 7); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + _("Test non-idle, numClients > 1, incoming items => immediateInterval."); + // idle == false && numClients > 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncSuccesses, 8); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); // gets reset to false + Assert.equal(scheduler.syncInterval, scheduler.immediateInterval); + + Svc.Obs.remove("weave:service:sync:finish", onSyncFinish); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_unsuccessful_sync_adjustSyncInterval() { + enableValidationPrefs(); + + _("Test unsuccessful sync calling adjustSyncInterval"); + + let syncFailures = 0; + function onSyncError() { + _("Sync error."); + syncFailures++; + } + Svc.Obs.add("weave:service:sync:error", onSyncError); + + _("Test unsuccessful sync calls adjustSyncInterval"); + // Force sync to fail. + Svc.PrefBranch.setStringPref("firstSync", "notReady"); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.hasIncomingItems); + + _("Test as long as numClients <= 1 our sync interval is SINGLE_USER."); + // idle == true && numClients <= 1 && hasIncomingItems == false + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncFailures, 1); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncFailures, 2); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncFailures, 3); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == true && numClients <= 1 && hasIncomingItems == true + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncFailures, 4); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _( + "Test as long as idle && numClients > 1 our sync interval is idleInterval." + ); + // idle == true && numClients > 1 && hasIncomingItems == true + Svc.PrefBranch.setIntPref("clients.devices.mobile", 2); + scheduler.updateClientMode(); + + await Service.sync(); + Assert.equal(syncFailures, 5); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + // idle == true && numClients > 1 && hasIncomingItems == false + scheduler.hasIncomingItems = false; + await Service.sync(); + Assert.equal(syncFailures, 6); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + _("Test non-idle, numClients > 1, no incoming items => activeInterval."); + // idle == false && numClients > 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncFailures, 7); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + _("Test non-idle, numClients > 1, incoming items => immediateInterval."); + // idle == false && numClients > 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncFailures, 8); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); // gets reset to false + Assert.equal(scheduler.syncInterval, scheduler.immediateInterval); + + await Service.startOver(); + Svc.Obs.remove("weave:service:sync:error", onSyncError); + await promiseStopServer(server); +}); + +add_task(async function test_back_triggers_sync() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Single device: no sync triggered. + scheduler.idle = true; + scheduler.observe( + null, + "active", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.ok(!scheduler.idle); + + // Multiple devices: sync is triggered. + Svc.PrefBranch.setIntPref("clients.devices.mobile", 2); + scheduler.updateClientMode(); + + let promiseDone = promiseOneObserver("weave:service:sync:finish"); + + scheduler.idle = true; + scheduler.observe( + null, + "active", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.ok(!scheduler.idle); + await promiseDone; + + Service.recordManager.clearCache(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + scheduler.setDefaults(); + await clientsEngine.resetClient(); + + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_adjust_interval_on_sync_error() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + let syncFailures = 0; + function onSyncError() { + _("Sync error."); + syncFailures++; + } + Svc.Obs.add("weave:service:sync:error", onSyncError); + + _("Test unsuccessful sync updates client mode & sync intervals"); + // Force a sync fail. + Svc.PrefBranch.setStringPref("firstSync", "notReady"); + + Assert.equal(syncFailures, 0); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + Svc.PrefBranch.setIntPref("clients.devices.mobile", 2); + await Service.sync(); + + Assert.equal(syncFailures, 1); + Assert.ok(scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + Svc.Obs.remove("weave:service:sync:error", onSyncError); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_bug671378_scenario() { + enableValidationPrefs(); + + // Test scenario similar to bug 671378. This bug appeared when a score + // update occurred that wasn't large enough to trigger a sync so + // scheduleNextSync() was called without a time interval parameter, + // setting nextSync to a non-zero value and preventing the timer from + // being adjusted in the next call to scheduleNextSync(). + let server = await sync_httpd_setup(); + await setUp(server); + + let syncSuccesses = 0; + function onSyncFinish() { + _("Sync success."); + syncSuccesses++; + } + Svc.Obs.add("weave:service:sync:finish", onSyncFinish); + + // After first sync call, syncInterval & syncTimer are singleDeviceInterval. + await Service.sync(); + Assert.equal(syncSuccesses, 1); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + + let promiseDone = new Promise(resolve => { + // Wrap scheduleNextSync so we are notified when it is finished. + scheduler._scheduleNextSync = scheduler.scheduleNextSync; + scheduler.scheduleNextSync = function () { + scheduler._scheduleNextSync(); + + // Check on sync:finish scheduleNextSync sets the appropriate + // syncInterval and syncTimer values. + if (syncSuccesses == 2) { + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + scheduler.scheduleNextSync = scheduler._scheduleNextSync; + Svc.Obs.remove("weave:service:sync:finish", onSyncFinish); + Service.startOver().then(() => { + server.stop(resolve); + }); + } + }; + }); + + // Set nextSync != 0 + // syncInterval still hasn't been set by call to updateClientMode. + // Explicitly trying to invoke scheduleNextSync during a sync + // (to immitate a score update that isn't big enough to trigger a sync). + Svc.Obs.add("weave:service:sync:start", function onSyncStart() { + // Wait for other sync:start observers to be called so that + // nextSync is set to 0. + CommonUtils.nextTick(function () { + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + + scheduler.scheduleNextSync(); + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + }); + }); + + await Service.clientsEngine._store.create({ + id: "foo", + cleartext: { name: "bar", type: "mobile" }, + }); + await Service.sync(); + await promiseDone; +}); + +add_task(async function test_adjust_timer_larger_syncInterval() { + _( + "Test syncInterval > current timout period && nextSync != 0, syncInterval is NOT used." + ); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 2); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + scheduler.scheduleNextSync(); + + // Ensure we have a small interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer.delay, scheduler.activeInterval); + + // Make interval large again + await clientsEngine._wipeClient(); + Svc.PrefBranch.clearUserPref("clients.devices.mobile"); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + scheduler.scheduleNextSync(); + + // Ensure timer delay remains as the small interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + // SyncSchedule. + await Service.startOver(); +}); + +add_task(async function test_adjust_timer_smaller_syncInterval() { + _( + "Test current timout > syncInterval period && nextSync != 0, syncInterval is used." + ); + scheduler.scheduleNextSync(); + + // Ensure we have a large interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + + // Make interval smaller + Svc.PrefBranch.setIntPref("clients.devices.mobile", 2); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + scheduler.scheduleNextSync(); + + // Ensure smaller timer delay is used. + Assert.notEqual(scheduler.nextSync, 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + // SyncSchedule. + await Service.startOver(); +}); diff --git a/services/sync/tests/unit/test_keys.js b/services/sync/tests/unit/test_keys.js new file mode 100644 index 0000000000..8cc5d4055c --- /dev/null +++ b/services/sync/tests/unit/test_keys.js @@ -0,0 +1,242 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { CollectionKeyManager, CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); + +var collectionKeys = new CollectionKeyManager(); + +function do_check_keypair_eq(a, b) { + Assert.equal(2, a.length); + Assert.equal(2, b.length); + Assert.equal(a[0], b[0]); + Assert.equal(a[1], b[1]); +} + +add_test(function test_set_invalid_values() { + _("Ensure that setting invalid encryption and HMAC key values is caught."); + + let bundle = new BulkKeyBundle("foo"); + + let thrown = false; + try { + bundle.encryptionKey = null; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("Encryption key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.encryptionKey = ["trollololol"]; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("Encryption key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = Utils.generateRandomBytesLegacy(15); + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key must be at least 128"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = null; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key can only be set to string"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = ["trollolol"]; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = Utils.generateRandomBytesLegacy(15); + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key must be at least 128"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + run_next_test(); +}); + +add_task(async function test_ensureLoggedIn() { + let log = Log.repository.getLogger("Test"); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + await configureIdentity(); + + let keyBundle = Weave.Service.identity.syncKeyBundle; + + /* + * Build a test version of storage/crypto/keys. + * Encrypt it with the sync key. + * Pass it into the CollectionKeyManager. + */ + + log.info("Building storage keys..."); + let storage_keys = new CryptoWrapper("crypto", "keys"); + let default_key64 = await Weave.Crypto.generateRandomKey(); + let default_hmac64 = await Weave.Crypto.generateRandomKey(); + let bookmarks_key64 = await Weave.Crypto.generateRandomKey(); + let bookmarks_hmac64 = await Weave.Crypto.generateRandomKey(); + + storage_keys.cleartext = { + default: [default_key64, default_hmac64], + collections: { bookmarks: [bookmarks_key64, bookmarks_hmac64] }, + }; + storage_keys.modified = Date.now() / 1000; + storage_keys.id = "keys"; + + log.info("Encrypting storage keys..."); + + // Use passphrase (sync key) itself to encrypt the key bundle. + await storage_keys.encrypt(keyBundle); + + // Sanity checking. + Assert.ok(null == storage_keys.cleartext); + Assert.ok(null != storage_keys.ciphertext); + + log.info("Updating collection keys."); + + // updateContents decrypts the object, releasing the payload for us to use. + // Returns true, because the default key has changed. + Assert.ok(await collectionKeys.updateContents(keyBundle, storage_keys)); + let payload = storage_keys.cleartext; + + _("CK: " + JSON.stringify(collectionKeys._collections)); + + // Test that the CollectionKeyManager returns a similar WBO. + let wbo = collectionKeys.asWBO("crypto", "keys"); + + _("WBO: " + JSON.stringify(wbo)); + _("WBO cleartext: " + JSON.stringify(wbo.cleartext)); + + // Check the individual contents. + Assert.equal(wbo.collection, "crypto"); + Assert.equal(wbo.id, "keys"); + Assert.equal(undefined, wbo.modified); + Assert.equal(collectionKeys.lastModified, storage_keys.modified); + Assert.ok(!!wbo.cleartext.default); + do_check_keypair_eq(payload.default, wbo.cleartext.default); + do_check_keypair_eq( + payload.collections.bookmarks, + wbo.cleartext.collections.bookmarks + ); + + Assert.ok("bookmarks" in collectionKeys._collections); + Assert.equal(false, "tabs" in collectionKeys._collections); + + _("Updating contents twice with the same data doesn't proceed."); + await storage_keys.encrypt(keyBundle); + Assert.equal( + false, + await collectionKeys.updateContents(keyBundle, storage_keys) + ); + + /* + * Test that we get the right keys out when we ask for + * a collection's tokens. + */ + let b1 = new BulkKeyBundle("bookmarks"); + b1.keyPairB64 = [bookmarks_key64, bookmarks_hmac64]; + let b2 = collectionKeys.keyForCollection("bookmarks"); + do_check_keypair_eq(b1.keyPair, b2.keyPair); + + // Check key equality. + Assert.ok(b1.equals(b2)); + Assert.ok(b2.equals(b1)); + + b1 = new BulkKeyBundle("[default]"); + b1.keyPairB64 = [default_key64, default_hmac64]; + + Assert.ok(!b1.equals(b2)); + Assert.ok(!b2.equals(b1)); + + b2 = collectionKeys.keyForCollection(null); + do_check_keypair_eq(b1.keyPair, b2.keyPair); + + /* + * Checking for update times. + */ + let info_collections = {}; + Assert.ok(collectionKeys.updateNeeded(info_collections)); + info_collections.crypto = 5000; + Assert.ok(!collectionKeys.updateNeeded(info_collections)); + info_collections.crypto = 1 + Date.now() / 1000; // Add one in case computers are fast! + Assert.ok(collectionKeys.updateNeeded(info_collections)); + + collectionKeys.lastModified = null; + Assert.ok(collectionKeys.updateNeeded({})); + + /* + * Check _compareKeyBundleCollections. + */ + async function newBundle(name) { + let r = new BulkKeyBundle(name); + await r.generateRandom(); + return r; + } + let k1 = await newBundle("k1"); + let k2 = await newBundle("k2"); + let k3 = await newBundle("k3"); + let k4 = await newBundle("k4"); + let k5 = await newBundle("k5"); + let coll1 = { foo: k1, bar: k2 }; + let coll2 = { foo: k1, bar: k2 }; + let coll3 = { foo: k1, bar: k3 }; + let coll4 = { foo: k4 }; + let coll5 = { baz: k5, bar: k2 }; + let coll6 = {}; + + let d1 = collectionKeys._compareKeyBundleCollections(coll1, coll2); // [] + let d2 = collectionKeys._compareKeyBundleCollections(coll1, coll3); // ["bar"] + let d3 = collectionKeys._compareKeyBundleCollections(coll3, coll2); // ["bar"] + let d4 = collectionKeys._compareKeyBundleCollections(coll1, coll4); // ["bar", "foo"] + let d5 = collectionKeys._compareKeyBundleCollections(coll5, coll2); // ["baz", "foo"] + let d6 = collectionKeys._compareKeyBundleCollections(coll6, coll1); // ["bar", "foo"] + let d7 = collectionKeys._compareKeyBundleCollections(coll5, coll5); // [] + let d8 = collectionKeys._compareKeyBundleCollections(coll6, coll6); // [] + + Assert.ok(d1.same); + Assert.ok(!d2.same); + Assert.ok(!d3.same); + Assert.ok(!d4.same); + Assert.ok(!d5.same); + Assert.ok(!d6.same); + Assert.ok(d7.same); + Assert.ok(d8.same); + + Assert.deepEqual(d1.changed, []); + Assert.deepEqual(d2.changed, ["bar"]); + Assert.deepEqual(d3.changed, ["bar"]); + Assert.deepEqual(d4.changed, ["bar", "foo"]); + Assert.deepEqual(d5.changed, ["baz", "foo"]); + Assert.deepEqual(d6.changed, ["bar", "foo"]); +}); diff --git a/services/sync/tests/unit/test_load_modules.js b/services/sync/tests/unit/test_load_modules.js new file mode 100644 index 0000000000..93ef883d4b --- /dev/null +++ b/services/sync/tests/unit/test_load_modules.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const modules = [ + "addonutils.sys.mjs", + "addonsreconciler.sys.mjs", + "constants.sys.mjs", + "engines/addons.sys.mjs", + "engines/clients.sys.mjs", + "engines/extension-storage.sys.mjs", + "engines/passwords.sys.mjs", + "engines/prefs.sys.mjs", + "engines.sys.mjs", + "keys.sys.mjs", + "main.sys.mjs", + "policies.sys.mjs", + "record.sys.mjs", + "resource.sys.mjs", + "service.sys.mjs", + "stages/declined.sys.mjs", + "stages/enginesync.sys.mjs", + "status.sys.mjs", + "sync_auth.sys.mjs", + "util.sys.mjs", +]; + +if (AppConstants.MOZ_APP_NAME != "thunderbird") { + modules.push( + "engines/bookmarks.sys.mjs", + "engines/forms.sys.mjs", + "engines/history.sys.mjs", + "engines/tabs.sys.mjs" + ); +} + +const testingModules = [ + "fakeservices.sys.mjs", + "rotaryengine.sys.mjs", + "utils.sys.mjs", + "fxa_utils.sys.mjs", +]; + +function run_test() { + for (let m of modules) { + let res = "resource://services-sync/" + m; + _("Attempting to load " + res); + ChromeUtils.importESModule(res); + } + + for (let m of testingModules) { + let res = "resource://testing-common/services/sync/" + m; + _("Attempting to load " + res); + ChromeUtils.importESModule(res); + } +} diff --git a/services/sync/tests/unit/test_node_reassignment.js b/services/sync/tests/unit/test_node_reassignment.js new file mode 100644 index 0000000000..e3352af318 --- /dev/null +++ b/services/sync/tests/unit/test_node_reassignment.js @@ -0,0 +1,523 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Test that node reassignment responses are respected on all kinds of " + + "requests." +); + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function setup() { + validate_all_future_pings(); +}); + +/** + * Emulate the following Zeus config: + * $draining = data.get($prefix . $host . " draining"); + * if ($draining == "drain.") { + * log.warn($log_host_db_status . " migrating=1 (node-reassignment)" . + * $log_suffix); + * http.sendResponse("401 Node reassignment", $content_type, + * '"server request: node reassignment"', ""); + * } + */ +const reassignBody = '"server request: node reassignment"'; + +// API-compatible with SyncServer handler. Bind `handler` to something to use +// as a ServerCollection handler. +function handleReassign(handler, req, resp) { + resp.setStatusLine(req.httpVersion, 401, "Node reassignment"); + resp.setHeader("Content-Type", "application/json"); + resp.bodyOutputStream.write(reassignBody, reassignBody.length); +} + +async function prepareServer() { + let server = new SyncServer(); + server.registerUser("johndoe"); + server.start(); + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + return server; +} + +function getReassigned() { + try { + return Services.prefs.getBoolPref("services.sync.lastSyncReassigned"); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_UNEXPECTED) { + do_throw( + "Got exception retrieving lastSyncReassigned: " + Log.exceptionStr(ex) + ); + } + } + return false; +} + +/** + * Make a test request to `url`, then watch the result of two syncs + * to ensure that a node request was made. + * Runs `between` between the two. This can be used to undo deliberate failure + * setup, detach observers, etc. + */ +async function syncAndExpectNodeReassignment( + server, + firstNotification, + between, + secondNotification, + url +) { + let deferred = Promise.withResolvers(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // Make sure that it works! + let request = new RESTRequest(url); + let response = await request.get(); + Assert.equal(response.status, 401); + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // Allow for tests to clean up error conditions. + between(); + } + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Second sync nextTick."); + Assert.equal(getTokenCount, 1); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + await Service.sync(); + + await deferred.promise; +} + +add_task(async function test_momentary_401_engine() { + enableValidationPrefs(); + + _("Test a failure for engine URLs that's resolved by reassignment."); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + let numResets = 0; + let observeReset = (obs, topic) => { + if (topic == "rotary") { + numResets += 1; + } + }; + _("Adding observer that we saw an engine reset."); + Svc.Obs.add("weave:engine:reset-client:finish", observeReset); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // We want to verify that the clusterURL pref has been cleared after a 401 + // inside a sync. Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function between() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Assert.ok(getReassigned()); + } + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:finish", + between, + "weave:service:sync:finish", + Service.storageURL + "rotary" + ); + + Svc.Obs.remove("weave:engine:reset-client:finish", observeReset); + Assert.equal(numResets, 1); + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +// This test ends up being a failing fetch *after we're already logged in*. +add_task(async function test_momentary_401_info_collections() { + enableValidationPrefs(); + + _("Test a failure for info/collections that's resolved by reassignment."); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + // Return a 401 for info requests, particularly info/collections. + let oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.info = oldHandler; + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.infoURL + ); +}); + +add_task(async function test_momentary_401_storage_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + _("Performing initial sync to ensure we are logged in."); + await Service.sync(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +add_task(async function test_momentary_401_storage_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + await syncAndExpectNodeReassignment( + server, + "weave:service:login:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +add_task(async function test_loop_avoidance_storage() { + enableValidationPrefs(); + + _( + "Test that a repeated failure doesn't result in a sync loop " + + "if node reassignment cannot resolve the failure." + ); + + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + let firstNotification = "weave:service:login:error"; + let secondNotification = "weave:service:login:error"; + let thirdNotification = "weave:service:sync:finish"; + + let deferred = Promise.withResolvers(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // Track the time. We want to make sure the duration between the first and + // second sync is small, and then that the duration between second and third + // is set to be large. + let now; + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // We got a 401 mid-sync, and set the pref accordingly. + Assert.ok(Services.prefs.getBoolPref("services.sync.lastSyncReassigned")); + + // Update the timestamp. + now = Date.now(); + } + + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Svc.Obs.add(thirdNotification, onThirdSync); + + // This sync occurred within the backoff interval. + let elapsedTime = Date.now() - now; + Assert.ok(elapsedTime < MINIMUM_BACKOFF_INTERVAL); + + // This pref will be true until a sync completes successfully. + Assert.ok(getReassigned()); + + // The timer will be set for some distant time. + // We store nextSync in prefs, which offers us only limited resolution. + // Include that logic here. + let expectedNextSync = + 1000 * Math.floor((now + MINIMUM_BACKOFF_INTERVAL) / 1000); + _("Next sync scheduled for " + Service.scheduler.nextSync); + _("Expected to be slightly greater than " + expectedNextSync); + + Assert.ok(Service.scheduler.nextSync >= expectedNextSync); + Assert.ok(!!Service.scheduler.syncTimer); + + // Undo our evil scheme. + server.toplevelHandlers.storage = oldHandler; + + // Bring the timer forward to kick off a successful sync, so we can watch + // the pref get cleared. + Service.scheduler.scheduleNextSync(0); + } + function onThirdSync() { + Svc.Obs.remove(thirdNotification, onThirdSync); + + // That'll do for now; no more syncs. + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Third sync nextTick."); + Assert.ok(!getReassigned()); + Assert.equal(getTokenCount, 2); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + + now = Date.now(); + await Service.sync(); + await deferred.promise; +}); + +add_task(async function test_loop_avoidance_engine() { + enableValidationPrefs(); + + _( + "Test that a repeated 401 in an engine doesn't result in a sync loop " + + "if node reassignment cannot resolve the failure." + ); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + let deferred = Promise.withResolvers(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Assert.ok(getReassigned()); + } + + function beforeSuccessfulSync() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + } + + let firstNotification = "weave:service:sync:finish"; + let secondNotification = "weave:service:sync:finish"; + let thirdNotification = "weave:service:sync:finish"; + + // Track the time. We want to make sure the duration between the first and + // second sync is small, and then that the duration between second and third + // is set to be large. + let now; + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + + // We got a 401 mid-sync, and set the pref accordingly. + Assert.ok(Services.prefs.getBoolPref("services.sync.lastSyncReassigned")); + + // Update the timestamp. + now = Date.now(); + } + + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Svc.Obs.add(thirdNotification, onThirdSync); + + // This sync occurred within the backoff interval. + let elapsedTime = Date.now() - now; + Assert.ok(elapsedTime < MINIMUM_BACKOFF_INTERVAL); + + // This pref will be true until a sync completes successfully. + Assert.ok(getReassigned()); + + // The timer will be set for some distant time. + // We store nextSync in prefs, which offers us only limited resolution. + // Include that logic here. + let expectedNextSync = + 1000 * Math.floor((now + MINIMUM_BACKOFF_INTERVAL) / 1000); + _("Next sync scheduled for " + Service.scheduler.nextSync); + _("Expected to be slightly greater than " + expectedNextSync); + + Assert.ok(Service.scheduler.nextSync >= expectedNextSync); + Assert.ok(!!Service.scheduler.syncTimer); + + // Undo our evil scheme. + beforeSuccessfulSync(); + + // Bring the timer forward to kick off a successful sync, so we can watch + // the pref get cleared. + Service.scheduler.scheduleNextSync(0); + } + + function onThirdSync() { + Svc.Obs.remove(thirdNotification, onThirdSync); + + // That'll do for now; no more syncs. + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Third sync nextTick."); + Assert.ok(!getReassigned()); + Assert.equal(getTokenCount, 2); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + + now = Date.now(); + await Service.sync(); + await deferred.promise; + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); diff --git a/services/sync/tests/unit/test_password_engine.js b/services/sync/tests/unit/test_password_engine.js new file mode 100644 index 0000000000..081403f63d --- /dev/null +++ b/services/sync/tests/unit/test_password_engine.js @@ -0,0 +1,1257 @@ +const { FXA_PWDMGR_HOST, FXA_PWDMGR_REALM } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsCommon.sys.mjs" +); +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const LoginInfo = Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +const { LoginCSVImport } = ChromeUtils.importESModule( + "resource://gre/modules/LoginCSVImport.sys.mjs" +); + +const { FileTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/FileTestUtils.sys.mjs" +); + +const PropertyBag = Components.Constructor( + "@mozilla.org/hash-property-bag;1", + Ci.nsIWritablePropertyBag +); + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine.wipeClient(); + engine.lastModified = null; + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + if (server) { + await promiseStopServer(server); + } +} + +add_task(async function setup() { + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); +}); + +add_task(async function test_ignored_fields() { + _("Only changes to syncable fields should be tracked"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + enableValidationPrefs(); + + let loginInfo = new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ); + + // Setting syncCounter to -1 so that it will be incremented to 0 when added. + loginInfo.syncCounter = -1; + let login = await Services.logins.addLoginAsync(loginInfo); + login.QueryInterface(Ci.nsILoginMetaInfo); // For `guid`. + + engine._tracker.start(); + + try { + let nonSyncableProps = new PropertyBag(); + nonSyncableProps.setProperty("timeLastUsed", Date.now()); + nonSyncableProps.setProperty("timesUsed", 3); + Services.logins.modifyLogin(login, nonSyncableProps); + + let noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track non-syncable fields"); + + let syncableProps = new PropertyBag(); + syncableProps.setProperty("username", "newuser"); + Services.logins.modifyLogin(login, syncableProps); + + let changes = await engine.pullNewChanges(); + deepEqual( + Object.keys(changes), + [login.guid], + "Should track syncable fields" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_ignored_sync_credentials() { + _("Sync credentials in login manager should be ignored"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + enableValidationPrefs(); + + engine._tracker.start(); + + try { + let login = await Services.logins.addLoginAsync( + new LoginInfo( + FXA_PWDMGR_HOST, + null, + FXA_PWDMGR_REALM, + "fxa-uid", + "creds", + "", + "" + ) + ); + + let noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track new FxA credentials"); + + let props = new PropertyBag(); + props.setProperty("password", "newcreds"); + Services.logins.modifyLogin(login, props); + + noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track changes to FxA credentials"); + + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: FXA_PWDMGR_HOST, + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 0); + equal(foundLogins[0].everSynced, false); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_password_engine() { + _("Basic password sync test"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + enableValidationPrefs(); + + _("Add new login to upload during first sync"); + let newLogin; + { + let login = new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ); + await Services.logins.addLoginAsync(login); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://example.com", + }); + equal(logins.length, 1, "Should find new login in login manager"); + newLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + + // Insert a server record that's older, so that we prefer the local one. + let rec = new LoginRec("passwords", newLogin.guid); + rec.formSubmitURL = newLogin.formActionOrigin; + rec.httpRealm = newLogin.httpRealm; + rec.hostname = newLogin.origin; + rec.username = newLogin.username; + rec.password = "sekrit"; + let remotePasswordChangeTime = Date.now() - 1 * 60 * 60 * 24 * 1000; + rec.timeCreated = remotePasswordChangeTime; + rec.timePasswordChanged = remotePasswordChangeTime; + collection.insert( + newLogin.guid, + encryptPayload(rec.cleartext), + remotePasswordChangeTime / 1000 + ); + } + + _("Add login with older password change time to replace during first sync"); + let oldLogin; + { + let login = new LoginInfo( + "https://mozilla.com", + "", + null, + "us3r", + "0ldpa55", + "", + "" + ); + await Services.logins.addLoginAsync(login); + + let props = new PropertyBag(); + let localPasswordChangeTime = Date.now() - 1 * 60 * 60 * 24 * 1000; + props.setProperty("timePasswordChanged", localPasswordChangeTime); + Services.logins.modifyLogin(login, props); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://mozilla.com", + }); + equal(logins.length, 1, "Should find old login in login manager"); + oldLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + equal(oldLogin.timePasswordChanged, localPasswordChangeTime); + + let rec = new LoginRec("passwords", oldLogin.guid); + rec.hostname = oldLogin.origin; + rec.formSubmitURL = oldLogin.formActionOrigin; + rec.httpRealm = oldLogin.httpRealm; + rec.username = oldLogin.username; + // Change the password and bump the password change time to ensure we prefer + // the remote one during reconciliation. + rec.password = "n3wpa55"; + rec.usernameField = oldLogin.usernameField; + rec.passwordField = oldLogin.usernameField; + rec.timeCreated = oldLogin.timeCreated; + rec.timePasswordChanged = Date.now(); + collection.insert(oldLogin.guid, encryptPayload(rec.cleartext)); + } + + await engine._tracker.stop(); + + try { + await sync_engine_and_validate_telem(engine, false); + + let newRec = collection.cleartext(newLogin.guid); + equal( + newRec.password, + "password", + "Should update remote password for newer login" + ); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://mozilla.com", + }); + equal( + logins[0].password, + "n3wpa55", + "Should update local password for older login" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_outgoing() { + _("Test syncing outgoing records"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + let loginInfo = new LoginInfo( + "http://mozilla.com", + "http://mozilla.com", + null, + "theuser", + "thepassword", + "username", + "password" + ); + let login = await Services.logins.addLoginAsync(loginInfo); + + engine._tracker.start(); + + try { + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 1); + equal(foundLogins[0].everSynced, false); + equal(collection.count(), 0); + + let guid = foundLogins[0].QueryInterface(Ci.nsILoginMetaInfo).guid; + + let changes = await engine.getChangedIDs(); + let change = changes[guid]; + equal(Object.keys(changes).length, 1); + equal(change.counter, 1); + ok(!change.deleted); + + // This test modifies the password and then performs a sync and + // then ensures that the synced record is correct. This is done twice + // to ensure that syncing occurs correctly when the server record does not + // yet exist and when it does already exist. + for (let i = 1; i <= 2; i++) { + _("Modify the password iteration " + i); + foundLogins[0].password = "newpassword" + i; + Services.logins.modifyLogin(login, foundLogins[0]); + foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 1); + // On the first pass, the counter should be 2, one for the add and one for the modify. + // No sync has occurred yet so everSynced should be false. + // On the second pass, the counter will only be 1 for the modify. The everSynced + // property should be true as the sync happened on the last iteration. + equal(foundLogins[0].syncCounter, i == 2 ? 1 : 2); + equal(foundLogins[0].everSynced, i == 2); + + changes = await engine.getChangedIDs(); + change = changes[guid]; + equal(Object.keys(changes).length, 1); + equal(change.counter, i == 2 ? 1 : 2); + ok(!change.deleted); + + _("Perform sync after modifying the password"); + await sync_engine_and_validate_telem(engine, false); + + equal(Object.keys(await engine.getChangedIDs()), 0); + + // The remote login should have the updated password. + let newRec = collection.cleartext(guid); + equal( + newRec.password, + "newpassword" + i, + "Should update remote password for login" + ); + + foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 0); + equal(foundLogins[0].everSynced, true); + + login.password = "newpassword" + i; + } + + // Next, modify the username and sync. + _("Modify the username"); + foundLogins[0].username = "newuser"; + Services.logins.modifyLogin(login, foundLogins[0]); + foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 1); + equal(foundLogins[0].everSynced, true); + + _("Perform sync after modifying the username"); + await sync_engine_and_validate_telem(engine, false); + + // The remote login should have the updated password. + let newRec = collection.cleartext(guid); + equal( + newRec.username, + "newuser", + "Should update remote username for login" + ); + + foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 0); + equal(foundLogins[0].everSynced, true); + + // Finally, remove the login. The server record should be marked as deleted. + _("Remove the login"); + equal(collection.count(), 1); + equal(Services.logins.countLogins("", "", ""), 2); + equal((await Services.logins.getAllLogins()).length, 2); + ok(await engine._store.itemExists(guid)); + + ok((await engine._store.getAllIDs())[guid]); + + Services.logins.removeLogin(foundLogins[0]); + foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + equal(foundLogins.length, 0); + + changes = await engine.getChangedIDs(); + change = changes[guid]; + equal(Object.keys(changes).length, 1); + equal(change.counter, 1); + ok(change.deleted); + + _("Perform sync after removing the login"); + await sync_engine_and_validate_telem(engine, false); + + equal(collection.count(), 1); + let payload = collection.payloads()[0]; + ok(payload.deleted); + + equal(Object.keys(await engine.getChangedIDs()), 0); + + // All of these should not include the deleted login. Only the FxA password should exist. + equal(Services.logins.countLogins("", "", ""), 1); + equal((await Services.logins.getAllLogins()).length, 1); + ok(!(await engine._store.itemExists(guid))); + + // getAllIDs includes deleted items but skips the FxA login. + ok((await engine._store.getAllIDs())[guid]); + let deletedLogin = await engine._store._getLoginFromGUID(guid); + + equal(deletedLogin.hostname, null, "deleted login hostname"); + equal( + deletedLogin.formActionOrigin, + null, + "deleted login formActionOrigin" + ); + equal(deletedLogin.formSubmitURL, null, "deleted login formSubmitURL"); + equal(deletedLogin.httpRealm, null, "deleted login httpRealm"); + equal(deletedLogin.username, null, "deleted login username"); + equal(deletedLogin.password, null, "deleted login password"); + equal(deletedLogin.usernameField, "", "deleted login usernameField"); + equal(deletedLogin.passwordField, "", "deleted login passwordField"); + equal(deletedLogin.unknownFields, null, "deleted login unknownFields"); + equal(deletedLogin.timeCreated, 0, "deleted login timeCreated"); + equal(deletedLogin.timeLastUsed, 0, "deleted login timeLastUsed"); + equal(deletedLogin.timesUsed, 0, "deleted login timesUsed"); + + // These fields are not reset when the login is removed. + equal(deletedLogin.guid, guid, "deleted login guid"); + equal(deletedLogin.everSynced, true, "deleted login everSynced"); + equal(deletedLogin.syncCounter, 0, "deleted login syncCounter"); + ok( + deletedLogin.timePasswordChanged > 0, + "deleted login timePasswordChanged" + ); + } finally { + await engine._tracker.stop(); + + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_incoming() { + _("Test syncing incoming records"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + const checkFields = [ + "formSubmitURL", + "hostname", + "httpRealm", + "username", + "password", + "usernameField", + "passwordField", + "timeCreated", + ]; + + let guid1 = Utils.makeGUID(); + let details = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: "camel", + password: "llama", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + // This test creates a remote server record and then verifies that the login + // has been added locally after the sync occurs. + _("Create remote login"); + collection.insertRecord(Object.assign({}, details, { id: guid1 })); + + _("Perform sync when remote login has been added"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + equal(logins.length, 1); + + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + checkFields.forEach(field => { + equal(logins[0][field], details[field]); + }); + equal(logins[0].timePasswordChanged, details.timePasswordChanged); + equal(logins[0].syncCounter, 0); + equal(logins[0].everSynced, true); + + // Modify the password within the remote record and then sync again. + _("Perform sync when remote login's password has been modified"); + let newTime = Date.now(); + collection.updateRecord( + guid1, + cleartext => { + cleartext.password = "alpaca"; + }, + newTime / 1000 + 10 + ); + + await engine.setLastSync(newTime / 1000 - 30); + await sync_engine_and_validate_telem(engine, false); + + logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + equal(logins.length, 1); + + details.password = "alpaca"; + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + checkFields.forEach(field => { + equal(logins[0][field], details[field]); + }); + ok(logins[0].timePasswordChanged > details.timePasswordChanged); + equal(logins[0].syncCounter, 0); + equal(logins[0].everSynced, true); + + // Modify the username within the remote record and then sync again. + _("Perform sync when remote login's username has been modified"); + newTime = Date.now(); + collection.updateRecord( + guid1, + cleartext => { + cleartext.username = "guanaco"; + }, + newTime / 1000 + 10 + ); + + await engine.setLastSync(newTime / 1000 - 30); + await sync_engine_and_validate_telem(engine, false); + + logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + equal(logins.length, 1); + + details.username = "guanaco"; + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + checkFields.forEach(field => { + equal(logins[0][field], details[field]); + }); + ok(logins[0].timePasswordChanged > details.timePasswordChanged); + equal(logins[0].syncCounter, 0); + equal(logins[0].everSynced, true); + + // Mark the remote record as deleted and then sync again. + _("Perform sync when remote login has been marked for deletion"); + newTime = Date.now(); + collection.updateRecord( + guid1, + cleartext => { + cleartext.deleted = true; + }, + newTime / 1000 + 10 + ); + + await engine.setLastSync(newTime / 1000 - 30); + await sync_engine_and_validate_telem(engine, false); + + logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + equal(logins.length, 0); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_incoming_deleted() { + _("Test syncing incoming deleted records"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let details2 = { + formSubmitURL: "https://www.example.org", + hostname: "https://www.example.org", + httpRealm: null, + username: "capybara", + password: "beaver", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + deleted: true, + }; + + try { + // This test creates a remote server record that has been deleted + // and then verifies that the login is not imported locally. + _("Create remote login"); + collection.insertRecord(Object.assign({}, details2, { id: guid1 })); + + _("Perform sync when remote login has been deleted"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + equal(logins.length, 0); + ok(!(await engine._store.getAllIDs())[guid1]); + ok(!(await engine._store.itemExists(guid1))); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_incoming_deleted_localchanged_remotenewer() { + _( + "Test syncing incoming deleted records where the local login has been changed but the remote record is newer" + ); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + let loginInfo = new LoginInfo( + "http://mozilla.com", + "http://mozilla.com", + null, + "kangaroo", + "kaola", + "username", + "password" + ); + let login = await Services.logins.addLoginAsync(loginInfo); + let guid = login.QueryInterface(Ci.nsILoginMetaInfo).guid; + + try { + _("Perform sync on new login"); + await sync_engine_and_validate_telem(engine, false); + + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://mozilla.com", + }); + foundLogins[0].password = "wallaby"; + Services.logins.modifyLogin(login, foundLogins[0]); + + // Use a time in the future to ensure that the remote record is newer. + collection.updateRecord( + guid, + cleartext => { + cleartext.deleted = true; + }, + Date.now() / 1000 + 1000 + ); + + _( + "Perform sync when remote login has been deleted and local login has been changed" + ); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://mozilla.com", + }); + equal(logins.length, 0); + ok(await engine._store.getAllIDs()); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_incoming_deleted_localchanged_localnewer() { + _( + "Test syncing incoming deleted records where the local login has been changed but the local record is newer" + ); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + let loginInfo = new LoginInfo( + "http://www.mozilla.com", + "http://www.mozilla.com", + null, + "lion", + "tiger", + "username", + "password" + ); + let login = await Services.logins.addLoginAsync(loginInfo); + let guid = login.QueryInterface(Ci.nsILoginMetaInfo).guid; + + try { + _("Perform sync on new login"); + await sync_engine_and_validate_telem(engine, false); + + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: "http://www.mozilla.com", + }); + foundLogins[0].password = "cheetah"; + Services.logins.modifyLogin(login, foundLogins[0]); + + // Use a time in the past to ensure that the local record is newer. + collection.updateRecord( + guid, + cleartext => { + cleartext.deleted = true; + }, + Date.now() / 1000 - 1000 + ); + + _( + "Perform sync when remote login has been deleted and local login has been changed" + ); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "http://www.mozilla.com", + }); + equal(logins.length, 1); + equal(logins[0].password, "cheetah"); + equal(logins[0].syncCounter, 0); + equal(logins[0].everSynced, true); + ok(await engine._store.getAllIDs()); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_incoming_no_formactionorigin() { + _("Test syncing incoming a record where there is no formActionOrigin"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + const checkFields = [ + "formSubmitURL", + "hostname", + "httpRealm", + "username", + "password", + "usernameField", + "passwordField", + "timeCreated", + ]; + + let guid1 = Utils.makeGUID(); + let details = { + formSubmitURL: "", + hostname: "https://www.example.com", + httpRealm: null, + username: "rabbit", + password: "squirrel", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + // This test creates a remote server record and then verifies that the login + // has been added locally after the sync occurs. + _("Create remote login"); + collection.insertRecord(Object.assign({}, details, { id: guid1 })); + + _("Perform sync when remote login has been added"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + formActionOrigin: "", + }); + equal(logins.length, 1); + + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + checkFields.forEach(field => { + equal(logins[0][field], details[field]); + }); + equal(logins[0].timePasswordChanged, details.timePasswordChanged); + equal(logins[0].syncCounter, 0); + equal(logins[0].everSynced, true); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_password_dupe() { + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + let guid2 = Utils.makeGUID(); + let cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: "foo", + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Math.round(Date.now()), + timePasswordChanged: Math.round(Date.now()), + }; + rec1.cleartext = cleartext; + + _("Create remote record with same details and guid1"); + collection.insert(guid1, encryptPayload(rec1.cleartext)); + + _("Create remote record with guid2"); + collection.insert(guid2, encryptPayload(cleartext)); + + _("Create local record with same details and guid1"); + await engine._store.create(rec1); + + try { + _("Perform sync"); + await sync_engine_and_validate_telem(engine, true); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid2); + equal(null, collection.payload(guid1)); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_updated_null_password_sync() { + _("Ensure updated null login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + let remoteDetails = { + formSubmitURL: "https://www.nullupdateexample.com", + hostname: "https://www.nullupdateexample.com", + httpRealm: null, + username: null, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + let localDetails = { + formSubmitURL: "https://www.nullupdateexample.com", + hostname: "https://www.nullupdateexample.com", + httpRealm: null, + username: "foo", + password: "foobar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + _("Create remote record with same details and guid1"); + collection.insertRecord(Object.assign({}, remoteDetails, { id: guid1 })); + + try { + _("Create local updated login with null password"); + await engine._store.update(Object.assign({}, localDetails, { id: guid2 })); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.nullupdateexample.com", + }); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_updated_undefined_password_sync() { + _("Ensure updated undefined login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + let remoteDetails = { + formSubmitURL: "https://www.undefinedupdateexample.com", + hostname: "https://www.undefinedupdateexample.com", + httpRealm: null, + username: undefined, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + let localDetails = { + formSubmitURL: "https://www.undefinedupdateexample.com", + hostname: "https://www.undefinedupdateexample.com", + httpRealm: null, + username: "foo", + password: "foobar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + _("Create remote record with same details and guid1"); + collection.insertRecord(Object.assign({}, remoteDetails, { id: guid1 })); + + try { + _("Create local updated login with undefined password"); + await engine._store.update(Object.assign({}, localDetails, { id: guid2 })); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.undefinedupdateexample.com", + }); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_new_null_password_sync() { + _("Ensure new null login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + rec1.cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: null, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + _("Create local login with null password"); + await engine._store.create(rec1); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + + equal(logins.length, 1); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, null); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, undefined); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, ""); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_new_undefined_password_sync() { + _("Ensure new undefined login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + rec1.cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: undefined, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + _("Create local login with undefined password"); + await engine._store.create(rec1); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://www.example.com", + }); + + equal(logins.length, 1); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, null); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, undefined); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, ""); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_password_validation() { + // This test isn't in test_password_validator to avoid duplicating cleanup. + _("Ensure that if a password validation happens, it ends up in the ping"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + Svc.PrefBranch.setIntPref("engine.passwords.validation.interval", 0); + Svc.PrefBranch.setIntPref( + "engine.passwords.validation.percentageChance", + 100 + ); + Svc.PrefBranch.setIntPref("engine.passwords.validation.maxRecords", -1); + Svc.PrefBranch.setBoolPref("engine.passwords.validation.enabled", true); + + try { + let ping = await wait_for_ping(() => Service.sync()); + + let engineInfo = ping.engines.find(e => e.name == "passwords"); + ok(engineInfo, "Engine should be in ping"); + + let validation = engineInfo.validation; + ok(validation, "Engine should have validation info"); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_roundtrip_unknown_fields() { + _( + "Testing that unknown fields from other clients get roundtripped back to server" + ); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + enableValidationPrefs(); + + _("Add login with older password change time to replace during first sync"); + let oldLogin; + { + let login = new LoginInfo( + "https://mozilla.com", + "", + null, + "us3r", + "0ldpa55", + "", + "" + ); + await Services.logins.addLoginAsync(login); + + let props = new PropertyBag(); + let localPasswordChangeTime = Math.round( + Date.now() - 1 * 60 * 60 * 24 * 1000 + ); + props.setProperty("timePasswordChanged", localPasswordChangeTime); + Services.logins.modifyLogin(login, props); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://mozilla.com", + }); + equal(logins.length, 1, "Should find old login in login manager"); + oldLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + equal(oldLogin.timePasswordChanged, localPasswordChangeTime); + + let rec = new LoginRec("passwords", oldLogin.guid); + rec.hostname = oldLogin.origin; + rec.formSubmitURL = oldLogin.formActionOrigin; + rec.httpRealm = oldLogin.httpRealm; + rec.username = oldLogin.username; + // Change the password and bump the password change time to ensure we prefer + // the remote one during reconciliation. + rec.password = "n3wpa55"; + rec.usernameField = oldLogin.usernameField; + rec.passwordField = oldLogin.usernameField; + rec.timeCreated = oldLogin.timeCreated; + rec.timePasswordChanged = Math.round(Date.now()); + + // pretend other clients have some snazzy new fields + // we don't quite understand yet + rec.cleartext.someStrField = "I am a str"; + rec.cleartext.someObjField = { newField: "I am a new field" }; + collection.insert(oldLogin.guid, encryptPayload(rec.cleartext)); + } + + await engine._tracker.stop(); + + try { + await sync_engine_and_validate_telem(engine, false); + + let logins = await Services.logins.searchLoginsAsync({ + origin: "https://mozilla.com", + }); + equal( + logins[0].password, + "n3wpa55", + "Should update local password for older login" + ); + let expectedUnknowns = JSON.stringify({ + someStrField: "I am a str", + someObjField: { newField: "I am a new field" }, + }); + // Check that the local record has all unknown fields properly + // stringified + equal(logins[0].unknownFields, expectedUnknowns); + + // Check that the server has the unknown fields unfurled and on the + // top-level record + let serverRec = collection.cleartext(oldLogin.guid); + equal(serverRec.someStrField, "I am a str"); + equal(serverRec.someObjField.newField, "I am a new field"); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_new_passwords_from_csv() { + _("Test syncing records imported from a csv file"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("passwords"); + + engine._tracker.start(); + + let data = [ + { + hostname: "https://example.com", + url: "https://example.com/path", + username: "exampleuser", + password: "examplepassword", + }, + { + hostname: "https://mozilla.org", + url: "https://mozilla.org", + username: "mozillauser", + password: "mozillapassword", + }, + { + hostname: "https://www.example.org", + url: "https://www.example.org/example1/example2", + username: "person", + password: "mypassword", + }, + ]; + + let csvData = ["url,username,login_password"]; + for (let row of data) { + csvData.push(row.url + "," + row.username + "," + row.password); + } + + let csvFile = FileTestUtils.getTempFile(`firefox_logins.csv`); + await IOUtils.writeUTF8(csvFile.path, csvData.join("\r\n")); + + await LoginCSVImport.importFromCSV(csvFile.path); + + equal( + engine._tracker.score, + SCORE_INCREMENT_XLARGE, + "Should only get one update notification for import" + ); + + _("Ensure that the csv import is correct"); + for (let item of data) { + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: item.hostname, + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 1); + equal(foundLogins[0].everSynced, false); + equal(foundLogins[0].username, item.username); + equal(foundLogins[0].password, item.password); + } + + _("Perform sync after modifying the password"); + await sync_engine_and_validate_telem(engine, false); + + _("Verify that the sync counter and status are updated"); + for (let item of data) { + let foundLogins = await Services.logins.searchLoginsAsync({ + origin: item.hostname, + }); + equal(foundLogins.length, 1); + equal(foundLogins[0].syncCounter, 0); + equal(foundLogins[0].everSynced, true); + equal(foundLogins[0].username, item.username); + equal(foundLogins[0].password, item.password); + item.guid = foundLogins[0].guid; + } + + equal(Object.keys(await engine.getChangedIDs()), 0); + equal(collection.count(), 3); + + for (let item of data) { + // The remote login should have the imported username and password. + let newRec = collection.cleartext(item.guid); + equal(newRec.username, item.username); + equal(newRec.password, item.password); + } +}); diff --git a/services/sync/tests/unit/test_password_store.js b/services/sync/tests/unit/test_password_store.js new file mode 100644 index 0000000000..ed393d6241 --- /dev/null +++ b/services/sync/tests/unit/test_password_store.js @@ -0,0 +1,398 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +async function checkRecord( + name, + record, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + recordIsUpdated +) { + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + let logins = await Services.logins.searchLoginsAsync({ + origin: record.hostname, + formActionOrigin: record.formSubmitURL, + }); + + _("Record" + name + ":" + JSON.stringify(logins)); + _("Count" + name + ":" + logins.length); + + Assert.equal(logins.length, expectedCount); + + if (expectedCount > 0) { + Assert.ok(!!(await store.getAllIDs())[record.id]); + let stored_record = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + + if (timeCreated !== undefined) { + Assert.equal(stored_record.timeCreated, expectedTimeCreated); + } + + if (timePasswordChanged !== undefined) { + if (recordIsUpdated) { + Assert.ok( + stored_record.timePasswordChanged >= expectedTimePasswordChanged + ); + } else { + Assert.equal( + stored_record.timePasswordChanged, + expectedTimePasswordChanged + ); + } + return stored_record.timePasswordChanged; + } + } else { + Assert.ok(!(await store.getAllIDs())[record.id]); + } + return undefined; +} + +async function changePassword( + name, + hostname, + password, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + insert, + recordIsUpdated +) { + const BOGUS_GUID = "zzzzzz" + hostname; + let record = new LoginRec("passwords", BOGUS_GUID); + record.cleartext = { + id: BOGUS_GUID, + hostname, + formSubmitURL: hostname, + username: "john", + password, + usernameField: "username", + passwordField: "password", + }; + + if (timeCreated !== undefined) { + record.timeCreated = timeCreated; + } + + if (timePasswordChanged !== undefined) { + record.timePasswordChanged = timePasswordChanged; + } + + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + if (insert) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch([record], countTelemetry)).length, + 0 + ); + } + + return checkRecord( + name, + record, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + recordIsUpdated + ); +} + +async function test_apply_records_with_times( + hostname, + timeCreated, + timePasswordChanged +) { + // The following record is going to be inserted in the store and it needs + // to be found there. Then its timestamps are going to be compared to + // the expected values. + await changePassword( + " ", + hostname, + "password", + 1, + timeCreated, + timeCreated, + timePasswordChanged, + timePasswordChanged, + true + ); +} + +async function test_apply_multiple_records_with_times() { + // The following records are going to be inserted in the store and they need + // to be found there. Then their timestamps are going to be compared to + // the expected values. + await changePassword( + "A", + "http://foo.a.com", + "password", + 1, + undefined, + undefined, + undefined, + undefined, + true + ); + await changePassword( + "B", + "http://foo.b.com", + "password", + 1, + 1000, + 1000, + undefined, + undefined, + true + ); + await changePassword( + "C", + "http://foo.c.com", + "password", + 1, + undefined, + undefined, + 1000, + 1000, + true + ); + await changePassword( + "D", + "http://foo.d.com", + "password", + 1, + 1000, + 1000, + 1000, + 1000, + true + ); + + // The following records are not going to be inserted in the store and they + // are not going to be found there. + await changePassword( + "NotInStoreA", + "http://foo.aaaa.com", + "password", + 0, + undefined, + undefined, + undefined, + undefined, + false + ); + await changePassword( + "NotInStoreB", + "http://foo.bbbb.com", + "password", + 0, + 1000, + 1000, + undefined, + undefined, + false + ); + await changePassword( + "NotInStoreC", + "http://foo.cccc.com", + "password", + 0, + undefined, + undefined, + 1000, + 1000, + false + ); + await changePassword( + "NotInStoreD", + "http://foo.dddd.com", + "password", + 0, + 1000, + 1000, + 1000, + 1000, + false + ); +} + +async function test_apply_same_record_with_different_times() { + // The following record is going to be inserted multiple times in the store + // and it needs to be found there. Then its timestamps are going to be + // compared to the expected values. + + /* eslint-disable no-unused-vars */ + /* The eslint linter thinks that timePasswordChanged is unused, even though + it is passed as an argument to changePassword. */ + var timePasswordChanged = 100; + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 100, + 100, + 100, + timePasswordChanged, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 100, + 100, + 800, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 500, + 100, + 800, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password2", + 1, + 500, + 100, + 1536213005222, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password2", + 1, + 500, + 100, + 800, + timePasswordChanged, + true, + true + ); + /* eslint-enable no-unused-vars */ +} + +async function test_LoginRec_toString(store, recordData) { + let rec = await store.createRecord(recordData.id); + ok(rec); + ok(!rec.toString().includes(rec.password)); +} + +add_task(async function run_test() { + const BOGUS_GUID_A = "zzzzzzzzzzzz"; + const BOGUS_GUID_B = "yyyyyyyyyyyy"; + let recordA = new LoginRec("passwords", BOGUS_GUID_A); + let recordB = new LoginRec("passwords", BOGUS_GUID_B); + recordA.cleartext = { + id: BOGUS_GUID_A, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + httpRealm: "secure", + username: "john", + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordB.cleartext = { + id: BOGUS_GUID_B, + hostname: "http://foo.baz.com", + formSubmitURL: "http://foo.baz.com", + username: "john", + password: "smith", + usernameField: "username", + passwordField: "password", + unknownStr: "an unknown string from another field", + }; + + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + try { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch([recordA, recordB], countTelemetry)) + .length, + 0 + ); + + // Only the good record makes it to Services.logins. + let badLogins = await Services.logins.searchLoginsAsync({ + origin: recordA.hostname, + formActionOrigin: recordA.formSubmitURL, + httpRealm: recordA.httpRealm, + }); + let goodLogins = await Services.logins.searchLoginsAsync({ + origin: recordB.hostname, + formActionOrigin: recordB.formSubmitURL, + }); + + _("Bad: " + JSON.stringify(badLogins)); + _("Good: " + JSON.stringify(goodLogins)); + _("Count: " + badLogins.length + ", " + goodLogins.length); + + Assert.equal(goodLogins.length, 1); + Assert.equal(badLogins.length, 0); + + // applyIncoming should've put any unknown fields from the server + // into a catch-all unknownFields field + Assert.equal( + goodLogins[0].unknownFields, + JSON.stringify({ + unknownStr: "an unknown string from another field", + }) + ); + + Assert.ok(!!(await store.getAllIDs())[BOGUS_GUID_B]); + Assert.ok(!(await store.getAllIDs())[BOGUS_GUID_A]); + + await test_LoginRec_toString(store, recordB); + + await test_apply_records_with_times( + "http://afoo.baz.com", + undefined, + undefined + ); + await test_apply_records_with_times("http://bfoo.baz.com", 1000, undefined); + await test_apply_records_with_times("http://cfoo.baz.com", undefined, 2000); + await test_apply_records_with_times("http://dfoo.baz.com", 1000, 2000); + + await test_apply_multiple_records_with_times(); + + await test_apply_same_record_with_different_times(); + } finally { + await store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_password_tracker.js b/services/sync/tests/unit/test_password_tracker.js new file mode 100644 index 0000000000..77b46d1d2c --- /dev/null +++ b/services/sync/tests/unit/test_password_tracker.js @@ -0,0 +1,248 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PasswordEngine, LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +let store; +let tracker; + +add_task(async function setup() { + await Service.engineManager.register(PasswordEngine); + engine = Service.engineManager.get("passwords"); + store = engine._store; + tracker = engine._tracker; +}); + +add_task(async function test_tracking() { + let recordNum = 0; + + _("Verify we've got an empty tracker to work with."); + let changes = await engine.getChangedIDs(); + do_check_empty(changes); + + let exceptionHappened = false; + try { + await tracker.getChangedIDs(); + } catch (ex) { + exceptionHappened = true; + } + ok(exceptionHappened, "tracker does not keep track of changes"); + + async function createPassword() { + _("RECORD NUM: " + recordNum); + let record = new LoginRec("passwords", "GUID" + recordNum); + record.cleartext = { + id: "GUID" + recordNum, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + username: "john" + recordNum, + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordNum++; + let login = store._nsLoginInfoFromRecord(record); + await Services.logins.addLoginAsync(login); + await tracker.asyncObserver.promiseObserversComplete(); + } + + try { + tracker.start(); + await createPassword(); + changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, 1); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(changes.GUID0.counter, 1); + Assert.ok(typeof changes.GUID0.modified, "number"); + + _("Starting twice won't do any harm."); + tracker.start(); + await createPassword(); + changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, 2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(changes.GUID0.counter, 1); + Assert.equal(changes.GUID1.counter, 1); + + // The tracker doesn't keep track of changes, so 3 changes + // should still be returned, but the score is not updated. + _("Let's stop tracking again."); + tracker.resetScore(); + await tracker.stop(); + await createPassword(); + changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, 3); + Assert.equal(tracker.score, 0); + Assert.equal(changes.GUID0.counter, 1); + Assert.equal(changes.GUID1.counter, 1); + Assert.equal(changes.GUID2.counter, 1); + + _("Stopping twice won't do any harm."); + await tracker.stop(); + await createPassword(); + changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, 4); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await store.wipe(); + tracker.resetScore(); + await tracker.stop(); + } +}); + +add_task(async function test_onWipe() { + _("Verify we've got an empty tracker to work with."); + const changes = await engine.getChangedIDs(); + do_check_empty(changes); + Assert.equal(tracker.score, 0); + + try { + _("A store wipe should increment the score"); + tracker.start(); + await store.wipe(); + await tracker.asyncObserver.promiseObserversComplete(); + + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + } finally { + tracker.resetScore(); + await tracker.stop(); + } +}); + +add_task(async function test_removeAllLogins() { + let recordNum = 0; + _("Verify that all tracked logins are removed."); + + // Perform this test twice, the first time where a sync is not performed + // between adding and removing items and the second time where a sync is + // performed. In the former case, the logins will just be deleted because + // they have never been synced, so they won't be detected as changes. In + // the latter case, the logins have been synced so they will be marked for + // deletion. + for (let syncBeforeRemove of [false, true]) { + async function createPassword() { + _("RECORD NUM: " + recordNum); + let record = new LoginRec("passwords", "GUID" + recordNum); + record.cleartext = { + id: "GUID" + recordNum, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + username: "john" + recordNum, + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordNum++; + let login = store._nsLoginInfoFromRecord(record); + await Services.logins.addLoginAsync(login); + + await tracker.asyncObserver.promiseObserversComplete(); + } + try { + _("Tell tracker to start tracking changes"); + tracker.start(); + await createPassword(); + await createPassword(); + let changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, 2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + + if (syncBeforeRemove) { + let logins = await Services.logins.getAllLogins(); + for (let login of logins) { + engine.markSynced(login.guid); + } + } + + _("Tell sync to remove all logins"); + Services.logins.removeAllUserFacingLogins(); + await tracker.asyncObserver.promiseObserversComplete(); + changes = await engine.getChangedIDs(); + do_check_attribute_count(changes, syncBeforeRemove ? 2 : 0); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 5); + + let deletedGuids = await engine._store.getAllIDs(); + if (syncBeforeRemove) { + for (let guid in deletedGuids) { + let deletedLogin = await engine._store._getLoginFromGUID(guid); + + Assert.equal(deletedLogin.hostname, null, "deleted login hostname"); + Assert.equal( + deletedLogin.formActionOrigin, + null, + "deleted login formActionOrigin" + ); + Assert.equal( + deletedLogin.formSubmitURL, + null, + "deleted login formSubmitURL" + ); + Assert.equal(deletedLogin.httpRealm, null, "deleted login httpRealm"); + Assert.equal(deletedLogin.username, null, "deleted login username"); + Assert.equal(deletedLogin.password, null, "deleted login password"); + Assert.equal( + deletedLogin.usernameField, + "", + "deleted login usernameField" + ); + Assert.equal( + deletedLogin.passwordField, + "", + "deleted login passwordField" + ); + Assert.equal( + deletedLogin.unknownFields, + null, + "deleted login unknownFields" + ); + Assert.equal( + deletedLogin.timeCreated, + 0, + "deleted login timeCreated" + ); + Assert.equal( + deletedLogin.timeLastUsed, + 0, + "deleted login timeLastUsed" + ); + Assert.equal(deletedLogin.timesUsed, 0, "deleted login timesUsed"); + + // These fields are not reset when the login is removed. + Assert.ok(deletedLogin.guid.startsWith("GUID"), "deleted login guid"); + Assert.equal( + deletedLogin.everSynced, + true, + "deleted login everSynced" + ); + Assert.equal( + deletedLogin.syncCounter, + 2, + "deleted login syncCounter" + ); + Assert.ok( + deletedLogin.timePasswordChanged > 0, + "deleted login timePasswordChanged" + ); + } + } else { + Assert.equal( + Object.keys(deletedGuids).length, + 0, + "no logins remain after removeAllUserFacingLogins" + ); + } + } finally { + _("Clean up."); + await store.wipe(); + tracker.resetScore(); + await tracker.stop(); + } + } +}); diff --git a/services/sync/tests/unit/test_password_validator.js b/services/sync/tests/unit/test_password_validator.js new file mode 100644 index 0000000000..445b119e1d --- /dev/null +++ b/services/sync/tests/unit/test_password_validator.js @@ -0,0 +1,176 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PasswordValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "11111", + guid: "11111", + hostname: "https://www.11111.com", + formSubmitURL: "https://www.11111.com", + password: "qwerty123", + passwordField: "pass", + username: "foobar", + usernameField: "user", + httpRealm: null, + }, + { + id: "22222", + guid: "22222", + hostname: "https://www.22222.org", + formSubmitURL: "https://www.22222.org", + password: "hunter2", + passwordField: "passwd", + username: "baz12345", + usernameField: "user", + httpRealm: null, + }, + { + id: "33333", + guid: "33333", + hostname: "https://www.33333.com", + formSubmitURL: "https://www.33333.com", + password: "p4ssw0rd", + passwordField: "passwad", + username: "quux", + usernameField: "user", + httpRealm: null, + }, + ], + client: [ + { + id: "11111", + guid: "11111", + hostname: "https://www.11111.com", + formSubmitURL: "https://www.11111.com", + password: "qwerty123", + passwordField: "pass", + username: "foobar", + usernameField: "user", + httpRealm: null, + }, + { + id: "22222", + guid: "22222", + hostname: "https://www.22222.org", + formSubmitURL: "https://www.22222.org", + password: "hunter2", + passwordField: "passwd", + username: "baz12345", + usernameField: "user", + httpRealm: null, + }, + { + id: "33333", + guid: "33333", + hostname: "https://www.33333.com", + formSubmitURL: "https://www.33333.com", + password: "p4ssw0rd", + passwordField: "passwad", + username: "quux", + usernameField: "user", + httpRealm: null, + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new PasswordValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 3); + equal(records.length, 3); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); + +add_task(async function test_missing() { + let validator = new PasswordValidator(); + { + let { server, client } = getDummyServerAndClient(); + + client.pop(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 2); + equal(records.length, 3); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + expected.clientMissing.push("33333"); + deepEqual(problemData, expected); + } + { + let { server, client } = getDummyServerAndClient(); + + server.pop(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 3); + equal(records.length, 2); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + expected.serverMissing.push("33333"); + deepEqual(problemData, expected); + } +}); + +add_task(async function test_deleted() { + let { server, client } = getDummyServerAndClient(); + let deletionRecord = { id: "444444", guid: "444444", deleted: true }; + + server.push(deletionRecord); + let validator = new PasswordValidator(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 3); + equal(records.length, 4); + deepEqual(deletedRecords, [deletionRecord]); + + let expected = validator.emptyProblemData(); + deepEqual(problemData, expected); +}); + +add_task(async function test_duplicates() { + let validator = new PasswordValidator(); + { + let { server, client } = getDummyServerAndClient(); + client.push(Cu.cloneInto(client[0], {})); + + let { problemData } = await validator.compareClientWithServer( + client, + server + ); + + let expected = validator.emptyProblemData(); + expected.clientDuplicates.push("11111"); + deepEqual(problemData, expected); + } + { + let { server, client } = getDummyServerAndClient(); + server.push(Cu.cloneInto(server[server.length - 1], {})); + + let { problemData } = await validator.compareClientWithServer( + client, + server + ); + + let expected = validator.emptyProblemData(); + expected.duplicates.push("33333"); + deepEqual(problemData, expected); + } +}); diff --git a/services/sync/tests/unit/test_postqueue.js b/services/sync/tests/unit/test_postqueue.js new file mode 100644 index 0000000000..2e687bce11 --- /dev/null +++ b/services/sync/tests/unit/test_postqueue.js @@ -0,0 +1,985 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +let { PostQueue } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); + +function makeRecord(nbytes) { + return { + toJSON: () => ({ payload: "x".repeat(nbytes) }), + }; +} + +// Note: This is 14 bytes. Tests make assumptions about this (even if it's just +// in setting config.max_request_bytes to a specific value). +makeRecord.nonPayloadOverhead = JSON.stringify(makeRecord(0).toJSON()).length; + +// Gives how many encoded bytes a request with the given payload +// sizes will be (assuming the records were created by makeRecord) +// requestBytesFor([20]) => 22, requestBytesFor([20, 20]) => 43 +function requestBytesFor(recordPayloadByteCounts) { + let requestBytes = 1; + for (let size of recordPayloadByteCounts) { + requestBytes += size + 1 + makeRecord.nonPayloadOverhead; + } + return requestBytes; +} + +function makePostQueue(config, lastModTime, responseGenerator) { + let stats = { + posts: [], + batches: [], + }; + let poster = (data, headers, batch, commit) => { + let payloadBytes = 0; + let numRecords = 0; + for (let record of JSON.parse(data)) { + if (config.max_record_payload_bytes) { + less( + record.payload.length, + config.max_record_payload_bytes, + "PostQueue should respect max_record_payload_bytes" + ); + } + payloadBytes += record.payload.length; + ++numRecords; + } + + let thisPost = { + nbytes: data.length, + batch, + commit, + payloadBytes, + numRecords, + }; + + if (headers.length) { + thisPost.headers = headers; + } + + // check that we respected the provided limits for the post + if (config.max_post_records) { + lessOrEqual( + numRecords, + config.max_post_records, + "PostQueue should respect max_post_records" + ); + } + + if (config.max_post_bytes) { + less( + payloadBytes, + config.max_post_bytes, + "PostQueue should respect max_post_bytes" + ); + } + + if (config.max_request_bytes) { + less( + thisPost.nbytes, + config.max_request_bytes, + "PostQueue should respect max_request_bytes" + ); + } + + stats.posts.push(thisPost); + + // Call this now so we can check if there's a batch id in it. + // Kind of cludgey, but allows us to have the correct batch id even + // before the next post is made. + let nextResponse = responseGenerator.next().value; + + // Record info for the batch. + + let curBatch = stats.batches[stats.batches.length - 1]; + // If there's no batch, it committed, or we requested a new one, + // then we need to start a new one. + if (!curBatch || batch == "true" || curBatch.didCommit) { + curBatch = { + posts: 0, + payloadBytes: 0, + numRecords: 0, + didCommit: false, + batch, + serverBatch: false, + }; + if (nextResponse.obj && nextResponse.obj.batch) { + curBatch.batch = nextResponse.obj.batch; + curBatch.serverBatch = true; + } + stats.batches.push(curBatch); + } + + // If we provided a batch id, it must be the same as the current batch + if (batch && batch != "true") { + equal(curBatch.batch, batch); + } + + curBatch.posts += 1; + curBatch.payloadBytes += payloadBytes; + curBatch.numRecords += numRecords; + curBatch.didCommit = commit; + + // if this is an actual server batch (or it's a one-shot batch), check that + // we respected the provided total limits + if (commit && (batch == "true" || curBatch.serverBatch)) { + if (config.max_total_records) { + lessOrEqual( + curBatch.numRecords, + config.max_total_records, + "PostQueue should respect max_total_records" + ); + } + + if (config.max_total_bytes) { + less( + curBatch.payloadBytes, + config.max_total_bytes, + "PostQueue should respect max_total_bytes" + ); + } + } + + return Promise.resolve(nextResponse); + }; + + let done = () => {}; + let pq = new PostQueue(poster, lastModTime, config, getTestLogger(), done); + return { pq, stats }; +} + +add_task(async function test_simple() { + let config = { + max_request_bytes: 1000, + max_record_payload_bytes: 1000, + }; + + const time = 11111111; + + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + await pq.enqueue(makeRecord(10)); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10]), + payloadBytes: 10, + numRecords: 1, + commit: true, // we don't know if we have batch semantics, so committed. + headers: [["x-if-unmodified-since", time]], + batch: "true", + }, + ]); + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 1, + didCommit: true, + batch: "true", + serverBatch: false, + }, + ]); +}); + +// Test we do the right thing when we need to make multiple posts when there +// are no batch semantics +add_task(async function test_max_request_bytes_no_batch() { + let config = { + max_request_bytes: 50, + max_record_payload_bytes: 50, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 200, + "x-last-modified": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + let payloadSize = 20 - makeRecord.nonPayloadOverhead; + await pq.enqueue(makeRecord(payloadSize)); // total size now 22 bytes - "[" + record + "]" + await pq.enqueue(makeRecord(payloadSize)); // total size now 43 bytes - "[" + record + "," + record + "]" + await pq.enqueue(makeRecord(payloadSize)); // this will exceed our byte limit, so will be in the 2nd POST. + await pq.flush(true); + deepEqual(stats.posts, [ + { + nbytes: 43, // 43 for the first part + payloadBytes: payloadSize * 2, + numRecords: 2, + commit: false, + headers: [["x-if-unmodified-since", time]], + batch: "true", + }, + { + nbytes: 22, + payloadBytes: payloadSize, + numRecords: 1, + commit: false, // we know we aren't in a batch, so never commit. + headers: [["x-if-unmodified-since", time + 100]], + batch: null, + }, + ]); + equal(stats.batches.filter(x => x.didCommit).length, 0); + equal(pq.lastModified, time + 200); +}); + +add_task(async function test_max_record_payload_bytes_no_batch() { + let config = { + max_request_bytes: 100, + max_record_payload_bytes: 50, + }; + + const time = 11111111; + + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + // Should trigger when the record really is too large to fit + let { enqueued } = await pq.enqueue(makeRecord(51)); + ok(!enqueued); + // Shouldn't trigger when the encoded record is too big + ok( + (await pq.enqueue(makeRecord(50 - makeRecord.nonPayloadOverhead))).enqueued + ); // total size now 52 bytes - "[" + record + "]" + ok( + (await pq.enqueue(makeRecord(46 - makeRecord.nonPayloadOverhead))).enqueued + ); // total size now 99 bytes - "[" + record0 + "," + record1 + "]" + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: 99, + payloadBytes: 50 + 46 - makeRecord.nonPayloadOverhead * 2, + numRecords: 2, + commit: true, // we know we aren't in a batch, so never commit. + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 50 + 46 - makeRecord.nonPayloadOverhead * 2, + numRecords: 2, + didCommit: true, + batch: "true", + serverBatch: false, + }, + ]); + + equal(pq.lastModified, time + 100); +}); + +// Batch tests. + +// Test making a single post when batch semantics are in place. + +add_task(async function test_single_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 100, + max_total_records: 200, + max_record_payload_bytes: 1000, + }; + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10]), + numRecords: 1, + payloadBytes: 10, + commit: true, // we don't know if we have batch semantics, so committed. + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 1, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); +}); + +// Test we do the right thing when we need to make multiple posts due to +// max_post_bytes when there are batch semantics in place. +add_task(async function test_max_post_bytes_batch() { + let config = { + max_post_bytes: 50, + max_post_records: 4, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 50, + max_request_bytes: 4000, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(20))).enqueued); // 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // 40 + // 60 would overflow, so post + ok((await pq.enqueue(makeRecord(20))).enqueued); // 20 + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when we need to make multiple posts due to +// max_request_bytes when there are batch semantics in place. +add_task(async function test_max_request_bytes_batch() { + let config = { + max_post_bytes: 60, + max_post_records: 40, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 500, + max_request_bytes: 100, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 10, request: 26 (10 + 14 + 2) + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 20, request: 51 (10 + 14 + 1) * 2 + 1 + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 30, request: 76 (10 + 14 + 1) * 3 + 1 + // 1 more would be post: 40 (fine), request: 101, So we should post. + ok((await pq.enqueue(makeRecord(10))).enqueued); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10, 10, 10]), + payloadBytes: 30, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([10]), + payloadBytes: 10, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 40, + numRecords: 4, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when the batch bytes limit is exceeded. +add_task(async function test_max_total_bytes_batch() { + let config = { + max_post_bytes: 50, + max_post_records: 20, + max_total_bytes: 70, + max_total_records: 100, + max_record_payload_bytes: 50, + max_request_bytes: 500, + }; + + const time0 = 11111111; + const time1 = 22222222; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time0, "x-weave-timestamp": time0 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { + "x-last-modified": time1 + 200, + "x-weave-timestamp": time1 + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time0, responseGenerator()); + + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 40, batch: 40 + + // this will exceed our POST byte limit, so will be in the 2nd POST - but still in the first batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 60 + + // this will exceed our batch byte limit, so will be in a new batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 40, batch: 40 + // This will exceed POST byte limit, so will be in the 4th post, part of the 2nd batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 60 + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time0]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time0]], + }, + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time1]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 5678, + headers: [["x-if-unmodified-since", time1]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 5678, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time1 + 200); +}); + +// Test we split up the posts when we exceed the record limit when batch semantics +// are in place. +add_task(async function test_max_post_records_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 2, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 1000, + max_request_bytes: 1000, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + // will exceed record limit of 2, so will be in 2nd post. + ok((await pq.enqueue(makeRecord(20))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + numRecords: 2, + payloadBytes: 40, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([20]), + numRecords: 1, + payloadBytes: 20, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when the batch record limit is exceeded. +add_task(async function test_max_records_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 3, + max_total_bytes: 10000, + max_total_records: 5, + max_record_payload_bytes: 1000, + max_request_bytes: 10000, + }; + + const time0 = 11111111; + const time1 = 22222222; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time0, "x-weave-timestamp": time0 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { + "x-last-modified": time1 + 200, + "x-weave-timestamp": time1 + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time0, responseGenerator()); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + // 3 records + nbytes: requestBytesFor([20, 20, 20]), + payloadBytes: 60, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time0]], + }, + { + // 2 records -- end batch1 + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time0]], + }, + { + // 3 records + nbytes: requestBytesFor([20, 20, 20]), + payloadBytes: 60, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time1]], + }, + { + // 1 record -- end batch2 + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 5678, + headers: [["x-if-unmodified-since", time1]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 100, + numRecords: 5, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + { + posts: 2, + payloadBytes: 80, + numRecords: 4, + batch: 5678, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time1 + 200); +}); + +// Test we do the right thing when the limits are met but not exceeded. +add_task(async function test_packed_batch() { + let config = { + max_post_bytes: 41, + max_post_records: 4, + + max_total_bytes: 81, + max_total_records: 8, + + max_record_payload_bytes: 20 + makeRecord.nonPayloadOverhead + 1, + max_request_bytes: requestBytesFor([10, 10, 10, 10]) + 1, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10, 10, 10, 10]), + numRecords: 4, + payloadBytes: 40, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([10, 10, 10, 10]), + numRecords: 4, + payloadBytes: 40, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 80, + numRecords: 8, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Tests that check that a single record fails to enqueue for the provided config +async function test_enqueue_failure_case(failureLimit, config) { + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 100, + "x-weave-timestamp": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + // Check on empty postqueue + let result = await pq.enqueue(makeRecord(failureLimit + 1)); + ok(!result.enqueued); + notEqual(result.error, undefined); + + ok((await pq.enqueue(makeRecord(5))).enqueued); + + // check on nonempty postqueue + result = await pq.enqueue(makeRecord(failureLimit + 1)); + ok(!result.enqueued); + notEqual(result.error, undefined); + + // make sure that we keep working, skipping the bad record entirely + // (handling the error the queue reported is left up to caller) + ok((await pq.enqueue(makeRecord(5))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([5, 5]), + numRecords: 2, + payloadBytes: 10, + commit: true, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 2, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 100); +} + +add_task(async function test_max_post_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 50, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 500, + max_request_bytes: 500, + }); +}); + +add_task(async function test_max_request_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 500, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 500, + max_request_bytes: 50, + }); +}); + +add_task(async function test_max_record_payload_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 500, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 50, + max_request_bytes: 500, + }); +}); diff --git a/services/sync/tests/unit/test_prefs_engine.js b/services/sync/tests/unit/test_prefs_engine.js new file mode 100644 index 0000000000..77a4474cb4 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_engine.js @@ -0,0 +1,134 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { getPrefsGUIDForTest } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); +const PREFS_GUID = getPrefsGUIDForTest(); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine.wipeClient(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function test_modified_after_fail() { + let engine = Service.engineManager.get("prefs"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + // The homepage pref is synced by default. + _("Set homepage before first sync"); + Services.prefs.setStringPref("browser.startup.homepage", "about:welcome"); + + _("First sync; create collection and pref record on server"); + await sync_engine_and_validate_telem(engine, false); + + let collection = server.user("foo").collection("prefs"); + equal( + collection.cleartext(PREFS_GUID).value["browser.startup.homepage"], + "about:welcome", + "Should upload homepage in pref record" + ); + ok( + !engine._tracker.modified, + "Tracker shouldn't be modified after first sync" + ); + + // Our tracker only has a `modified` flag that's reset after a + // successful upload. Force it to remain set by failing the + // upload. + _("Second sync; flag tracker as modified and throw on upload"); + Services.prefs.setStringPref("browser.startup.homepage", "about:robots"); + engine._tracker.modified = true; + let oldPost = collection.post; + collection.post = () => { + throw new Error("Sync this!"); + }; + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex.success === false + ); + ok( + engine._tracker.modified, + "Tracker should remain modified after failed sync" + ); + + _("Third sync"); + collection.post = oldPost; + await sync_engine_and_validate_telem(engine, false); + equal( + collection.cleartext(PREFS_GUID).value["browser.startup.homepage"], + "about:robots", + "Should upload new homepage on third sync" + ); + ok( + !engine._tracker.modified, + "Tracker shouldn't be modified again after third sync" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_allow_arbitrary() { + let engine = Service.engineManager.get("prefs"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + _("Create collection and pref record on server"); + await sync_engine_and_validate_telem(engine, false); + + let collection = server.user("foo").collection("prefs"); + + _("Insert arbitrary pref into remote record"); + let cleartext1 = collection.cleartext(PREFS_GUID); + cleartext1.value.let_viruses_take_over = true; + collection.insert( + PREFS_GUID, + encryptPayload(cleartext1), + new_timestamp() + 5 + ); + + _("Sync again; client shouldn't allow pref"); + await sync_engine_and_validate_telem(engine, false); + ok( + !Services.prefs.getBoolPref("let_viruses_take_over", false), + "Shouldn't allow arbitrary remote prefs without control pref" + ); + + _("Sync with control pref set; client should set new pref"); + Services.prefs.setBoolPref( + "services.sync.prefs.sync.let_viruses_take_over_take_two", + true + ); + + let cleartext2 = collection.cleartext(PREFS_GUID); + cleartext2.value.let_viruses_take_over_take_two = true; + collection.insert( + PREFS_GUID, + encryptPayload(cleartext2), + new_timestamp() + 5 + ); + // Reset the last sync time so that the engine fetches the record again. + await engine.setLastSync(0); + await sync_engine_and_validate_telem(engine, false); + ok( + Services.prefs.getBoolPref("let_viruses_take_over_take_two"), + "Should set arbitrary remote pref with control pref" + ); + } finally { + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_prefs_store.js b/services/sync/tests/unit/test_prefs_store.js new file mode 100644 index 0000000000..53ee68fc95 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_store.js @@ -0,0 +1,391 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PromiseTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PromiseTestUtils.sys.mjs" +); +PromiseTestUtils.allowMatchingRejectionsGlobally( + /Unable to arm timer, the object has been finalized\./ +); +PromiseTestUtils.allowMatchingRejectionsGlobally( + /IOUtils\.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished/ +); + +const { PrefRec, getPrefsGUIDForTest } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); +const PREFS_GUID = getPrefsGUIDForTest(); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const DEFAULT_THEME_ID = "default-theme@mozilla.org"; +const COMPACT_THEME_ID = "firefox-compact-light@mozilla.org"; + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +add_task(async function run_test() { + _("Test fixtures."); + // Part of this test ensures the default theme, via the preference + // extensions.activeThemeID, is synced correctly - so we do a little + // addons initialization to allow this to work. + + // Enable application scopes to ensure the builtin theme is going to + // be installed as part of the the addon manager startup. + Services.prefs.setIntPref( + "extensions.enabledScopes", + AddonManager.SCOPE_APPLICATION + ); + await AddonTestUtils.promiseStartupManager(); + + // Install another built-in theme. + await AddonManager.installBuiltinAddon("resource://builtin-themes/light/"); + + const defaultThemeAddon = await AddonManager.getAddonByID(DEFAULT_THEME_ID); + ok(defaultThemeAddon, "Got an addon wrapper for the default theme"); + + const otherThemeAddon = await AddonManager.getAddonByID(COMPACT_THEME_ID); + ok(otherThemeAddon, "Got an addon wrapper for the compact theme"); + + await otherThemeAddon.enable(); + + // read our custom prefs file before doing anything. + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + try { + _("Expect the compact light theme to be active"); + Assert.strictEqual( + Services.prefs.getStringPref("extensions.activeThemeID"), + COMPACT_THEME_ID + ); + + _("The GUID corresponds to XUL App ID."); + let allIDs = await store.getAllIDs(); + let ids = Object.keys(allIDs); + Assert.equal(ids.length, 1); + Assert.equal(ids[0], PREFS_GUID); + Assert.ok(allIDs[PREFS_GUID]); + + Assert.ok(await store.itemExists(PREFS_GUID)); + Assert.equal(false, await store.itemExists("random-gibberish")); + + _("Unknown prefs record is created as deleted."); + let record = await store.createRecord("random-gibberish", "prefs"); + Assert.ok(record.deleted); + + _("Prefs record contains only prefs that should be synced."); + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual(record.value["testing.int"], 123); + Assert.strictEqual(record.value["testing.string"], "ohai"); + Assert.strictEqual(record.value["testing.bool"], true); + // non-existing prefs get null as the value + Assert.strictEqual(record.value["testing.nonexistent"], null); + // as do prefs that have a default value. + Assert.strictEqual(record.value["testing.default"], null); + Assert.strictEqual(record.value["testing.turned.off"], undefined); + Assert.strictEqual(record.value["testing.not.turned.on"], undefined); + + _("Prefs record contains the correct control prefs."); + // All control prefs which have the default value and where the pref + // itself is synced should appear, but with null as the value. + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.int"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.string"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.bool"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.dont.change"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.nonexistent"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.default"], + null + ); + + // but this control pref has a non-default value so that value is synced. + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.turned.off"], + false + ); + + _("Unsyncable prefs are treated correctly."); + // Prefs we consider unsyncable (since they are URLs that won't be stable on + // another firefox) shouldn't be included - neither the value nor the + // control pref should appear. + Assert.strictEqual(record.value["testing.unsynced.url"], undefined); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.unsynced.url"], + undefined + ); + // Other URLs with user prefs should be synced, though. + Assert.strictEqual( + record.value["testing.synced.url"], + "https://www.example.com" + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.synced.url"], + null + ); + + _("Update some prefs, including one that's to be reset/deleted."); + // This pref is not going to be reset or deleted as there's no "control pref" + // in either the incoming record or locally. + Services.prefs.setStringPref( + "testing.deleted-without-control-pref", + "I'm deleted-without-control-pref" + ); + // Another pref with only a local control pref. + Services.prefs.setStringPref( + "testing.deleted-with-local-control-pref", + "I'm deleted-with-local-control-pref" + ); + Services.prefs.setBoolPref( + "services.sync.prefs.sync.testing.deleted-with-local-control-pref", + true + ); + // And a pref without a local control pref but one that's incoming. + Services.prefs.setStringPref( + "testing.deleted-with-incoming-control-pref", + "I'm deleted-with-incoming-control-pref" + ); + record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "extensions.activeThemeID": DEFAULT_THEME_ID, + "testing.int": 42, + "testing.string": "im in ur prefs", + "testing.bool": false, + "testing.deleted-without-control-pref": null, + "testing.deleted-with-local-control-pref": null, + "testing.deleted-with-incoming-control-pref": null, + "services.sync.prefs.sync.testing.deleted-with-incoming-control-pref": true, + "testing.somepref": "im a new pref from other device", + "services.sync.prefs.sync.testing.somepref": true, + // Pretend some a stale remote client is overwriting it with a value + // we consider unsyncable. + "testing.synced.url": "blob:ebeb707a-502e-40c6-97a5-dd4bda901463", + // Make sure we can replace the unsynced URL with a valid URL. + "testing.unsynced.url": "https://www.example.com/2", + // Make sure our "master control pref" is ignored. + "services.sync.prefs.dangerously_allow_arbitrary": true, + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary": true, + }; + + const onceAddonEnabled = AddonTestUtils.promiseAddonEvent("onEnabled"); + + await store.update(record); + Assert.strictEqual(Services.prefs.getIntPref("testing.int"), 42); + Assert.strictEqual( + Services.prefs.getStringPref("testing.string"), + "im in ur prefs" + ); + Assert.strictEqual(Services.prefs.getBoolPref("testing.bool"), false); + Assert.strictEqual( + Services.prefs.getStringPref("testing.deleted-without-control-pref"), + "I'm deleted-without-control-pref" + ); + Assert.strictEqual( + Services.prefs.getPrefType("testing.deleted-with-local-control-pref"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.strictEqual( + Services.prefs.getStringPref( + "testing.deleted-with-incoming-control-pref" + ), + "I'm deleted-with-incoming-control-pref" + ); + Assert.strictEqual( + Services.prefs.getStringPref("testing.dont.change"), + "Please don't change me." + ); + Assert.strictEqual( + Services.prefs.getPrefType("testing.somepref"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.strictEqual( + Services.prefs.getStringPref("testing.synced.url"), + "https://www.example.com" + ); + Assert.strictEqual( + Services.prefs.getStringPref("testing.unsynced.url"), + "https://www.example.com/2" + ); + Assert.strictEqual( + Svc.PrefBranch.getPrefType("prefs.sync.testing.somepref"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.strictEqual( + Services.prefs.getBoolPref( + "services.sync.prefs.dangerously_allow_arbitrary" + ), + false + ); + Assert.strictEqual( + Services.prefs.getPrefType( + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary" + ), + Ci.nsIPrefBranch.PREF_INVALID + ); + + await onceAddonEnabled; + ok( + !defaultThemeAddon.userDisabled, + "the default theme should have been enabled" + ); + ok( + otherThemeAddon.userDisabled, + "the compact theme should have been disabled" + ); + + _("Only the current app's preferences are applied."); + record = new PrefRec("prefs", "some-fake-app"); + record.value = { + "testing.int": 98, + }; + await store.update(record); + Assert.equal(Services.prefs.getIntPref("testing.int"), 42); + } finally { + for (const pref of Services.prefs.getChildList("")) { + Services.prefs.clearUserPref(pref); + } + } +}); + +add_task(async function test_dangerously_allow() { + _("services.sync.prefs.dangerously_allow_arbitrary"); + // Bug 1538015 added a capability to "dangerously allow" arbitrary prefs. + // Bug 1854698 removed that capability but did keep the fact we never + // sync the pref which enabled the "dangerous" behaviour, just incase someone + // tries to sync it back to a profile which *does* support that pref. + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + try { + // an incoming record with our old "dangerous" pref. + let record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "services.sync.prefs.dangerously_allow_arbitrary": true, + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary": true, + }; + await store.update(record); + Assert.strictEqual( + Services.prefs.getBoolPref( + "services.sync.prefs.dangerously_allow_arbitrary" + ), + false + ); + Assert.strictEqual( + Services.prefs.getPrefType( + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary" + ), + Ci.nsIPrefBranch.PREF_INVALID + ); + } finally { + for (const pref of Services.prefs.getChildList("")) { + Services.prefs.clearUserPref(pref); + } + } +}); + +add_task(async function test_incoming_sets_seen() { + _("Test the sync-seen allow-list"); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + const defaultValue = "the value"; + Assert.equal(Services.prefs.getStringPref("testing.seen"), defaultValue); + + let record = await store.createRecord(PREFS_GUID, "prefs"); + // Haven't seen a non-default value before, so remains null. + Assert.strictEqual(record.value["testing.seen"], null); + + // pretend an incoming record with the default value - it might not be + // the default everywhere, so we treat it specially. + record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "testing.seen": defaultValue, + }; + await store.update(record); + // Our special control value should now be set. + Assert.strictEqual( + Services.prefs.getBoolPref("services.sync.prefs.sync-seen.testing.seen"), + true + ); + // It's still the default value, so the value is not considered changed + Assert.equal(Services.prefs.prefHasUserValue("testing.seen"), false); + + // But now that special control value is set, the record always contains the value. + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual(record.value["testing.seen"], defaultValue); +}); + +add_task(async function test_outgoing_when_changed() { + _("Test the 'seen' pref is set first sync of non-default value"); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + for (const pref of Services.prefs.getChildList("")) { + Services.prefs.clearUserPref(pref); + } + + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + const defaultValue = "the value"; + Assert.equal(Services.prefs.getStringPref("testing.seen"), defaultValue); + + let record = await store.createRecord(PREFS_GUID, "prefs"); + // Haven't seen a non-default value before, so remains null. + Assert.strictEqual(record.value["testing.seen"], null); + + // Change the value. + Services.prefs.setStringPref("testing.seen", "new value"); + record = await store.createRecord(PREFS_GUID, "prefs"); + // creating the record toggled that "seen" pref. + Assert.strictEqual( + Services.prefs.getBoolPref("services.sync.prefs.sync-seen.testing.seen"), + true + ); + Assert.strictEqual(Services.prefs.getStringPref("testing.seen"), "new value"); + + // Resetting the pref does not change that seen value. + Services.prefs.clearUserPref("testing.seen"); + Assert.strictEqual( + Services.prefs.getStringPref("testing.seen"), + defaultValue + ); + + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual( + Services.prefs.getBoolPref("services.sync.prefs.sync-seen.testing.seen"), + true + ); +}); diff --git a/services/sync/tests/unit/test_prefs_tracker.js b/services/sync/tests/unit/test_prefs_tracker.js new file mode 100644 index 0000000000..ecf3f18420 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_tracker.js @@ -0,0 +1,93 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + let engine = Service.engineManager.get("prefs"); + let tracker = engine._tracker; + + try { + _("tracker.modified corresponds to preference."); + // Assert preference is not defined. + Assert.equal( + Svc.PrefBranch.getPrefType("engine.prefs.modified"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.ok(!tracker.modified); + + tracker.modified = true; + Assert.equal(Svc.PrefBranch.getBoolPref("engine.prefs.modified"), true); + Assert.ok(tracker.modified); + + _("Engine's getChangedID() just returns the one GUID we have."); + let changedIDs = await engine.getChangedIDs(); + let ids = Object.keys(changedIDs); + Assert.equal(ids.length, 1); + Assert.equal(ids[0], CommonUtils.encodeBase64URL(Services.appinfo.ID)); + + Svc.PrefBranch.setBoolPref("engine.prefs.modified", false); + Assert.ok(!tracker.modified); + + _("No modified state, so no changed IDs."); + do_check_empty(await engine.getChangedIDs()); + + _("Initial score is 0"); + Assert.equal(tracker.score, 0); + + _("Test fixtures."); + Svc.PrefBranch.setBoolPref("prefs.sync.testing.int", true); + + _( + "Test fixtures haven't upped the tracker score yet because it hasn't started tracking yet." + ); + Assert.equal(tracker.score, 0); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + Services.prefs.setIntPref("testing.int", 23); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(tracker.modified, true); + + _("Clearing changed IDs reset modified status."); + await tracker.clearChangedIDs(); + Assert.equal(tracker.modified, false); + + _("Resetting a pref ups the score, too."); + Services.prefs.clearUserPref("testing.int"); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(tracker.modified, true); + await tracker.clearChangedIDs(); + + _("So does changing a pref sync pref."); + Svc.PrefBranch.setBoolPref("prefs.sync.testing.int", false); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, true); + await tracker.clearChangedIDs(); + + _( + "Now that the pref sync pref has been flipped, changes to it won't be picked up." + ); + Services.prefs.setIntPref("testing.int", 42); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, false); + await tracker.clearChangedIDs(); + + _("Changing some other random pref won't do anything."); + Services.prefs.setStringPref("testing.other", "blergh"); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, false); + } finally { + await tracker.stop(); + for (const pref of Services.prefs.getChildList("")) { + Services.prefs.clearUserPref(pref); + } + } +}); diff --git a/services/sync/tests/unit/test_records_crypto.js b/services/sync/tests/unit/test_records_crypto.js new file mode 100644 index 0000000000..8b841653cd --- /dev/null +++ b/services/sync/tests/unit/test_records_crypto.js @@ -0,0 +1,189 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { CollectionKeyManager, CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +var cryptoWrap; + +function crypted_resource_handler(metadata, response) { + let obj = { + id: "resource", + modified: cryptoWrap.modified, + payload: JSON.stringify(cryptoWrap.payload), + }; + return httpd_basic_auth_handler(JSON.stringify(obj), metadata, response); +} + +function prepareCryptoWrap(collection, id) { + let w = new CryptoWrapper(); + w.cleartext.stuff = "my payload here"; + w.collection = collection; + w.id = id; + return w; +} + +add_task(async function test_records_crypto() { + let server; + + await configureIdentity({ username: "john@example.com" }); + let keyBundle = Service.identity.syncKeyBundle; + + try { + let log = Log.repository.getLogger("Test"); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + log.info("Setting up server and authenticator"); + + server = httpd_setup({ "/steam/resource": crypted_resource_handler }); + + log.info("Creating a record"); + + cryptoWrap = prepareCryptoWrap("steam", "resource"); + + log.info("cryptoWrap: " + cryptoWrap.toString()); + + log.info("Encrypting a record"); + + await cryptoWrap.encrypt(keyBundle); + log.info("Ciphertext is " + cryptoWrap.ciphertext); + Assert.ok(cryptoWrap.ciphertext != null); + + let firstIV = cryptoWrap.IV; + + log.info("Decrypting the record"); + + let payload = await cryptoWrap.decrypt(keyBundle); + Assert.equal(payload.stuff, "my payload here"); + Assert.notEqual(payload, cryptoWrap.payload); // wrap.data.payload is the encrypted one + + log.info("Make sure multiple decrypts cause failures"); + let error = ""; + try { + payload = await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal(error.message, "No ciphertext: nothing to decrypt?"); + + log.info("Re-encrypting the record with alternate payload"); + + cryptoWrap.cleartext.stuff = "another payload"; + await cryptoWrap.encrypt(keyBundle); + let secondIV = cryptoWrap.IV; + payload = await cryptoWrap.decrypt(keyBundle); + Assert.equal(payload.stuff, "another payload"); + + log.info("Make sure multiple encrypts use different IVs"); + Assert.notEqual(firstIV, secondIV); + + log.info(await "Make sure differing ids cause failures"); + await cryptoWrap.encrypt(keyBundle); + cryptoWrap.data.id = "other"; + error = ""; + try { + await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal(error.message, "Record id mismatch: resource != other"); + + log.info("Make sure wrong hmacs cause failures"); + await cryptoWrap.encrypt(keyBundle); + cryptoWrap.hmac = "foo"; + error = ""; + try { + await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal( + error.message.substr(0, 42), + "Record SHA256 HMAC mismatch: should be foo" + ); + + // Checking per-collection keys and default key handling. + + await generateNewKeys(Service.collectionKeys); + let bookmarkItem = prepareCryptoWrap("bookmarks", "foo"); + await bookmarkItem.encrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ); + log.info("Ciphertext is " + bookmarkItem.ciphertext); + Assert.ok(bookmarkItem.ciphertext != null); + log.info("Decrypting the record explicitly with the default key."); + Assert.equal( + (await bookmarkItem.decrypt(Service.collectionKeys._default)).stuff, + "my payload here" + ); + + // Per-collection keys. + // Generate a key for "bookmarks". + await generateNewKeys(Service.collectionKeys, ["bookmarks"]); + bookmarkItem = prepareCryptoWrap("bookmarks", "foo"); + Assert.equal(bookmarkItem.collection, "bookmarks"); + + // Encrypt. This'll use the "bookmarks" encryption key, because we have a + // special key for it. The same key will need to be used for decryption. + await bookmarkItem.encrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ); + Assert.ok(bookmarkItem.ciphertext != null); + + // Attempt to use the default key, because this is a collision that could + // conceivably occur in the real world. Decryption will error, because + // it's not the bookmarks key. + let err; + try { + await bookmarkItem.decrypt(Service.collectionKeys._default); + } catch (ex) { + err = ex; + } + Assert.equal("Record SHA256 HMAC mismatch", err.message.substr(0, 27)); + + // Explicitly check that it's using the bookmarks key. + // This should succeed. + Assert.equal( + ( + await bookmarkItem.decrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ) + ).stuff, + "my payload here" + ); + + Assert.ok(Service.collectionKeys.hasKeysFor(["bookmarks"])); + + // Add a key for some new collection and verify that it isn't the + // default key. + Assert.ok(!Service.collectionKeys.hasKeysFor(["forms"])); + Assert.ok(!Service.collectionKeys.hasKeysFor(["bookmarks", "forms"])); + let oldFormsKey = Service.collectionKeys.keyForCollection("forms"); + Assert.equal(oldFormsKey, Service.collectionKeys._default); + let newKeys = await Service.collectionKeys.ensureKeysFor(["forms"]); + Assert.ok(newKeys.hasKeysFor(["forms"])); + Assert.ok(newKeys.hasKeysFor(["bookmarks", "forms"])); + let newFormsKey = newKeys.keyForCollection("forms"); + Assert.notEqual(newFormsKey, oldFormsKey); + + // Verify that this doesn't overwrite keys + let regetKeys = await newKeys.ensureKeysFor(["forms"]); + Assert.equal(regetKeys.keyForCollection("forms"), newFormsKey); + + const emptyKeys = new CollectionKeyManager(); + payload = { + default: Service.collectionKeys._default.keyPairB64, + collections: {}, + }; + // Verify that not passing `modified` doesn't throw + emptyKeys.setContents(payload, null); + + log.info("Done!"); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_records_wbo.js b/services/sync/tests/unit/test_records_wbo.js new file mode 100644 index 0000000000..61ba33d749 --- /dev/null +++ b/services/sync/tests/unit/test_records_wbo.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_test(function test_toJSON() { + _("Create a record, for now without a TTL."); + let wbo = new WBORecord("coll", "a_record"); + wbo.modified = 12345; + wbo.sortindex = 42; + wbo.payload = {}; + + _( + "Verify that the JSON representation contains the WBO properties, but not TTL." + ); + let json = JSON.parse(JSON.stringify(wbo)); + Assert.equal(json.modified, 12345); + Assert.equal(json.sortindex, 42); + Assert.equal(json.payload, "{}"); + Assert.equal(false, "ttl" in json); + + _("Set a TTL, make sure it's present in the JSON representation."); + wbo.ttl = 30 * 60; + json = JSON.parse(JSON.stringify(wbo)); + Assert.equal(json.ttl, 30 * 60); + run_next_test(); +}); + +add_task(async function test_fetch() { + let record = { + id: "asdf-1234-asdf-1234", + modified: 2454725.98283, + payload: JSON.stringify({ cheese: "roquefort" }), + }; + let record2 = { + id: "record2", + modified: 2454725.98284, + payload: JSON.stringify({ cheese: "gruyere" }), + }; + let coll = [ + { + id: "record2", + modified: 2454725.98284, + payload: JSON.stringify({ cheese: "gruyere" }), + }, + ]; + + _("Setting up server."); + let server = httpd_setup({ + "/record": httpd_handler(200, "OK", JSON.stringify(record)), + "/record2": httpd_handler(200, "OK", JSON.stringify(record2)), + "/coll": httpd_handler(200, "OK", JSON.stringify(coll)), + }); + + try { + _("Fetching a WBO record"); + let rec = new WBORecord("coll", "record"); + await rec.fetch(Service.resource(server.baseURI + "/record")); + Assert.equal(rec.id, "asdf-1234-asdf-1234"); // NOT "record"! + + Assert.equal(rec.modified, 2454725.98283); + Assert.equal(typeof rec.payload, "object"); + Assert.equal(rec.payload.cheese, "roquefort"); + + _("Fetching a WBO record using the record manager"); + let rec2 = await Service.recordManager.get(server.baseURI + "/record2"); + Assert.equal(rec2.id, "record2"); + Assert.equal(rec2.modified, 2454725.98284); + Assert.equal(typeof rec2.payload, "object"); + Assert.equal(rec2.payload.cheese, "gruyere"); + Assert.equal(Service.recordManager.response.status, 200); + + // Testing collection extraction. + _("Extracting collection."); + let rec3 = new WBORecord("tabs", "foo"); // Create through constructor. + Assert.equal(rec3.collection, "tabs"); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_resource.js b/services/sync/tests/unit/test_resource.js new file mode 100644 index 0000000000..5182784639 --- /dev/null +++ b/services/sync/tests/unit/test_resource.js @@ -0,0 +1,554 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); + +var fetched = false; +function server_open(metadata, response) { + let body; + if (metadata.method == "GET") { + fetched = true; + body = "This path exists"; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Wrong request method"; + response.setStatusLine(metadata.httpVersion, 405, "Method Not Allowed"); + } + response.bodyOutputStream.write(body, body.length); +} + +function server_protected(metadata, response) { + let body; + + if (has_hawk_header(metadata)) { + body = "This path exists and is protected"; + response.setStatusLine(metadata.httpVersion, 200, "OK, authorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } else { + body = "This path exists and is protected - failed"; + response.setStatusLine(metadata.httpVersion, 401, "Unauthorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } + + response.bodyOutputStream.write(body, body.length); +} + +function server_404(metadata, response) { + let body = "File not found"; + response.setStatusLine(metadata.httpVersion, 404, "Not Found"); + response.bodyOutputStream.write(body, body.length); +} + +var pacFetched = false; +function server_pac(metadata, response) { + _("Invoked PAC handler."); + pacFetched = true; + let body = 'function FindProxyForURL(url, host) { return "DIRECT"; }'; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.setHeader( + "Content-Type", + "application/x-ns-proxy-autoconfig", + false + ); + response.bodyOutputStream.write(body, body.length); +} + +var sample_data = { + some: "sample_data", + injson: "format", + number: 42, +}; + +function server_upload(metadata, response) { + let body; + + let input = readBytesFromInputStream(metadata.bodyInputStream); + if (input == JSON.stringify(sample_data)) { + body = "Valid data upload via " + metadata.method; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Invalid data upload via " + metadata.method + ": " + input; + response.setStatusLine(metadata.httpVersion, 500, "Internal Server Error"); + } + + response.bodyOutputStream.write(body, body.length); +} + +function server_delete(metadata, response) { + let body; + if (metadata.method == "DELETE") { + body = "This resource has been deleted"; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Wrong request method"; + response.setStatusLine(metadata.httpVersion, 405, "Method Not Allowed"); + } + response.bodyOutputStream.write(body, body.length); +} + +function server_json(metadata, response) { + let body = JSON.stringify(sample_data); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +const TIMESTAMP = 1274380461; + +function server_timestamp(metadata, response) { + let body = "Thank you for your request"; + response.setHeader("X-Weave-Timestamp", "" + TIMESTAMP, false); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_backoff(metadata, response) { + let body = "Hey, back off!"; + response.setHeader("X-Weave-Backoff", "600", false); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_quota_notice(request, response) { + let body = "You're approaching quota."; + response.setHeader("X-Weave-Quota-Remaining", "1048576", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_quota_error(request, response) { + let body = "14"; + response.setHeader("X-Weave-Quota-Remaining", "-1024", false); + response.setStatusLine(request.httpVersion, 400, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_headers(metadata, response) { + let ignore_headers = [ + "host", + "user-agent", + "accept-language", + "accept-encoding", + "accept-charset", + "keep-alive", + "connection", + "pragma", + "origin", + "cache-control", + "content-length", + ]; + let headers = metadata.headers; + let header_names = []; + while (headers.hasMoreElements()) { + let header = headers.getNext().toString(); + if (!ignore_headers.includes(header)) { + header_names.push(header); + } + } + header_names = header_names.sort(); + + headers = {}; + for (let header of header_names) { + headers[header] = metadata.getHeader(header); + } + let body = JSON.stringify(headers); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +var quotaValue; +Observers.add("weave:service:quota:remaining", function (subject) { + quotaValue = subject; +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + Svc.PrefBranch.setIntPref("network.numRetries", 1); // speed up test + run_next_test(); +} + +// This apparently has to come first in order for our PAC URL to be hit. +// Don't put any other HTTP requests earlier in the file! +add_task(async function test_proxy_auth_redirect() { + _( + "Ensure that a proxy auth redirect (which switches out our channel) " + + "doesn't break Resource." + ); + let server = httpd_setup({ + "/open": server_open, + "/pac2": server_pac, + }); + + PACSystemSettings.PACURI = server.baseURI + "/pac2"; + installFakePAC(); + let res = new Resource(server.baseURI + "/open"); + let result = await res.get(); + Assert.ok(pacFetched); + Assert.ok(fetched); + Assert.equal("This path exists", result.data); + pacFetched = fetched = false; + uninstallFakePAC(); + await promiseStopServer(server); +}); + +add_task(async function test_new_channel() { + _("Ensure a redirect to a new channel is handled properly."); + + let resourceRequested = false; + function resourceHandler(metadata, response) { + resourceRequested = true; + + let body = "Test"; + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(body, body.length); + } + + let locationURL; + function redirectHandler(metadata, response) { + let body = "Redirecting"; + response.setStatusLine(metadata.httpVersion, 307, "TEMPORARY REDIRECT"); + response.setHeader("Location", locationURL); + response.bodyOutputStream.write(body, body.length); + } + + let server = httpd_setup({ + "/resource": resourceHandler, + "/redirect": redirectHandler, + }); + locationURL = server.baseURI + "/resource"; + + let request = new Resource(server.baseURI + "/redirect"); + let content = await request.get(); + Assert.ok(resourceRequested); + Assert.equal(200, content.status); + Assert.ok("content-type" in content.headers); + Assert.equal("text/plain", content.headers["content-type"]); + + await promiseStopServer(server); +}); + +var server; + +add_test(function setup() { + server = httpd_setup({ + "/open": server_open, + "/protected": server_protected, + "/404": server_404, + "/upload": server_upload, + "/delete": server_delete, + "/json": server_json, + "/timestamp": server_timestamp, + "/headers": server_headers, + "/backoff": server_backoff, + "/pac2": server_pac, + "/quota-notice": server_quota_notice, + "/quota-error": server_quota_error, + }); + + run_next_test(); +}); + +add_test(function test_members() { + _("Resource object members"); + let uri = server.baseURI + "/open"; + let res = new Resource(uri); + Assert.ok(res.uri instanceof Ci.nsIURI); + Assert.equal(res.uri.spec, uri); + Assert.equal(res.spec, uri); + Assert.equal(typeof res.headers, "object"); + Assert.equal(typeof res.authenticator, "object"); + + run_next_test(); +}); + +add_task(async function test_get() { + _("GET a non-password-protected resource"); + let res = new Resource(server.baseURI + "/open"); + let content = await res.get(); + Assert.equal(content.data, "This path exists"); + Assert.equal(content.status, 200); + Assert.ok(content.success); + + // Observe logging messages. + let resLogger = res._log; + let dbg = resLogger.debug; + let debugMessages = []; + resLogger.debug = function (msg, extra) { + debugMessages.push(`${msg}: ${JSON.stringify(extra)}`); + dbg.call(this, msg); + }; + + // Since we didn't receive proper JSON data, accessing content.obj + // will result in a SyntaxError from JSON.parse + let didThrow = false; + try { + content.obj; + } catch (ex) { + didThrow = true; + } + Assert.ok(didThrow); + Assert.equal(debugMessages.length, 1); + Assert.equal( + debugMessages[0], + 'Parse fail: Response body starts: "This path exists"' + ); + resLogger.debug = dbg; +}); + +add_test(function test_basicauth() { + _("Test that the BasicAuthenticator doesn't screw up header case."); + let res1 = new Resource(server.baseURI + "/foo"); + res1.setHeader("Authorization", "Basic foobar"); + Assert.equal(res1._headers.authorization, "Basic foobar"); + Assert.equal(res1.headers.authorization, "Basic foobar"); + + run_next_test(); +}); + +add_task(async function test_get_protected_fail() { + _( + "GET a password protected resource (test that it'll fail w/o pass, no throw)" + ); + let res2 = new Resource(server.baseURI + "/protected"); + let content = await res2.get(); + Assert.equal(content.data, "This path exists and is protected - failed"); + Assert.equal(content.status, 401); + Assert.ok(!content.success); +}); + +add_task(async function test_get_protected_success() { + _("GET a password protected resource"); + let identityConfig = makeIdentityConfig(); + let syncAuthManager = new SyncAuthManager(); + configureFxAccountIdentity(syncAuthManager, identityConfig); + let auth = syncAuthManager.getResourceAuthenticator(); + let res3 = new Resource(server.baseURI + "/protected"); + res3.authenticator = auth; + Assert.equal(res3.authenticator, auth); + let content = await res3.get(); + Assert.equal(content.data, "This path exists and is protected"); + Assert.equal(content.status, 200); + Assert.ok(content.success); +}); + +add_task(async function test_get_404() { + _("GET a non-existent resource (test that it'll fail, but not throw)"); + let res4 = new Resource(server.baseURI + "/404"); + let content = await res4.get(); + Assert.equal(content.data, "File not found"); + Assert.equal(content.status, 404); + Assert.ok(!content.success); + + // Check some headers of the 404 response + Assert.equal(content.headers.connection, "close"); + Assert.equal(content.headers.server, "httpd.js"); + Assert.equal(content.headers["content-length"], 14); +}); + +add_task(async function test_put_string() { + _("PUT to a resource (string)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.put(JSON.stringify(sample_data)); + Assert.equal(content.data, "Valid data upload via PUT"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_put_object() { + _("PUT to a resource (object)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.put(sample_data); + Assert.equal(content.data, "Valid data upload via PUT"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_post_string() { + _("POST to a resource (string)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.post(JSON.stringify(sample_data)); + Assert.equal(content.data, "Valid data upload via POST"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_post_object() { + _("POST to a resource (object)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.post(sample_data); + Assert.equal(content.data, "Valid data upload via POST"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_delete() { + _("DELETE a resource"); + let res6 = new Resource(server.baseURI + "/delete"); + let content = await res6.delete(); + Assert.equal(content.data, "This resource has been deleted"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_json_body() { + _("JSON conversion of response body"); + let res7 = new Resource(server.baseURI + "/json"); + let content = await res7.get(); + Assert.equal(content.data, JSON.stringify(sample_data)); + Assert.equal(content.status, 200); + Assert.equal(JSON.stringify(content.obj), JSON.stringify(sample_data)); +}); + +add_task(async function test_weave_timestamp() { + _("X-Weave-Timestamp header updates Resource.serverTime"); + // Before having received any response containing the + // X-Weave-Timestamp header, Resource.serverTime is null. + Assert.equal(Resource.serverTime, null); + let res8 = new Resource(server.baseURI + "/timestamp"); + await res8.get(); + Assert.equal(Resource.serverTime, TIMESTAMP); +}); + +add_task(async function test_get_default_headers() { + _("GET: Accept defaults to application/json"); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); +}); + +add_task(async function test_put_default_headers() { + _( + "PUT: Accept defaults to application/json, Content-Type defaults to text/plain" + ); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.put("data")).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); + Assert.equal(content["content-type"], "text/plain"); +}); + +add_task(async function test_post_default_headers() { + _( + "POST: Accept defaults to application/json, Content-Type defaults to text/plain" + ); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.post("data")).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); + Assert.equal(content["content-type"], "text/plain"); +}); + +add_task(async function test_setHeader() { + _("setHeader(): setting simple header"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("X-What-Is-Weave", "awesome"); + Assert.equal(res_headers.headers["x-what-is-weave"], "awesome"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content["x-what-is-weave"], "awesome"); +}); + +add_task(async function test_setHeader_overwrite() { + _("setHeader(): setting multiple headers, overwriting existing header"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("X-WHAT-is-Weave", "more awesomer"); + res_headers.setHeader("X-Another-Header", "hello world"); + Assert.equal(res_headers.headers["x-what-is-weave"], "more awesomer"); + Assert.equal(res_headers.headers["x-another-header"], "hello world"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content["x-what-is-weave"], "more awesomer"); + Assert.equal(content["x-another-header"], "hello world"); +}); + +add_task(async function test_put_override_content_type() { + _("PUT: override default Content-Type"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("Content-Type", "application/foobar"); + Assert.equal(res_headers.headers["content-type"], "application/foobar"); + let content = JSON.parse((await res_headers.put("data")).data); + Assert.equal(content["content-type"], "application/foobar"); +}); + +add_task(async function test_post_override_content_type() { + _("POST: override default Content-Type"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("Content-Type", "application/foobar"); + let content = JSON.parse((await res_headers.post("data")).data); + Assert.equal(content["content-type"], "application/foobar"); +}); + +add_task(async function test_weave_backoff() { + _("X-Weave-Backoff header notifies observer"); + let backoffInterval; + function onBackoff(subject, data) { + backoffInterval = subject; + } + Observers.add("weave:service:backoff:interval", onBackoff); + + let res10 = new Resource(server.baseURI + "/backoff"); + await res10.get(); + Assert.equal(backoffInterval, 600); +}); + +add_task(async function test_quota_error() { + _("X-Weave-Quota-Remaining header notifies observer on successful requests."); + let res10 = new Resource(server.baseURI + "/quota-error"); + let content = await res10.get(); + Assert.equal(content.status, 400); + Assert.equal(quotaValue, undefined); // HTTP 400, so no observer notification. +}); + +add_task(async function test_quota_notice() { + let res10 = new Resource(server.baseURI + "/quota-notice"); + let content = await res10.get(); + Assert.equal(content.status, 200); + Assert.equal(quotaValue, 1048576); +}); + +add_task(async function test_preserve_exceptions() { + _("Error handling preserves exception information"); + let res11 = new Resource("http://localhost:12345/does/not/exist"); + await Assert.rejects(res11.get(), error => { + Assert.notEqual(error, null); + Assert.equal(error.result, Cr.NS_ERROR_CONNECTION_REFUSED); + Assert.equal(error.name, "NS_ERROR_CONNECTION_REFUSED"); + return true; + }); +}); + +add_task(async function test_timeout() { + _("Ensure channel timeouts are thrown appropriately."); + let res19 = new Resource(server.baseURI + "/json"); + res19.ABORT_TIMEOUT = 0; + await Assert.rejects(res19.get(), error => { + Assert.equal(error.result, Cr.NS_ERROR_NET_TIMEOUT); + return true; + }); +}); + +add_test(function test_uri_construction() { + _("Testing URI construction."); + let args = []; + args.push("newer=" + 1234); + args.push("limit=" + 1234); + args.push("sort=" + 1234); + + let query = "?" + args.join("&"); + + let uri1 = CommonUtils.makeURI("http://foo/" + query).QueryInterface( + Ci.nsIURL + ); + let uri2 = CommonUtils.makeURI("http://foo/").QueryInterface(Ci.nsIURL); + uri2 = uri2.mutate().setQuery(query).finalize().QueryInterface(Ci.nsIURL); + Assert.equal(uri1.query, uri2.query); + + run_next_test(); +}); + +/** + * End of tests that rely on a single HTTP server. + * All tests after this point must begin and end their own. + */ +add_test(function eliminate_server() { + server.stop(run_next_test); +}); diff --git a/services/sync/tests/unit/test_resource_header.js b/services/sync/tests/unit/test_resource_header.js new file mode 100644 index 0000000000..e45b4a9864 --- /dev/null +++ b/services/sync/tests/unit/test_resource_header.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); + +var httpServer = new HttpServer(); +httpServer.registerPathHandler("/content", contentHandler); +httpServer.start(-1); + +const HTTP_PORT = httpServer.identity.primaryPort; +const TEST_URL = "http://localhost:" + HTTP_PORT + "/content"; +const BODY = "response body"; + +// Keep headers for later inspection. +var auth = null; +var foo = null; +function contentHandler(metadata, response) { + _("Handling request."); + auth = metadata.getHeader("Authorization"); + foo = metadata.getHeader("X-Foo"); + + _("Extracted headers. " + auth + ", " + foo); + + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(BODY, BODY.length); +} + +// Set a proxy function to cause an internal redirect. +function triggerRedirect() { + const PROXY_FUNCTION = + "function FindProxyForURL(url, host) {" + + " return 'PROXY a_non_existent_domain_x7x6c572v:80; " + + "PROXY localhost:" + + HTTP_PORT + + "';" + + "}"; + + let prefs = Services.prefs.getBranch("network.proxy."); + prefs.setIntPref("type", 2); + prefs.setStringPref("autoconfig_url", "data:text/plain," + PROXY_FUNCTION); +} + +add_task(async function test_headers_copied() { + triggerRedirect(); + + _("Issuing request."); + let resource = new Resource(TEST_URL); + resource.setHeader("Authorization", "Basic foobar"); + resource.setHeader("X-Foo", "foofoo"); + + let result = await resource.get(TEST_URL); + _("Result: " + result.data); + + Assert.equal(result.data, BODY); + Assert.equal(auth, "Basic foobar"); + Assert.equal(foo, "foofoo"); + + await promiseStopServer(httpServer); +}); diff --git a/services/sync/tests/unit/test_resource_ua.js b/services/sync/tests/unit/test_resource_ua.js new file mode 100644 index 0000000000..115fa85b84 --- /dev/null +++ b/services/sync/tests/unit/test_resource_ua.js @@ -0,0 +1,96 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +var httpProtocolHandler = Cc[ + "@mozilla.org/network/protocol;1?name=http" +].getService(Ci.nsIHttpProtocolHandler); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); + +var meta_global; +var server; + +var expectedUA; +var ua; +function uaHandler(f) { + return function (request, response) { + ua = request.getHeader("User-Agent"); + return f(request, response); + }; +} + +add_task(async function setup() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + meta_global = new ServerWBO("global"); + server = httpd_setup({ + "/1.1/johndoe/info/collections": uaHandler(collectionsHelper.handler), + "/1.1/johndoe/storage/meta/global": uaHandler(meta_global.handler()), + }); + + await configureIdentity({ username: "johndoe" }, server); + _("Server URL: " + server.baseURI); + + // Note this string is missing the trailing ".destkop" as the test + // adjusts the "client.type" pref where that portion comes from. + expectedUA = + Services.appinfo.name + + "/" + + Services.appinfo.version + + " (" + + httpProtocolHandler.oscpu + + ")" + + " FxSync/" + + WEAVE_VERSION + + "." + + Services.appinfo.appBuildID; +}); + +add_task(async function test_fetchInfo() { + _("Testing _fetchInfo."); + await Service.login(); + await Service._fetchInfo(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_desktop_post() { + _("Testing direct Resource POST."); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.post("foo=bar"); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_desktop_get() { + _("Testing async."); + Svc.PrefBranch.setStringPref("client.type", "desktop"); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.get(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_mobile_get() { + _("Testing mobile."); + Svc.PrefBranch.setStringPref("client.type", "mobile"); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.get(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".mobile"); + ua = ""; +}); + +add_test(function tear_down() { + server.stop(run_next_test); +}); diff --git a/services/sync/tests/unit/test_score_triggers.js b/services/sync/tests/unit/test_score_triggers.js new file mode 100644 index 0000000000..c6afa06407 --- /dev/null +++ b/services/sync/tests/unit/test_score_triggers.js @@ -0,0 +1,151 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup() { + let handlers = {}; + + handlers["/1.1/johndoe/storage/meta/global"] = new ServerWBO( + "global", + {} + ).handler(); + handlers["/1.1/johndoe/storage/steam"] = new ServerWBO("steam", {}).handler(); + + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp(server) { + let engineInfo = await registerRotaryEngine(); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + return engineInfo; +} + +add_task(async function test_tracker_score_updated() { + enableValidationPrefs(); + let { engine, tracker } = await registerRotaryEngine(); + + let scoreUpdated = 0; + + function onScoreUpdated() { + scoreUpdated++; + } + + Svc.Obs.add("weave:engine:score:updated", onScoreUpdated); + + try { + Assert.equal(engine.score, 0); + + tracker.score += SCORE_INCREMENT_SMALL; + Assert.equal(engine.score, SCORE_INCREMENT_SMALL); + + Assert.equal(scoreUpdated, 1); + } finally { + Svc.Obs.remove("weave:engine:score:updated", onScoreUpdated); + tracker.resetScore(); + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_sync_triggered() { + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + + await Service.login(); + + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + + Assert.equal(Status.login, LOGIN_SUCCEEDED); + tracker.score += SCORE_INCREMENT_XLARGE; + + await promiseOneObserver("weave:service:sync:finish"); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +add_task(async function test_clients_engine_sync_triggered() { + enableValidationPrefs(); + + _("Ensure that client engine score changes trigger a sync."); + + // The clients engine is not registered like other engines. Therefore, + // it needs special treatment throughout the code. Here, we verify the + // global score tracker gives it that treatment. See bug 676042 for more. + + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + await Service.login(); + + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + Assert.equal(Status.login, LOGIN_SUCCEEDED); + Service.clientsEngine._tracker.score += SCORE_INCREMENT_XLARGE; + + await promiseOneObserver("weave:service:sync:finish"); + _("Sync due to clients engine change completed."); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +add_task(async function test_incorrect_credentials_sync_not_triggered() { + enableValidationPrefs(); + + _( + "Ensure that score changes don't trigger a sync if Status.login != LOGIN_SUCCEEDED." + ); + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + + // Ensure we don't actually try to sync. + function onSyncStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:sync:start", onSyncStart); + + // Faking incorrect credentials to prevent score update. + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + tracker.score += SCORE_INCREMENT_XLARGE; + + // First wait >100ms (nsITimers can take up to that much time to fire, so + // we can account for the timer in delayedAutoconnect) and then one event + // loop tick (to account for a possible call to weave:service:sync:start). + await promiseNamedTimer(150, {}, "timer"); + await Async.promiseYield(); + + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); diff --git a/services/sync/tests/unit/test_service_attributes.js b/services/sync/tests/unit/test_service_attributes.js new file mode 100644 index 0000000000..86ca3c30c0 --- /dev/null +++ b/services/sync/tests/unit/test_service_attributes.js @@ -0,0 +1,92 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { FakeGUIDService } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/fakeservices.sys.mjs" +); + +add_task(async function test_urls() { + _("URL related Service properties correspond to preference settings."); + try { + Assert.equal(Service.clusterURL, ""); + Assert.ok(!Service.userBaseURL); + Assert.equal(Service.infoURL, undefined); + Assert.equal(Service.storageURL, undefined); + Assert.equal(Service.metaURL, undefined); + + _("The 'clusterURL' attribute updates preferences and cached URLs."); + + // Since we don't have a cluster URL yet, these will still not be defined. + Assert.equal(Service.infoURL, undefined); + Assert.ok(!Service.userBaseURL); + Assert.equal(Service.storageURL, undefined); + Assert.equal(Service.metaURL, undefined); + + Service.clusterURL = "http://weave.cluster/1.1/johndoe/"; + + Assert.equal(Service.userBaseURL, "http://weave.cluster/1.1/johndoe/"); + Assert.equal( + Service.infoURL, + "http://weave.cluster/1.1/johndoe/info/collections" + ); + Assert.equal( + Service.storageURL, + "http://weave.cluster/1.1/johndoe/storage/" + ); + Assert.equal( + Service.metaURL, + "http://weave.cluster/1.1/johndoe/storage/meta/global" + ); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_test(function test_syncID() { + _("Service.syncID is auto-generated, corresponds to preference."); + new FakeGUIDService(); + + try { + // Ensure pristine environment + Assert.equal( + Svc.PrefBranch.getPrefType("client.syncID"), + Ci.nsIPrefBranch.PREF_INVALID + ); + + // Performing the first get on the attribute will generate a new GUID. + Assert.equal(Service.syncID, "fake-guid-00"); + Assert.equal(Svc.PrefBranch.getStringPref("client.syncID"), "fake-guid-00"); + + Svc.PrefBranch.setStringPref("client.syncID", Utils.makeGUID()); + Assert.equal(Svc.PrefBranch.getStringPref("client.syncID"), "fake-guid-01"); + Assert.equal(Service.syncID, "fake-guid-01"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + new FakeGUIDService(); + run_next_test(); + } +}); + +add_test(function test_locked() { + _("The 'locked' attribute can be toggled with lock() and unlock()"); + + // Defaults to false + Assert.equal(Service.locked, false); + + Assert.equal(Service.lock(), true); + Assert.equal(Service.locked, true); + + // Locking again will return false + Assert.equal(Service.lock(), false); + + Service.unlock(); + Assert.equal(Service.locked, false); + run_next_test(); +}); diff --git a/services/sync/tests/unit/test_service_cluster.js b/services/sync/tests/unit/test_service_cluster.js new file mode 100644 index 0000000000..b4c14f910d --- /dev/null +++ b/services/sync/tests/unit/test_service_cluster.js @@ -0,0 +1,61 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_findCluster() { + syncTestLogging(); + _("Test Service._findCluster()"); + try { + let whenReadyToAuthenticate = Promise.withResolvers(); + Service.identity.whenReadyToAuthenticate = whenReadyToAuthenticate; + whenReadyToAuthenticate.resolve(true); + + Service.identity._ensureValidToken = () => + Promise.reject(new Error("Connection refused")); + + _("_findCluster() throws on network errors (e.g. connection refused)."); + await Assert.rejects(Service.identity._findCluster(), /Connection refused/); + + Service.identity._ensureValidToken = () => + Promise.resolve({ endpoint: "http://weave.user.node" }); + + _("_findCluster() returns the user's cluster node"); + let cluster = await Service.identity._findCluster(); + Assert.equal(cluster, "http://weave.user.node/"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_setCluster() { + syncTestLogging(); + _("Test Service._setCluster()"); + try { + _("Check initial state."); + Assert.equal(Service.clusterURL, ""); + + Service.identity._findCluster = () => "http://weave.user.node/"; + + _("Set the cluster URL."); + Assert.ok(await Service.identity.setCluster()); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + + _("Setting it again won't make a difference if it's the same one."); + Assert.ok(!(await Service.identity.setCluster())); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + + _("A 'null' response won't make a difference either."); + Service.identity._findCluster = () => null; + Assert.ok(!(await Service.identity.setCluster())); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); diff --git a/services/sync/tests/unit/test_service_detect_upgrade.js b/services/sync/tests/unit/test_service_detect_upgrade.js new file mode 100644 index 0000000000..d0db19af93 --- /dev/null +++ b/services/sync/tests/unit/test_service_detect_upgrade.js @@ -0,0 +1,274 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { CryptoWrapper, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function v4_upgrade() { + enableValidationPrefs(); + + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let collections = collectionsHelper.collections; + + let keysWBO = new ServerWBO("keys"); + let server = httpd_setup({ + // Special. + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/meta/global": upd("meta", meta_global.handler()), + + // Track modified times. + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/tabs": upd("tabs", new ServerCollection().handler()), + + // Just so we don't get 404s in the logs. + "/1.1/johndoe/storage/bookmarks": new ServerCollection().handler(), + "/1.1/johndoe/storage/forms": new ServerCollection().handler(), + "/1.1/johndoe/storage/history": new ServerCollection().handler(), + "/1.1/johndoe/storage/passwords": new ServerCollection().handler(), + "/1.1/johndoe/storage/prefs": new ServerCollection().handler(), + }); + + try { + Service.status.resetSync(); + + _("Logging in."); + + await configureIdentity({ username: "johndoe" }, server); + + await Service.login(); + Assert.ok(Service.isLoggedIn); + await Service.verifyAndFetchSymmetricKeys(); + Assert.ok(await Service._remoteSetup()); + + async function test_out_of_date() { + _("Old meta/global: " + JSON.stringify(meta_global)); + meta_global.payload = JSON.stringify({ + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION + 1, + }); + collections.meta = Date.now() / 1000; + _("New meta/global: " + JSON.stringify(meta_global)); + Service.recordManager.set(Service.metaURL, meta_global); + try { + await Service.sync(); + } catch (ex) {} + Assert.equal(Service.status.sync, VERSION_OUT_OF_DATE); + } + + // See what happens when we bump the storage version. + _("Syncing after server has been upgraded."); + await test_out_of_date(); + + // Same should happen after a wipe. + _("Syncing after server has been upgraded and wiped."); + await Service.wipeServer(); + await test_out_of_date(); + + // Now's a great time to test what happens when keys get replaced. + _("Syncing afresh..."); + Service.logout(); + Service.collectionKeys.clear(); + meta_global.payload = JSON.stringify({ + syncID: "foooooooooooooobbbbbbbbbbbb", + storageVersion: STORAGE_VERSION, + }); + collections.meta = Date.now() / 1000; + Service.recordManager.set(Service.metaURL, meta_global); + await Service.login(); + Assert.ok(Service.isLoggedIn); + await Service.sync(); + Assert.ok(Service.isLoggedIn); + + let serverDecrypted; + let serverKeys; + let serverResp; + + async function retrieve_server_default() { + serverKeys = serverResp = serverDecrypted = null; + + serverKeys = new CryptoWrapper("crypto", "keys"); + serverResp = ( + await serverKeys.fetch(Service.resource(Service.cryptoKeysURL)) + ).response; + Assert.ok(serverResp.success); + + serverDecrypted = await serverKeys.decrypt( + Service.identity.syncKeyBundle + ); + _("Retrieved WBO: " + JSON.stringify(serverDecrypted)); + _("serverKeys: " + JSON.stringify(serverKeys)); + + return serverDecrypted.default; + } + + async function retrieve_and_compare_default(should_succeed) { + let serverDefault = await retrieve_server_default(); + let localDefault = Service.collectionKeys.keyForCollection().keyPairB64; + + _("Retrieved keyBundle: " + JSON.stringify(serverDefault)); + _("Local keyBundle: " + JSON.stringify(localDefault)); + + if (should_succeed) { + Assert.equal( + JSON.stringify(serverDefault), + JSON.stringify(localDefault) + ); + } else { + Assert.notEqual( + JSON.stringify(serverDefault), + JSON.stringify(localDefault) + ); + } + } + + // Uses the objects set above. + async function set_server_keys(pair) { + serverDecrypted.default = pair; + serverKeys.cleartext = serverDecrypted; + await serverKeys.encrypt(Service.identity.syncKeyBundle); + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)); + } + + _("Checking we have the latest keys."); + await retrieve_and_compare_default(true); + + _("Update keys on server."); + await set_server_keys([ + "KaaaaaaaaaaaHAtfmuRY0XEJ7LXfFuqvF7opFdBD/MY=", + "aaaaaaaaaaaapxMO6TEWtLIOv9dj6kBAJdzhWDkkkis=", + ]); + + _("Checking that we no longer have the latest keys."); + await retrieve_and_compare_default(false); + + _("Indeed, they're what we set them to..."); + Assert.equal( + "KaaaaaaaaaaaHAtfmuRY0XEJ7LXfFuqvF7opFdBD/MY=", + (await retrieve_server_default())[0] + ); + + _("Sync. Should download changed keys automatically."); + let oldClientsModified = collections.clients; + let oldTabsModified = collections.tabs; + + await Service.login(); + await Service.sync(); + _("New key should have forced upload of data."); + _("Tabs: " + oldTabsModified + " < " + collections.tabs); + _("Clients: " + oldClientsModified + " < " + collections.clients); + Assert.ok(collections.clients > oldClientsModified); + Assert.ok(collections.tabs > oldTabsModified); + + _("... and keys will now match."); + await retrieve_and_compare_default(true); + + // Clean up. + await Service.startOver(); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); + +add_task(async function v5_upgrade() { + enableValidationPrefs(); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let keysWBO = new ServerWBO("keys"); + let bulkWBO = new ServerWBO("bulk"); + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + let server = httpd_setup({ + // Special. + "/1.1/johndoe/storage/meta/global": upd("meta", meta_global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/crypto/bulk": upd("crypto", bulkWBO.handler()), + + // Track modified times. + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/tabs": upd("tabs", new ServerCollection().handler()), + }); + + try { + Service.status.resetSync(); + + Service.clusterURL = server.baseURI + "/"; + + await configureIdentity({ username: "johndoe" }, server); + + // Test an upgrade where the contents of the server would cause us to error + // -- keys decrypted with a different sync key, for example. + _("Testing v4 -> v5 (or similar) upgrade."); + async function update_server_keys(syncKeyBundle, wboName, collWBO) { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", wboName); + await serverKeys.encrypt(syncKeyBundle); + let res = Service.resource(Service.storageURL + collWBO); + Assert.ok((await serverKeys.upload(res)).success); + } + + _("Bumping version."); + // Bump version on the server. + let m = new WBORecord("meta", "global"); + m.payload = { + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION + 1, + }; + await m.upload(Service.resource(Service.metaURL)); + + _("New meta/global: " + JSON.stringify(meta_global)); + + // Fill the keys with bad data. + let badKeys = new BulkKeyBundle("crypto"); + await badKeys.generateRandom(); + await update_server_keys(badKeys, "keys", "crypto/keys"); // v4 + await update_server_keys(badKeys, "bulk", "crypto/bulk"); // v5 + + _("Generating new keys."); + await generateNewKeys(Service.collectionKeys); + + // Now sync and see what happens. It should be a version fail, not a crypto + // fail. + + _("Logging in."); + try { + await Service.login(); + } catch (e) { + _("Exception: " + e); + } + _("Status: " + Service.status); + Assert.ok(!Service.isLoggedIn); + Assert.equal(VERSION_OUT_OF_DATE, Service.status.sync); + + // Clean up. + await Service.startOver(); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + run_next_test(); +} diff --git a/services/sync/tests/unit/test_service_login.js b/services/sync/tests/unit/test_service_login.js new file mode 100644 index 0000000000..c75799d38e --- /dev/null +++ b/services/sync/tests/unit/test_service_login.js @@ -0,0 +1,224 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + +function login_handling(handler) { + return function (request, response) { + if (has_hawk_header(request)) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +add_task(async function test_offline() { + try { + _("The right bits are set when we're offline."); + Services.io.offline = true; + Assert.ok(!(await Service.login())); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + Services.io.offline = false; + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +function setup() { + let janeHelper = track_collections_helper(); + let janeU = janeHelper.with_updated_collection; + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + + let server = httpd_setup({ + "/1.1/johndoe/info/collections": login_handling(johnHelper.handler), + "/1.1/janedoe/info/collections": login_handling(janeHelper.handler), + + // We need these handlers because we test login, and login + // is where keys are generated or fetched. + // TODO: have Jane fetch her keys, not generate them... + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + "/1.1/janedoe/storage/crypto/keys": janeU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/janedoe/storage/meta/global": janeU( + "meta", + new ServerWBO("global").handler() + ), + }); + + return server; +} + +add_task(async function test_not_logged_in() { + let server = setup(); + try { + await Service.login(); + Assert.ok(!Service.isLoggedIn, "no user configured, so can't be logged in"); + Assert.equal(Service._checkSync(), kSyncNotConfigured); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); + +add_task(async function test_login_logout() { + enableValidationPrefs(); + + let server = setup(); + + try { + _("Force the initial state."); + Service.status.service = STATUS_OK; + Assert.equal(Service.status.service, STATUS_OK); + + _("Try logging in. It won't work because we're not configured yet."); + await Service.login(); + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_USERNAME); + Assert.ok(!Service.isLoggedIn); + + _("Try again with a configured account"); + await configureIdentity({ username: "johndoe" }, server); + await Service.login(); + Assert.equal(Service.status.service, STATUS_OK); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + _("Logout."); + Service.logout(); + Assert.ok(!Service.isLoggedIn); + + _("Logging out again won't do any harm."); + Service.logout(); + Assert.ok(!Service.isLoggedIn); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); + +add_task(async function test_login_on_sync() { + enableValidationPrefs(); + + let server = setup(); + await configureIdentity({ username: "johndoe" }, server); + + try { + _("Sync calls login."); + let oldLogin = Service.login; + let loginCalled = false; + Service.login = async function () { + loginCalled = true; + Service.status.login = LOGIN_SUCCEEDED; + this._loggedIn = false; // So that sync aborts. + return true; + }; + + await Service.sync(); + + Assert.ok(loginCalled); + Service.login = oldLogin; + + // Stub mpLocked. + let mpLocked = true; + Utils.mpLocked = () => mpLocked; + + // Stub scheduleNextSync. This gets called within checkSyncStatus if we're + // ready to sync, so use it as an indicator. + let scheduleNextSyncF = Service.scheduler.scheduleNextSync; + let scheduleCalled = false; + Service.scheduler.scheduleNextSync = function (wait) { + scheduleCalled = true; + scheduleNextSyncF.call(this, wait); + }; + + // Autoconnect still tries to connect in the background (useful behavior: + // for non-MP users and unlocked MPs, this will detect version expiry + // earlier). + // + // Consequently, non-MP users will be logged in as in the pre-Bug 543784 world, + // and checkSyncStatus reflects that by waiting for login. + // + // This process doesn't apply if your MP is still locked, so we make + // checkSyncStatus accept a locked MP in place of being logged in. + // + // This test exercises these two branches. + + _("We're ready to sync if locked."); + Service.enabled = true; + Services.io.offline = false; + Service.scheduler.checkSyncStatus(); + Assert.ok(scheduleCalled); + + _("... and also if we're not locked."); + scheduleCalled = false; + mpLocked = false; + Service.scheduler.checkSyncStatus(); + Assert.ok(scheduleCalled); + Service.scheduler.scheduleNextSync = scheduleNextSyncF; + + // TODO: need better tests around master password prompting. See Bug 620583. + + mpLocked = true; + + // Testing exception handling if master password dialog is canceled. + // Do this by monkeypatching. + Service.identity.unlockAndVerifyAuthState = () => + Promise.resolve(MASTER_PASSWORD_LOCKED); + + let cSTCalled = false; + let lockedSyncCalled = false; + + Service.scheduler.clearSyncTriggers = function () { + cSTCalled = true; + }; + Service._lockedSync = async function () { + lockedSyncCalled = true; + }; + + _("If master password is canceled, login fails and we report lockage."); + Assert.ok(!(await Service.login())); + Assert.equal(Service.status.login, MASTER_PASSWORD_LOCKED); + Assert.equal(Service.status.service, LOGIN_FAILED); + _("Locked? " + Utils.mpLocked()); + _("checkSync reports the correct term."); + Assert.equal(Service._checkSync(), kSyncMasterPasswordLocked); + + _("Sync doesn't proceed and clears triggers if MP is still locked."); + await Service.sync(); + + Assert.ok(cSTCalled); + Assert.ok(!lockedSyncCalled); + + // N.B., a bunch of methods are stubbed at this point. Be careful putting + // new tests after this point! + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_startOver.js b/services/sync/tests/unit/test_service_startOver.js new file mode 100644 index 0000000000..22d92c76ef --- /dev/null +++ b/services/sync/tests/unit/test_service_startOver.js @@ -0,0 +1,91 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function BlaEngine() { + SyncEngine.call(this, "Bla", Service); +} +BlaEngine.prototype = { + removed: false, + async removeClientData() { + this.removed = true; + }, +}; +Object.setPrototypeOf(BlaEngine.prototype, SyncEngine.prototype); + +add_task(async function setup() { + await Service.engineManager.register(BlaEngine); +}); + +add_task(async function test_resetLocalData() { + await configureIdentity(); + Service.status.enforceBackoff = true; + Service.status.backoffInterval = 42; + Service.status.minimumNextSync = 23; + + // Verify set up. + Assert.equal(Service.status.checkSetup(), STATUS_OK); + + // Verify state that the observer sees. + let observerCalled = false; + Svc.Obs.add("weave:service:start-over", function onStartOver() { + Svc.Obs.remove("weave:service:start-over", onStartOver); + observerCalled = true; + + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + }); + + await Service.startOver(); + Assert.ok(observerCalled); + + // Verify the site was nuked from orbit. + Assert.equal( + Svc.PrefBranch.getPrefType("username"), + Ci.nsIPrefBranch.PREF_INVALID + ); + + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.ok(!Service.status.enforceBackoff); + Assert.equal(Service.status.backoffInterval, 0); + Assert.equal(Service.status.minimumNextSync, 0); +}); + +add_task(async function test_removeClientData() { + let engine = Service.engineManager.get("bla"); + + // No cluster URL = no removal. + Assert.ok(!engine.removed); + await Service.startOver(); + Assert.ok(!engine.removed); + + Service.clusterURL = "https://localhost/"; + + Assert.ok(!engine.removed); + await Service.startOver(); + Assert.ok(engine.removed); +}); + +add_task(async function test_reset_SyncScheduler() { + // Some non-default values for SyncScheduler's attributes. + Service.scheduler.idle = true; + Service.scheduler.hasIncomingItems = true; + Svc.PrefBranch.setIntPref("clients.devices.desktop", 42); + Service.scheduler.nextSync = Date.now(); + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + Service.scheduler.syncInterval = Service.scheduler.activeInterval; + + await Service.startOver(); + + Assert.ok(!Service.scheduler.idle); + Assert.ok(!Service.scheduler.hasIncomingItems); + Assert.equal(Service.scheduler.numClients, 0); + Assert.equal(Service.scheduler.nextSync, 0); + Assert.equal(Service.scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal( + Service.scheduler.syncInterval, + Service.scheduler.singleDeviceInterval + ); +}); diff --git a/services/sync/tests/unit/test_service_startup.js b/services/sync/tests/unit/test_service_startup.js new file mode 100644 index 0000000000..66623a951a --- /dev/null +++ b/services/sync/tests/unit/test_service_startup.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +// Svc.PrefBranch.setStringPref("services.sync.log.appender.dump", "All"); +Svc.PrefBranch.setStringPref("registerEngines", "Tab,Bookmarks,Form,History"); + +add_task(async function run_test() { + validate_all_future_pings(); + _("When imported, Service.onStartup is called"); + + let xps = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + Assert.ok(!xps.enabled); + + // Test fixtures + let { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" + ); + Services.prefs.setStringPref("services.sync.username", "johndoe"); + Assert.ok(xps.enabled); + + _("Service is enabled."); + Assert.equal(Service.enabled, true); + + _("Observers are notified of startup"); + Assert.ok(!Service.status.ready); + Assert.ok(!xps.ready); + + await promiseOneObserver("weave:service:ready"); + + Assert.ok(Service.status.ready); + Assert.ok(xps.ready); + + _("Engines are registered."); + let engines = Service.engineManager.getAll(); + if (AppConstants.MOZ_APP_NAME == "thunderbird") { + // Thunderbird's engines are registered later, so they're not here yet. + Assert.deepEqual( + engines.map(engine => engine.name), + [] + ); + } else { + Assert.deepEqual( + engines.map(engine => engine.name), + ["tabs", "bookmarks", "forms", "history"] + ); + } + + // Clean up. + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + + do_test_finished(); +}); diff --git a/services/sync/tests/unit/test_service_sync_401.js b/services/sync/tests/unit/test_service_sync_401.js new file mode 100644 index 0000000000..a0bde0b0ab --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_401.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function login_handling(handler) { + return function (request, response) { + if ( + request.hasHeader("Authorization") && + request.getHeader("Authorization").includes('Hawk id="id"') + ) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +add_task(async function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let server = httpd_setup({ + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + "/1.1/johndoe/info/collections": login_handling(collectionsHelper.handler), + }); + + const GLOBAL_SCORE = 42; + + try { + _("Set up test fixtures."); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + Service.scheduler.globalScore = GLOBAL_SCORE; + // Avoid daily ping + Svc.PrefBranch.setIntPref("lastPing", Math.floor(Date.now() / 1000)); + + let threw = false; + Svc.Obs.add("weave:service:sync:error", function (subject, data) { + threw = true; + }); + + _("Initial state: We're successfully logged in."); + await Service.login(); + Assert.ok(Service.isLoggedIn); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + + _("Simulate having changed the password somewhere else."); + Service.identity._token.id = "somethingelse"; + Service.identity.unlockAndVerifyAuthState = () => + Promise.resolve(LOGIN_FAILED_LOGIN_REJECTED); + + _("Let's try to sync."); + await Service.sync(); + + _("Verify that sync() threw an exception."); + Assert.ok(threw); + + _("We're no longer logged in."); + Assert.ok(!Service.isLoggedIn); + + _("Sync status won't have changed yet, because we haven't tried again."); + + _("globalScore is reset upon starting a sync."); + Assert.equal(Service.scheduler.globalScore, 0); + + _("Our next sync will fail appropriately."); + try { + await Service.sync(); + } catch (ex) {} + Assert.equal(Service.status.login, LOGIN_FAILED_LOGIN_REJECTED); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_locked.js b/services/sync/tests/unit/test_service_sync_locked.js new file mode 100644 index 0000000000..5a872e2708 --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_locked.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + validate_all_future_pings(); + let debug = []; + let info = []; + + function augmentLogger(old) { + let d = old.debug; + let i = old.info; + // For the purposes of this test we don't need to do full formatting + // of the 2nd param, as the ones we care about are always strings. + old.debug = function (m, p) { + debug.push(p ? m + ": " + (p.message || p) : m); + d.call(old, m, p); + }; + old.info = function (m, p) { + info.push(p ? m + ": " + (p.message || p) : m); + i.call(old, m, p); + }; + return old; + } + + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + augmentLogger(Service._log); + + // Avoid daily ping + Svc.PrefBranch.setIntPref("lastPing", Math.floor(Date.now() / 1000)); + + _("Check that sync will log appropriately if already in 'progress'."); + Service._locked = true; + await Service.sync(); + Service._locked = false; + + Assert.ok( + debug[debug.length - 2].startsWith( + 'Exception calling WrappedLock: Could not acquire lock. Label: "service.js: login".' + ) + ); + Assert.equal(info[info.length - 1], "Cannot start sync: already syncing?"); +}); diff --git a/services/sync/tests/unit/test_service_sync_remoteSetup.js b/services/sync/tests/unit/test_service_sync_remoteSetup.js new file mode 100644 index 0000000000..ec95e69c78 --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_remoteSetup.js @@ -0,0 +1,241 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// This sucks, but this test fails if this engine is enabled, due to dumb +// things that aren't related to this engine. In short: +// * Because the addon manager isn't initialized, the addons engine fails to +// initialize. So we end up writing a meta/global with `extension-storage` +// but not addons. +// * After we sync, we discover 'addons' is locally enabled, but because it's +// not in m/g, we decide it's been remotely declined (and it decides this +// without even considering `declined`). So we disable 'addons'. +// * Disabling 'addons' means 'extension-storage' is disabled - but because +// that *is* in meta/global we re-update meta/global to remove it. +// * This test fails due to the extra, unexpected update of m/g. +// +// Another option would be to ensure the addons manager is initialized, but +// that's a larger patch and still isn't strictly relevant to what's being +// tested here, so... +Services.prefs.setBoolPref( + "services.sync.engine.extension-storage.force", + false +); + +add_task(async function run_test() { + enableValidationPrefs(); + + validate_all_future_pings(); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let collections = collectionsHelper.collections; + + function wasCalledHandler(wbo) { + let handler = wbo.handler(); + return function () { + wbo.wasCalled = true; + handler.apply(this, arguments); + }; + } + + let keysWBO = new ServerWBO("keys"); + let cryptoColl = new ServerCollection({ keys: keysWBO }); + let metaColl = new ServerCollection({ global: meta_global }); + do_test_pending(); + + /** + * Handle the bulk DELETE request sent by wipeServer. + */ + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + + _("Wiping out all collections."); + cryptoColl.delete({}); + clients.delete({}); + metaColl.delete({}); + + let ts = new_timestamp(); + collectionsHelper.update_collection("crypto", ts); + collectionsHelper.update_collection("clients", ts); + collectionsHelper.update_collection("meta", ts); + return_timestamp(request, response, ts); + } + + const GLOBAL_PATH = "/1.1/johndoe/storage/meta/global"; + + let handlers = { + "/1.1/johndoe/storage": storageHandler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/crypto": upd("crypto", cryptoColl.handler()), + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/meta": upd("meta", wasCalledHandler(metaColl)), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + wasCalledHandler(meta_global) + ), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + }; + + function mockHandler(path, mock) { + server.registerPathHandler(path, mock(handlers[path])); + return { + restore() { + server.registerPathHandler(path, handlers[path]); + }, + }; + } + + let server = httpd_setup(handlers); + + try { + _("Checking Status.sync with no credentials."); + await Service.verifyAndFetchSymmetricKeys(); + Assert.equal(Service.status.sync, CREDENTIALS_CHANGED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await configureIdentity({ username: "johndoe" }, server); + + await Service.login(); + _("Checking that remoteSetup returns true when credentials have changed."); + (await Service.recordManager.get(Service.metaURL)).payload.syncID = + "foobar"; + Assert.ok(await Service._remoteSetup()); + + let returnStatusCode = (method, code) => oldMethod => (req, res) => { + if (req.method === method) { + res.setStatusLine(req.httpVersion, code, ""); + } else { + oldMethod(req, res); + } + }; + + let mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 401)); + Service.recordManager.del(Service.metaURL); + _( + "Checking that remoteSetup returns false on 401 on first get /meta/global." + ); + Assert.equal(false, await Service._remoteSetup()); + mock.restore(); + + await Service.login(); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 503)); + Service.recordManager.del(Service.metaURL); + _( + "Checking that remoteSetup returns false on 503 on first get /meta/global." + ); + Assert.equal(false, await Service._remoteSetup()); + Assert.equal(Service.status.sync, METARECORD_DOWNLOAD_FAIL); + mock.restore(); + + await Service.login(); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 404)); + Service.recordManager.del(Service.metaURL); + _("Checking that remoteSetup recovers on 404 on first get /meta/global."); + Assert.ok(await Service._remoteSetup()); + mock.restore(); + + let makeOutdatedMeta = async () => { + Service.metaModified = 0; + let infoResponse = await Service._fetchInfo(); + return { + status: infoResponse.status, + obj: { + crypto: infoResponse.obj.crypto, + clients: infoResponse.obj.clients, + meta: 1, + }, + }; + }; + + _( + "Checking that remoteSetup recovers on 404 on get /meta/global after clear cached one." + ); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 404)); + Service.recordManager.set(Service.metaURL, { isNew: false }); + Assert.ok(await Service._remoteSetup(await makeOutdatedMeta())); + mock.restore(); + + _( + "Checking that remoteSetup returns false on 503 on get /meta/global after clear cached one." + ); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 503)); + Service.status.sync = ""; + Service.recordManager.set(Service.metaURL, { isNew: false }); + Assert.equal(false, await Service._remoteSetup(await makeOutdatedMeta())); + Assert.equal(Service.status.sync, ""); + mock.restore(); + + metaColl.delete({}); + + _("Do an initial sync."); + await Service.sync(); + + _("Checking that remoteSetup returns true."); + Assert.ok(await Service._remoteSetup()); + + _("Verify that the meta record was uploaded."); + Assert.equal(meta_global.data.syncID, Service.syncID); + Assert.equal(meta_global.data.storageVersion, STORAGE_VERSION); + Assert.equal( + meta_global.data.engines.clients.version, + Service.clientsEngine.version + ); + Assert.equal( + meta_global.data.engines.clients.syncID, + await Service.clientsEngine.getSyncID() + ); + + _( + "Set the collection info hash so that sync() will remember the modified times for future runs." + ); + let lastSync = await Service.clientsEngine.getLastSync(); + collections.meta = lastSync; + collections.clients = lastSync; + await Service.sync(); + + _("Sync again and verify that meta/global wasn't downloaded again"); + meta_global.wasCalled = false; + await Service.sync(); + Assert.ok(!meta_global.wasCalled); + + _( + "Fake modified records. This will cause a redownload, but not reupload since it hasn't changed." + ); + collections.meta += 42; + meta_global.wasCalled = false; + + let metaModified = meta_global.modified; + + await Service.sync(); + Assert.ok(meta_global.wasCalled); + Assert.equal(metaModified, meta_global.modified); + + // Try to screw up HMAC calculation. + // Re-encrypt keys with a new random keybundle, and upload them to the + // server, just as might happen with a second client. + _("Attempting to screw up HMAC by re-encrypting keys."); + let keys = Service.collectionKeys.asWBO(); + let b = new BulkKeyBundle("hmacerror"); + await b.generateRandom(); + collections.crypto = keys.modified = 100 + Date.now() / 1000; // Future modification time. + await keys.encrypt(b); + await keys.upload(Service.resource(Service.cryptoKeysURL)); + + Assert.equal(false, await Service.verifyAndFetchSymmetricKeys()); + Assert.equal(Service.status.login, LOGIN_FAILED_INVALID_PASSPHRASE); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + server.stop(do_test_finished); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_specified.js b/services/sync/tests/unit/test_service_sync_specified.js new file mode 100644 index 0000000000..845cdb3669 --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_specified.js @@ -0,0 +1,150 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let syncedEngines = []; + +function SteamEngine() { + SyncEngine.call(this, "Steam", Service); +} +SteamEngine.prototype = { + async _sync() { + syncedEngines.push(this.name); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function StirlingEngine() { + SyncEngine.call(this, "Stirling", Service); +} +StirlingEngine.prototype = { + async _sync() { + syncedEngines.push(this.name); + }, +}; +Object.setPrototypeOf(StirlingEngine.prototype, SteamEngine.prototype); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup(handlers) { + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp() { + syncedEngines = []; + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + engine.syncPriority = 1; + + engine = Service.engineManager.get("stirling"); + engine.enabled = true; + engine.syncPriority = 2; + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": new ServerWBO("global", {}).handler(), + }); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + return server; +} + +add_task(async function setup() { + await Service.engineManager.clear(); + validate_all_future_pings(); + + await Service.engineManager.register(SteamEngine); + await Service.engineManager.register(StirlingEngine); +}); + +add_task(async function test_noEngines() { + enableValidationPrefs(); + + _("Test: An empty array of engines to sync does nothing."); + let server = await setUp(); + + try { + _("Sync with no engines specified."); + await Service.sync({ engines: [] }); + deepEqual(syncedEngines, [], "no engines were synced"); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_oneEngine() { + enableValidationPrefs(); + + _("Test: Only one engine is synced."); + let server = await setUp(); + + try { + _("Sync with 1 engine specified."); + await Service.sync({ engines: ["steam"] }); + deepEqual(syncedEngines, ["steam"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesSpecified() { + enableValidationPrefs(); + + _("Test: All engines are synced when specified in the correct order (1)."); + let server = await setUp(); + + try { + _("Sync with both engines specified."); + await Service.sync({ engines: ["steam", "stirling"] }); + deepEqual(syncedEngines, ["steam", "stirling"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesSpecified() { + enableValidationPrefs(); + + _("Test: All engines are synced when specified in the correct order (2)."); + let server = await setUp(); + + try { + _("Sync with both engines specified."); + await Service.sync({ engines: ["stirling", "steam"] }); + deepEqual(syncedEngines, ["stirling", "steam"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesDefault() { + enableValidationPrefs(); + + _("Test: All engines are synced when nothing is specified."); + let server = await setUp(); + + try { + await Service.sync(); + deepEqual(syncedEngines, ["steam", "stirling"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js b/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js new file mode 100644 index 0000000000..fd8b3f71bc --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js @@ -0,0 +1,587 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { EngineSynchronizer } = ChromeUtils.importESModule( + "resource://services-sync/stages/enginesync.sys.mjs" +); + +function QuietStore() { + Store.call("Quiet"); +} +QuietStore.prototype = { + async getAllIDs() { + return []; + }, +}; + +function SteamEngine() { + SyncEngine.call(this, "Steam", Service); +} +SteamEngine.prototype = { + // We're not interested in engine sync but what the service does. + _storeObj: QuietStore, + + _sync: async function _sync() { + await this._syncStartup(); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function StirlingEngine() { + SyncEngine.call(this, "Stirling", Service); +} +StirlingEngine.prototype = { + // This engine's enabled state is the same as the SteamEngine's. + get prefName() { + return "steam"; + }, +}; +Object.setPrototypeOf(StirlingEngine.prototype, SteamEngine.prototype); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup(handlers) { + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp(server) { + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + // Ensure that the server has valid keys so that logging in will work and not + // result in a server wipe, rendering many of these tests useless. + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let { success } = await serverKeys.upload( + Service.resource(Service.cryptoKeysURL) + ); + ok(success); +} + +const PAYLOAD = 42; + +add_task(async function setup() { + await Service.engineManager.clear(); + validate_all_future_pings(); + + await Service.engineManager.register(SteamEngine); + await Service.engineManager.register(StirlingEngine); +}); + +add_task(async function test_newAccount() { + enableValidationPrefs(); + + _("Test: New account does not disable locally enabled engines."); + let engine = Service.engineManager.get("steam"); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": new ServerWBO("global", {}).handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Engine is enabled from the beginning."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Engine continues to be enabled."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_enabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is disabled on remote clients and enabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally."); + engine.enabled = true; + + _("Sync."); + await Service.sync(); + + _("Meta record now contains the new engine."); + Assert.ok(!!metaWBO.data.engines.steam); + + _("Engine continues to be enabled."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is enabled on remote clients and disabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + let steamCollection = new ServerWBO("steam", PAYLOAD); + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": steamCollection.handler(), + }); + await setUp(server); + + try { + _("Disable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + engine.enabled = false; + + _("Sync."); + await Service.sync(); + + _("Meta record no longer contains engine."); + Assert.ok(!metaWBO.data.engines.steam); + + _("Server records are wiped."); + Assert.equal(steamCollection.payload, undefined); + + _("Engine continues to be disabled."); + Assert.ok(!engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledLocally_wipe503() { + enableValidationPrefs(); + + _("Test: Engine is enabled on remote clients and disabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + + function service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "23"); + response.bodyOutputStream.write(body, body.length); + } + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": service_unavailable, + }); + await setUp(server); + + _("Disable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + engine.enabled = false; + + _("Sync."); + await Service.sync(); + Assert.equal(Service.status.sync, SERVER_MAINTENANCE); + + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_enabledRemotely() { + enableValidationPrefs(); + + _("Test: Engine is disabled locally and enabled on a remote client"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", metaWBO.handler()), + + "/1.1/johndoe/storage/steam": upd( + "steam", + new ServerWBO("steam", {}).handler() + ), + }); + await setUp(server); + + // We need to be very careful how we do this, so that we don't trigger a + // fresh start! + try { + _("Upload some keys to avoid a fresh start."); + let wbo = await Service.collectionKeys.generateNewKeysWBO(); + await wbo.encrypt(Service.identity.syncKeyBundle); + Assert.equal( + 200, + (await wbo.upload(Service.resource(Service.cryptoKeysURL))).status + ); + + _("Engine is disabled."); + Assert.ok(!engine.enabled); + + _("Sync."); + await Service.sync(); + + _("Engine is enabled."); + Assert.ok(engine.enabled); + + _("Meta record still present."); + Assert.equal(metaWBO.data.engines.steam.syncID, await engine.getSyncID()); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledRemotelyTwoClients() { + enableValidationPrefs(); + + _( + "Test: Engine is enabled locally and disabled on a remote client... with two clients." + ); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", metaWBO.handler()), + + "/1.1/johndoe/storage/steam": upd( + "steam", + new ServerWBO("steam", {}).handler() + ), + }); + await setUp(server); + + try { + _("Enable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Disable engine by deleting from meta/global."); + let d = metaWBO.data; + delete d.engines.steam; + metaWBO.payload = JSON.stringify(d); + metaWBO.modified = Date.now() / 1000; + + _("Add a second client and verify that the local pref is changed."); + Service.clientsEngine._store._remoteClients.foobar = { + name: "foobar", + type: "desktop", + }; + await Service.sync(); + + _("Engine is disabled."); + Assert.ok(!engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledRemotely() { + enableValidationPrefs(); + + _("Test: Engine is enabled locally and disabled on a remote client"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Engine is not disabled: only one client."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_dependentEnginesEnabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is disabled on remote clients and enabled locally"); + Service.syncID = "abcdefghij"; + let steamEngine = Service.engineManager.get("steam"); + let stirlingEngine = Service.engineManager.get("stirling"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + "/1.1/johndoe/storage/stirling": new ServerWBO("stirling", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally. Doing it on one is enough."); + steamEngine.enabled = true; + + _("Sync."); + await Service.sync(); + + _("Meta record now contains the new engines."); + Assert.ok(!!metaWBO.data.engines.steam); + Assert.ok(!!metaWBO.data.engines.stirling); + + _("Engines continue to be enabled."); + Assert.ok(steamEngine.enabled); + Assert.ok(stirlingEngine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_dependentEnginesDisabledLocally() { + enableValidationPrefs(); + + _( + "Test: Two dependent engines are enabled on remote clients and disabled locally" + ); + Service.syncID = "abcdefghij"; + let steamEngine = Service.engineManager.get("steam"); + let steamSyncID = await steamEngine.resetLocalSyncID(); + let stirlingEngine = Service.engineManager.get("stirling"); + let stirlingSyncID = await stirlingEngine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + steam: { syncID: steamSyncID, version: steamEngine.version }, + stirling: { syncID: stirlingSyncID, version: stirlingEngine.version }, + }, + }); + + let steamCollection = new ServerWBO("steam", PAYLOAD); + let stirlingCollection = new ServerWBO("stirling", PAYLOAD); + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": steamCollection.handler(), + "/1.1/johndoe/storage/stirling": stirlingCollection.handler(), + }); + await setUp(server); + + try { + _("Disable engines locally. Doing it on one is enough."); + Service._ignorePrefObserver = true; + steamEngine.enabled = true; + Assert.ok(stirlingEngine.enabled); + Service._ignorePrefObserver = false; + steamEngine.enabled = false; + Assert.ok(!stirlingEngine.enabled); + + _("Sync."); + await Service.sync(); + + _("Meta record no longer contains engines."); + Assert.ok(!metaWBO.data.engines.steam); + Assert.ok(!metaWBO.data.engines.stirling); + + _("Server records are wiped."); + Assert.equal(steamCollection.payload, undefined); + Assert.equal(stirlingCollection.payload, undefined); + + _("Engines continue to be disabled."); + Assert.ok(!steamEngine.enabled); + Assert.ok(!stirlingEngine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_service_updateLocalEnginesState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: ["steam"], + engines: {}, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service._ignorePrefObserver = true; + // Steam engine is enabled on our machine. + engine.enabled = true; + Service._ignorePrefObserver = false; + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now disabled. + Assert.ok(!engine.enabled); +}); + +add_task(async function test_service_enableAfterUpdateState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: ["steam"], + engines: { someengine: {} }, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now disabled, reflecting what's on the server. + Assert.ok(!engine.enabled); + // Enable the engine, as though the user selected it via CWTS. + engine.enabled = true; + + // Do the "reconcile local and remote states" dance. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledEngines(); + await Service._maybeUpdateDeclined(); + // engine should remain enabled. + Assert.ok(engine.enabled); + // engine should no longer appear in declined on the server. + Assert.deepEqual(metaWBO.data.declined, []); +}); + +add_task(async function test_service_disableAfterUpdateState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: [], + engines: { steam: {} }, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now enabled, reflecting what's on the server. + Assert.ok(engine.enabled); + // Disable the engine, as though via CWTS. + engine.enabled = false; + + // Do the "reconcile local and remote states" dance. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledEngines(); + await Service._maybeUpdateDeclined(); + // engine should remain disabled. + Assert.ok(!engine.enabled); + // engine should now appear in declined on the server. + Assert.deepEqual(metaWBO.data.declined, ["steam"]); + // and should have been removed from engines. + Assert.deepEqual(metaWBO.data.engines, {}); +}); + +add_task(async function test_service_updateLocalEnginesState_no_meta_global() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + // The server doesn't contain /meta/global (sync was never enabled). + const server = httpd_setup({}); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service._ignorePrefObserver = true; + // Steam engine is enabled on our machine. + engine.enabled = true; + Service._ignorePrefObserver = false; + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Still enabled. + Assert.ok(engine.enabled); +}); diff --git a/services/sync/tests/unit/test_service_verifyLogin.js b/services/sync/tests/unit/test_service_verifyLogin.js new file mode 100644 index 0000000000..b99b5c692c --- /dev/null +++ b/services/sync/tests/unit/test_service_verifyLogin.js @@ -0,0 +1,118 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function login_handling(handler) { + return function (request, response) { + if (has_hawk_header(request)) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +function service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "42"); + response.bodyOutputStream.write(body, body.length); +} + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + run_next_test(); +} + +add_task(async function test_verifyLogin() { + // This test expects a clean slate -- no saved passphrase. + Services.logins.removeAllUserFacingLogins(); + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + + do_test_pending(); + + let server = httpd_setup({ + "/1.1/johndoe/info/collections": login_handling(johnHelper.handler), + "/1.1/janedoe/info/collections": service_unavailable, + + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + }); + + try { + _("Force the initial state."); + Service.status.service = STATUS_OK; + Assert.equal(Service.status.service, STATUS_OK); + + _("Credentials won't check out because we're not configured yet."); + Service.status.resetSync(); + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_USERNAME); + + _("Success if syncBundleKey is set."); + Service.status.resetSync(); + await configureIdentity({ username: "johndoe" }, server); + Assert.ok(await Service.verifyLogin()); + Assert.equal(Service.status.service, STATUS_OK); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + + _( + "If verifyLogin() encounters a server error, it flips on the backoff flag and notifies observers on a 503 with Retry-After." + ); + Service.status.resetSync(); + await configureIdentity({ username: "janedoe" }, server); + Service._updateCachedURLs(); + Assert.ok(!Service.status.enforceBackoff); + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + Assert.equal(false, await Service.verifyLogin()); + Assert.ok(Service.status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, SERVER_MAINTENANCE); + + _( + "Ensure a network error when finding the cluster sets the right Status bits." + ); + Service.status.resetSync(); + Service.clusterURL = ""; + Service.identity._findCluster = () => "http://localhost:12345/"; + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + + _( + "Ensure a network error when getting the collection info sets the right Status bits." + ); + Service.status.resetSync(); + Service.clusterURL = "http://localhost:12345/"; + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + server.stop(do_test_finished); + } +}); diff --git a/services/sync/tests/unit/test_service_wipeClient.js b/services/sync/tests/unit/test_service_wipeClient.js new file mode 100644 index 0000000000..aa48868ca0 --- /dev/null +++ b/services/sync/tests/unit/test_service_wipeClient.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function CanDecryptEngine() { + SyncEngine.call(this, "CanDecrypt", Service); +} +CanDecryptEngine.prototype = { + // Override these methods with mocks for the test + async canDecrypt() { + return true; + }, + + wasWiped: false, + async wipeClient() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(CanDecryptEngine.prototype, SyncEngine.prototype); + +function CannotDecryptEngine() { + SyncEngine.call(this, "CannotDecrypt", Service); +} +CannotDecryptEngine.prototype = { + // Override these methods with mocks for the test + async canDecrypt() { + return false; + }, + + wasWiped: false, + async wipeClient() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(CannotDecryptEngine.prototype, SyncEngine.prototype); + +let canDecryptEngine; +let cannotDecryptEngine; + +add_task(async function setup() { + await Service.engineManager.clear(); + + await Service.engineManager.register(CanDecryptEngine); + await Service.engineManager.register(CannotDecryptEngine); + canDecryptEngine = Service.engineManager.get("candecrypt"); + cannotDecryptEngine = Service.engineManager.get("cannotdecrypt"); +}); + +add_task(async function test_withEngineList() { + try { + _("Ensure initial scenario."); + Assert.ok(!canDecryptEngine.wasWiped); + Assert.ok(!cannotDecryptEngine.wasWiped); + + _("Wipe local engine data."); + await Service.wipeClient(["candecrypt", "cannotdecrypt"]); + + _("Ensure only the engine that can decrypt was wiped."); + Assert.ok(canDecryptEngine.wasWiped); + Assert.ok(!cannotDecryptEngine.wasWiped); + } finally { + canDecryptEngine.wasWiped = false; + cannotDecryptEngine.wasWiped = false; + await Service.startOver(); + } +}); + +add_task(async function test_startOver_clears_keys() { + syncTestLogging(); + await generateNewKeys(Service.collectionKeys); + Assert.ok(!!Service.collectionKeys.keyForCollection()); + await Service.startOver(); + syncTestLogging(); + Assert.ok(!Service.collectionKeys.keyForCollection()); +}); diff --git a/services/sync/tests/unit/test_service_wipeServer.js b/services/sync/tests/unit/test_service_wipeServer.js new file mode 100644 index 0000000000..9fc2592aa8 --- /dev/null +++ b/services/sync/tests/unit/test_service_wipeServer.js @@ -0,0 +1,240 @@ +Svc.PrefBranch.setStringPref("registerEngines", ""); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// configure the identity we use for this test. +const identityConfig = makeIdentityConfig({ username: "johndoe" }); + +function FakeCollection() { + this.deleted = false; +} +FakeCollection.prototype = { + handler() { + let self = this; + return function (request, response) { + let body = ""; + self.timestamp = new_timestamp(); + let timestamp = "" + self.timestamp; + if (request.method == "DELETE") { + body = timestamp; + self.deleted = true; + } + response.setHeader("X-Weave-Timestamp", timestamp); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + }; + }, +}; + +async function setUpTestFixtures(server) { + Service.clusterURL = server.baseURI + "/"; + + await configureIdentity(identityConfig); +} + +add_task(async function test_wipeServer_list_success() { + _("Service.wipeServer() deletes collections given as argument."); + + let steam_coll = new FakeCollection(); + let diesel_coll = new FakeCollection(); + + let server = httpd_setup({ + "/1.1/johndoe/storage/steam": steam_coll.handler(), + "/1.1/johndoe/storage/diesel": diesel_coll.handler(), + "/1.1/johndoe/storage/petrol": httpd_handler(404, "Not Found"), + }); + + try { + await setUpTestFixtures(server); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + + _("Confirm initial environment."); + Assert.ok(!steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + + _( + "wipeServer() will happily ignore the non-existent collection and use the timestamp of the last DELETE that was successful." + ); + let timestamp = await Service.wipeServer(["steam", "diesel", "petrol"]); + Assert.equal(timestamp, diesel_coll.timestamp); + + _( + "wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted." + ); + Assert.ok(steam_coll.deleted); + Assert.ok(diesel_coll.deleted); + } finally { + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_wipeServer_list_503() { + _("Service.wipeServer() deletes collections given as argument."); + + let steam_coll = new FakeCollection(); + let diesel_coll = new FakeCollection(); + + let server = httpd_setup({ + "/1.1/johndoe/storage/steam": steam_coll.handler(), + "/1.1/johndoe/storage/petrol": httpd_handler(503, "Service Unavailable"), + "/1.1/johndoe/storage/diesel": diesel_coll.handler(), + }); + + try { + await setUpTestFixtures(server); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + + _("Confirm initial environment."); + Assert.ok(!steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + + _( + "wipeServer() will happily ignore the non-existent collection, delete the 'steam' collection and abort after an receiving an error on the 'petrol' collection." + ); + let error; + try { + await Service.wipeServer(["non-existent", "steam", "petrol", "diesel"]); + do_throw("Should have thrown!"); + } catch (ex) { + error = ex; + } + _("wipeServer() threw this exception: " + error); + Assert.equal(error.status, 503); + + _( + "wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted." + ); + Assert.ok(steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + } finally { + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_wipeServer_all_success() { + _("Service.wipeServer() deletes all the things."); + + /** + * Handle the bulk DELETE request sent by wipeServer. + */ + let deleted = false; + let serverTimestamp; + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + deleted = true; + serverTimestamp = return_timestamp(request, response); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + let returnedTimestamp = await Service.wipeServer(); + Assert.ok(deleted); + Assert.equal(returnedTimestamp, serverTimestamp); + + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } +}); + +add_task(async function test_wipeServer_all_404() { + _("Service.wipeServer() accepts a 404."); + + /** + * Handle the bulk DELETE request sent by wipeServer. Returns a 404. + */ + let deleted = false; + let serverTimestamp; + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + deleted = true; + serverTimestamp = new_timestamp(); + response.setHeader("X-Weave-Timestamp", "" + serverTimestamp); + response.setStatusLine(request.httpVersion, 404, "Not Found"); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + let returnedTimestamp = await Service.wipeServer(); + Assert.ok(deleted); + Assert.equal(returnedTimestamp, serverTimestamp); + + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } +}); + +add_task(async function test_wipeServer_all_503() { + _("Service.wipeServer() throws if it encounters a non-200/404 response."); + + /** + * Handle the bulk DELETE request sent by wipeServer. Returns a 503. + */ + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + let error; + try { + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + await Service.wipeServer(); + do_throw("Should have thrown!"); + } catch (ex) { + error = ex; + } + Assert.equal(error.status, 503); + + await promiseStopServer(server); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } +}); + +add_task(async function test_wipeServer_all_connectionRefused() { + _("Service.wipeServer() throws if it encounters a network problem."); + let server = httpd_setup({}); + await setUpTestFixtures(server); + + Service.clusterURL = "http://localhost:4352/"; + + _("Try deletion."); + try { + await Service.wipeServer(); + do_throw("Should have thrown!"); + } catch (ex) { + Assert.equal(ex.result, Cr.NS_ERROR_CONNECTION_REFUSED); + } + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_status.js b/services/sync/tests/unit/test_status.js new file mode 100644 index 0000000000..5bcfa182c3 --- /dev/null +++ b/services/sync/tests/unit/test_status.js @@ -0,0 +1,83 @@ +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +function run_test() { + // Check initial states + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + if (Status.engines.length) { + do_throw("Status.engines should be empty."); + } + Assert.equal(Status.partial, false); + + // Check login status + for (let code of [LOGIN_FAILED_NO_USERNAME, LOGIN_FAILED_NO_PASSPHRASE]) { + Status.login = code; + Assert.equal(Status.login, code); + Assert.equal(Status.service, CLIENT_NOT_CONFIGURED); + Status.resetSync(); + } + + Status.login = LOGIN_FAILED; + Assert.equal(Status.login, LOGIN_FAILED); + Assert.equal(Status.service, LOGIN_FAILED); + Status.resetSync(); + + Status.login = LOGIN_SUCCEEDED; + Assert.equal(Status.login, LOGIN_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + Status.resetSync(); + + // Check sync status + Status.sync = SYNC_FAILED; + Assert.equal(Status.sync, SYNC_FAILED); + Assert.equal(Status.service, SYNC_FAILED); + + Status.sync = SYNC_SUCCEEDED; + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + + Status.resetSync(); + + // Check engine status + Status.engines = ["testEng1", ENGINE_SUCCEEDED]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + + Status.engines = ["testEng2", ENGINE_DOWNLOAD_FAIL]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.engines.testEng2, ENGINE_DOWNLOAD_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + Status.engines = ["testEng3", ENGINE_SUCCEEDED]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.engines.testEng2, ENGINE_DOWNLOAD_FAIL); + Assert.equal(Status.engines.testEng3, ENGINE_SUCCEEDED); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // Check resetSync + Status.sync = SYNC_FAILED; + Status.resetSync(); + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + if (Status.engines.length) { + do_throw("Status.engines should be empty."); + } + + // Check resetBackoff + Status.enforceBackoff = true; + Status.backOffInterval = 4815162342; + Status.backOffInterval = 42; + Status.resetBackoff(); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); +} diff --git a/services/sync/tests/unit/test_status_checkSetup.js b/services/sync/tests/unit/test_status_checkSetup.js new file mode 100644 index 0000000000..fe3e7cad8d --- /dev/null +++ b/services/sync/tests/unit/test_status_checkSetup.js @@ -0,0 +1,26 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +add_task(async function test_status_checkSetup() { + try { + _("Fresh setup, we're not configured."); + Assert.equal(Status.checkSetup(), CLIENT_NOT_CONFIGURED); + Assert.equal(Status.login, LOGIN_FAILED_NO_USERNAME); + Status.resetSync(); + + _("Let's provide the syncKeyBundle"); + await configureIdentity(); + + _("checkSetup()"); + Assert.equal(Status.checkSetup(), STATUS_OK); + Status.resetSync(); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); diff --git a/services/sync/tests/unit/test_sync_auth_manager.js b/services/sync/tests/unit/test_sync_auth_manager.js new file mode 100644 index 0000000000..9af40d26c6 --- /dev/null +++ b/services/sync/tests/unit/test_sync_auth_manager.js @@ -0,0 +1,1027 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AuthenticationError, SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { initializeIdentityWithTokenServerResponse } = + ChromeUtils.importESModule( + "resource://testing-common/services/sync/fxa_utils.sys.mjs" + ); +const { HawkClient } = ChromeUtils.importESModule( + "resource://services-common/hawkclient.sys.mjs" +); +const { FxAccounts } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const { FxAccountsClient } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsClient.sys.mjs" +); +const { + ERRNO_INVALID_AUTH_TOKEN, + ONLOGIN_NOTIFICATION, + ONVERIFIED_NOTIFICATION, +} = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsCommon.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { TokenServerClient, TokenServerClientServerError } = + ChromeUtils.importESModule( + "resource://services-common/tokenserverclient.sys.mjs" + ); +const { AccountState } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); + +const SECOND_MS = 1000; +const MINUTE_MS = SECOND_MS * 60; +const HOUR_MS = MINUTE_MS * 60; + +const MOCK_ACCESS_TOKEN = + "e3c5caf17f27a0d9e351926a928938b3737df43e91d4992a5a5fca9a7bdef8ba"; + +var globalIdentityConfig = makeIdentityConfig(); +var globalSyncAuthManager = new SyncAuthManager(); +configureFxAccountIdentity(globalSyncAuthManager, globalIdentityConfig); + +/** + * Mock client clock and skew vs server in FxAccounts signed-in user module and + * API client. sync_auth.js queries these values to construct HAWK + * headers. We will use this to test clock skew compensation in these headers + * below. + */ +var MockFxAccountsClient = function () { + FxAccountsClient.apply(this); +}; +MockFxAccountsClient.prototype = { + accountStatus() { + return Promise.resolve(true); + }, + getScopedKeyData() { + return Promise.resolve({ + "https://identity.mozilla.com/apps/oldsync": { + identifier: "https://identity.mozilla.com/apps/oldsync", + keyRotationSecret: + "0000000000000000000000000000000000000000000000000000000000000000", + keyRotationTimestamp: 1234567890123, + }, + }); + }, +}; +Object.setPrototypeOf( + MockFxAccountsClient.prototype, + FxAccountsClient.prototype +); + +add_test(function test_initial_state() { + _("Verify initial state"); + Assert.ok(!globalSyncAuthManager._token); + Assert.ok(!globalSyncAuthManager._hasValidToken()); + run_next_test(); +}); + +add_task(async function test_initialialize() { + _("Verify start after fetching token"); + await globalSyncAuthManager._ensureValidToken(); + Assert.ok(!!globalSyncAuthManager._token); + Assert.ok(globalSyncAuthManager._hasValidToken()); +}); + +add_task(async function test_refreshOAuthTokenOn401() { + _("Refreshes the FXA OAuth token after a 401."); + let getTokenCount = 0; + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + syncAuthManager._fxaService.getOAuthToken = () => { + ++getTokenCount; + return Promise.resolve(MOCK_ACCESS_TOKEN); + }; + + let didReturn401 = false; + let didReturn200 = false; + let mockTSC = mockTokenServer(() => { + if (getTokenCount <= 1) { + didReturn401 = true; + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + didReturn200 = true; + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "id", + key: "key", + api_endpoint: "http://example.com/", + uid: "uid", + duration: 300, + }), + }; + }); + + syncAuthManager._tokenServerClient = mockTSC; + + await syncAuthManager._ensureValidToken(); + + Assert.equal(getTokenCount, 2); + Assert.ok(didReturn401); + Assert.ok(didReturn200); + Assert.ok(syncAuthManager._token); + Assert.ok(syncAuthManager._hasValidToken()); +}); + +add_task(async function test_initialializeWithAuthErrorAndDeletedAccount() { + _("Verify sync state with auth error + account deleted"); + + var identityConfig = makeIdentityConfig(); + var syncAuthManager = new SyncAuthManager(); + + // Use the real `getOAuthToken` method that calls + // `mockFxAClient.accessTokenWithSessionToken`. + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + delete fxaInternal.getOAuthToken; + + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + + let accessTokenWithSessionTokenCalled = false; + let accountStatusCalled = false; + let sessionStatusCalled = false; + + let AuthErrorMockFxAClient = function () { + FxAccountsClient.apply(this); + }; + AuthErrorMockFxAClient.prototype = { + accessTokenWithSessionToken() { + accessTokenWithSessionTokenCalled = true; + return Promise.reject({ + code: 401, + errno: ERRNO_INVALID_AUTH_TOKEN, + }); + }, + accountStatus() { + accountStatusCalled = true; + return Promise.resolve(false); + }, + sessionStatus() { + sessionStatusCalled = true; + return Promise.resolve(false); + }, + }; + Object.setPrototypeOf( + AuthErrorMockFxAClient.prototype, + FxAccountsClient.prototype + ); + + let mockFxAClient = new AuthErrorMockFxAClient(); + syncAuthManager._fxaService._internal._fxAccountsClient = mockFxAClient; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + AuthenticationError, + "should reject due to an auth error" + ); + + Assert.ok(accessTokenWithSessionTokenCalled); + Assert.ok(sessionStatusCalled); + Assert.ok(accountStatusCalled); + Assert.ok(!syncAuthManager._token); + Assert.ok(!syncAuthManager._hasValidToken()); +}); + +add_task(async function test_getResourceAuthenticator() { + _( + "SyncAuthManager supplies a Resource Authenticator callback which returns a Hawk header." + ); + configureFxAccountIdentity(globalSyncAuthManager); + let authenticator = globalSyncAuthManager.getResourceAuthenticator(); + Assert.ok(!!authenticator); + let req = { + uri: CommonUtils.makeURI("https://example.net/somewhere/over/the/rainbow"), + method: "GET", + }; + let output = await authenticator(req, "GET"); + Assert.ok("headers" in output); + Assert.ok("authorization" in output.headers); + Assert.ok(output.headers.authorization.startsWith("Hawk")); + _("Expected internal state after successful call."); + Assert.equal( + globalSyncAuthManager._token.uid, + globalIdentityConfig.fxaccount.token.uid + ); +}); + +add_task(async function test_resourceAuthenticatorSkew() { + _( + "SyncAuthManager Resource Authenticator compensates for clock skew in Hawk header." + ); + + // Clock is skewed 12 hours into the future + // We pick a date in the past so we don't risk concealing bugs in code that + // uses new Date() instead of our given date. + let now = + new Date("Fri Apr 09 2004 00:00:00 GMT-0700").valueOf() + 12 * HOUR_MS; + let syncAuthManager = new SyncAuthManager(); + let hawkClient = new HawkClient("https://example.net/v1", "/foo"); + + // mock fxa hawk client skew + hawkClient.now = function () { + dump("mocked client now: " + now + "\n"); + return now; + }; + // Imagine there's already been one fxa request and the hawk client has + // already detected skew vs the fxa auth server. + let localtimeOffsetMsec = -1 * 12 * HOUR_MS; + hawkClient._localtimeOffsetMsec = localtimeOffsetMsec; + + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = hawkClient; + + // Sanity check + Assert.equal(hawkClient.now(), now); + Assert.equal(hawkClient.localtimeOffsetMsec, localtimeOffsetMsec); + + // Properly picked up by the client + Assert.equal(fxaClient.now(), now); + Assert.equal(fxaClient.localtimeOffsetMsec, localtimeOffsetMsec); + + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal._now_is = now; + fxaInternal.fxAccountsClient = fxaClient; + + // Mocks within mocks... + configureFxAccountIdentity( + syncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + Assert.equal( + syncAuthManager._fxaService._internal.localtimeOffsetMsec, + localtimeOffsetMsec + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + Assert.equal( + syncAuthManager._fxaService._internal.localtimeOffsetMsec, + localtimeOffsetMsec + ); + + let request = new Resource("https://example.net/i/like/pie/"); + let authenticator = syncAuthManager.getResourceAuthenticator(); + let output = await authenticator(request, "GET"); + dump("output" + JSON.stringify(output)); + let authHeader = output.headers.authorization; + Assert.ok(authHeader.startsWith("Hawk")); + + // Skew correction is applied in the header and we're within the two-minute + // window. + Assert.equal(getTimestamp(authHeader), now - 12 * HOUR_MS); + Assert.ok(getTimestampDelta(authHeader, now) - 12 * HOUR_MS < 2 * MINUTE_MS); +}); + +add_task(async function test_RESTResourceAuthenticatorSkew() { + _( + "SyncAuthManager REST Resource Authenticator compensates for clock skew in Hawk header." + ); + + // Clock is skewed 12 hours into the future from our arbitary date + let now = + new Date("Fri Apr 09 2004 00:00:00 GMT-0700").valueOf() + 12 * HOUR_MS; + let syncAuthManager = new SyncAuthManager(); + let hawkClient = new HawkClient("https://example.net/v1", "/foo"); + + // mock fxa hawk client skew + hawkClient.now = function () { + return now; + }; + // Imagine there's already been one fxa request and the hawk client has + // already detected skew vs the fxa auth server. + hawkClient._localtimeOffsetMsec = -1 * 12 * HOUR_MS; + + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = hawkClient; + + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal._now_is = now; + fxaInternal.fxAccountsClient = fxaClient; + + configureFxAccountIdentity( + syncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + + let request = new Resource("https://example.net/i/like/pie/"); + let authenticator = syncAuthManager.getResourceAuthenticator(); + let output = await authenticator(request, "GET"); + dump("output" + JSON.stringify(output)); + let authHeader = output.headers.authorization; + Assert.ok(authHeader.startsWith("Hawk")); + + // Skew correction is applied in the header and we're within the two-minute + // window. + Assert.equal(getTimestamp(authHeader), now - 12 * HOUR_MS); + Assert.ok(getTimestampDelta(authHeader, now) - 12 * HOUR_MS < 2 * MINUTE_MS); +}); + +add_task(async function test_ensureLoggedIn() { + configureFxAccountIdentity(globalSyncAuthManager); + await globalSyncAuthManager._ensureValidToken(); + Assert.equal(Status.login, LOGIN_SUCCEEDED, "original initialize worked"); + Assert.ok(globalSyncAuthManager._token); + + // arrange for no logged in user. + let fxa = globalSyncAuthManager._fxaService; + let signedInUser = + fxa._internal.currentAccountState.storageManager.accountData; + fxa._internal.currentAccountState.storageManager.accountData = null; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + /no user is logged in/, + "expecting rejection due to no user" + ); + // Restore the logged in user to what it was. + fxa._internal.currentAccountState.storageManager.accountData = signedInUser; + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + await globalSyncAuthManager._ensureValidToken(true); + Assert.equal(Status.login, LOGIN_SUCCEEDED, "final ensureLoggedIn worked"); +}); + +add_task(async function test_syncState() { + // Avoid polling for an unverified user. + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal.startVerifiedCheck = () => {}; + configureFxAccountIdentity( + globalSyncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + // arrange for no logged in user. + let fxa = globalSyncAuthManager._fxaService; + let signedInUser = + fxa._internal.currentAccountState.storageManager.accountData; + fxa._internal.currentAccountState.storageManager.accountData = null; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + /no user is logged in/, + "expecting rejection due to no user" + ); + // Restore to an unverified user. + Services.prefs.setStringPref("services.sync.username", signedInUser.email); + signedInUser.verified = false; + fxa._internal.currentAccountState.storageManager.accountData = signedInUser; + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + // The sync_auth observers are async, so call them directly. + await globalSyncAuthManager.observe(null, ONLOGIN_NOTIFICATION, ""); + Assert.equal( + Status.login, + LOGIN_FAILED_LOGIN_REJECTED, + "should not have changed the login state for an unverified user" + ); + + // now pretend the user because verified. + signedInUser.verified = true; + await globalSyncAuthManager.observe(null, ONVERIFIED_NOTIFICATION, ""); + Assert.equal( + Status.login, + LOGIN_SUCCEEDED, + "should have changed the login state to success" + ); +}); + +add_task(async function test_tokenExpiration() { + _("SyncAuthManager notices token expiration:"); + let bimExp = new SyncAuthManager(); + configureFxAccountIdentity(bimExp, globalIdentityConfig); + + let authenticator = bimExp.getResourceAuthenticator(); + Assert.ok(!!authenticator); + let req = { + uri: CommonUtils.makeURI("https://example.net/somewhere/over/the/rainbow"), + method: "GET", + }; + await authenticator(req, "GET"); + + // Mock the clock. + _("Forcing the token to expire ..."); + Object.defineProperty(bimExp, "_now", { + value: function customNow() { + return Date.now() + 3000001; + }, + writable: true, + }); + Assert.ok(bimExp._token.expiration < bimExp._now()); + _("... means SyncAuthManager knows to re-fetch it on the next call."); + Assert.ok(!bimExp._hasValidToken()); +}); + +add_task(async function test_getTokenErrors() { + _("SyncAuthManager correctly handles various failures to get a token."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + initializeIdentityWithTokenServerResponse({ + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }); + let syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + AuthenticationError, + "should reject due to 401" + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); + + // XXX - other interesting responses to return? + + // And for good measure, some totally "unexpected" errors - we generally + // assume these problems are going to magically go away at some point. + _( + "Arrange for an empty body with a 200 response - should reflect a network error." + ); + initializeIdentityWithTokenServerResponse({ + status: 200, + headers: [], + body: "", + }); + syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to non-JSON response" + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "login state is LOGIN_FAILED_NETWORK_ERROR" + ); +}); + +add_task(async function test_refreshAccessTokenOn401() { + _("SyncAuthManager refreshes the FXA OAuth access token after a 401."); + var identityConfig = makeIdentityConfig(); + var syncAuthManager = new SyncAuthManager(); + // Use the real `getOAuthToken` method that calls + // `mockFxAClient.accessTokenWithSessionToken`. + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + delete fxaInternal.getOAuthToken; + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + + let getTokenCount = 0; + + let CheckSignMockFxAClient = function () { + FxAccountsClient.apply(this); + }; + CheckSignMockFxAClient.prototype = { + accessTokenWithSessionToken() { + ++getTokenCount; + return Promise.resolve({ access_token: "token" }); + }, + }; + Object.setPrototypeOf( + CheckSignMockFxAClient.prototype, + FxAccountsClient.prototype + ); + + let mockFxAClient = new CheckSignMockFxAClient(); + syncAuthManager._fxaService._internal._fxAccountsClient = mockFxAClient; + + let didReturn401 = false; + let didReturn200 = false; + let mockTSC = mockTokenServer(() => { + if (getTokenCount <= 1) { + didReturn401 = true; + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + didReturn200 = true; + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "id", + key: "key", + api_endpoint: "http://example.com/", + uid: "uid", + duration: 300, + }), + }; + }); + + syncAuthManager._tokenServerClient = mockTSC; + + await syncAuthManager._ensureValidToken(); + + Assert.equal(getTokenCount, 2); + Assert.ok(didReturn401); + Assert.ok(didReturn200); + Assert.ok(syncAuthManager._token); + Assert.ok(syncAuthManager._hasValidToken()); +}); + +add_task(async function test_getTokenErrorWithRetry() { + _("tokenserver sends an observer notification on various backoff headers."); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a Retry-After header."); + initializeIdentityWithTokenServerResponse({ + status: 503, + headers: { "content-type": "application/json", "retry-after": "100" }, + body: JSON.stringify({}), + }); + let syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); + + _("Arrange for a 200 with an X-Backoff header."); + Status.backoffInterval = 0; + initializeIdentityWithTokenServerResponse({ + status: 503, + headers: { "content-type": "application/json", "x-backoff": "200" }, + body: JSON.stringify({}), + }); + syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to no token in response" + ); + + // The observer should have fired - check it got the value in the response. + Assert.ok(Status.backoffInterval >= 200000); +}); + +add_task(async function test_getKeysErrorWithBackoff() { + _( + "Auth server (via hawk) sends an observer notification on backoff headers." + ); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a X-Backoff header."); + + let config = makeIdentityConfig(); + // We want no scopedKeys so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json", "x-backoff": "100" }, + body: "{}", + }; + } + ); + + let syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); +}); + +add_task(async function test_getKeysErrorWithRetry() { + _("Auth server (via hawk) sends an observer notification on retry headers."); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a Retry-After header."); + + let config = makeIdentityConfig(); + // We want no scopedKeys so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json", "retry-after": "100" }, + body: "{}", + }; + } + ); + + let syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); +}); + +add_task(async function test_getHAWKErrors() { + _("SyncAuthManager correctly handles various HAWK failures."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + let config = makeIdentityConfig(); + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + if (uri == "http://mockedserver:9999/oauth/token") { + Assert.equal(method, "post"); + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + code: 401, + errno: 110, + error: "invalid token", + }), + }; + } + // For any follow-up requests that check account status. + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); + + // XXX - other interesting responses to return? + + // And for good measure, some totally "unexpected" errors - we generally + // assume these problems are going to magically go away at some point. + _( + "Arrange for an empty body with a 200 response - should reflect a network error." + ); + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "post"); + Assert.equal(uri, "http://mockedserver:9999/oauth/token"); + return { + status: 200, + headers: [], + body: "", + }; + } + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "login state is LOGIN_FAILED_NETWORK_ERROR" + ); +}); + +add_task(async function test_getGetKeysFailing401() { + _("SyncAuthManager correctly handles 401 responses fetching keys."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + let config = makeIdentityConfig(); + // We want no scopedKeys so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 401, + headers: { "content-type": "application/json" }, + body: "{}", + }; + } + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); +}); + +add_task(async function test_getGetKeysFailing503() { + _("SyncAuthManager correctly handles 5XX responses fetching keys."); + + _("Arrange for a 503 - Sync should reflect a network error."); + let config = makeIdentityConfig(); + // We want no scopedKeys so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json" }, + body: "{}", + }; + } + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "state reflects network error" + ); +}); + +add_task(async function test_getKeysMissing() { + _( + "SyncAuthManager correctly handles getKeyForScope succeeding but not returning the key." + ); + + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + // our mock identity config already has scopedKeys remove them or we never + // try and fetch them. + delete identityConfig.fxaccount.user.scopedKeys; + identityConfig.fxaccount.user.keyFetchToken = "keyFetchToken"; + + configureFxAccountIdentity(syncAuthManager, identityConfig); + + // Mock a fxAccounts object + let fxa = new FxAccounts({ + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(identityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + // And the keys object with a mock that returns no keys. + keys: { + getKeyForScope() { + return Promise.resolve(null); + }, + }, + }); + + syncAuthManager._fxaService = fxa; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + /browser does not have the sync key, cannot sync/ + ); +}); + +add_task(async function test_getKeysUnexpecedError() { + _( + "SyncAuthManager correctly handles getKeyForScope throwing an unexpected error." + ); + + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + // our mock identity config already has scopedKeys - remove them or we never + // try and fetch them. + delete identityConfig.fxaccount.user.scopedKeys; + identityConfig.fxaccount.user.keyFetchToken = "keyFetchToken"; + + configureFxAccountIdentity(syncAuthManager, identityConfig); + + // Mock a fxAccounts object + let fxa = new FxAccounts({ + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(identityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + // And the keys object with a mock that returns no keys. + keys: { + async getKeyForScope() { + throw new Error("well that was unexpected"); + }, + }, + }); + + syncAuthManager._fxaService = fxa; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + /well that was unexpected/ + ); +}); + +add_task(async function test_signedInUserMissing() { + _( + "SyncAuthManager detects getSignedInUser returning incomplete account data" + ); + + let syncAuthManager = new SyncAuthManager(); + // Delete stored keys and the key fetch token. + delete globalIdentityConfig.fxaccount.user.scopedKeys; + delete globalIdentityConfig.fxaccount.user.keyFetchToken; + + configureFxAccountIdentity(syncAuthManager, globalIdentityConfig); + + let fxa = new FxAccounts({ + fetchAndUnwrapKeys() { + return Promise.resolve({}); + }, + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(globalIdentityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + }); + + syncAuthManager._fxaService = fxa; + + let status = await syncAuthManager.unlockAndVerifyAuthState(); + Assert.equal(status, LOGIN_FAILED_LOGIN_REJECTED); +}); + +// End of tests +// Utility functions follow + +// Create a new sync_auth object and initialize it with a +// hawk mock that simulates HTTP responses. +// The callback function will be called each time the mocked hawk server wants +// to make a request. The result of the callback should be the mock response +// object that will be returned to hawk. +// A token server mock will be used that doesn't hit a server, so we move +// directly to a hawk request. +async function initializeIdentityWithHAWKResponseFactory( + config, + cbGetResponse +) { + // A mock request object. + function MockRESTRequest(uri, credentials, extra) { + this._uri = uri; + this._credentials = credentials; + this._extra = extra; + } + MockRESTRequest.prototype = { + setHeader() {}, + async post(data) { + this.response = cbGetResponse( + "post", + data, + this._uri, + this._credentials, + this._extra + ); + return this.response; + }, + async get() { + // Skip /status requests (sync_auth checks if the account still + // exists after an auth error) + if (this._uri.startsWith("http://mockedserver:9999/account/status")) { + this.response = { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ exists: true }), + }; + } else { + this.response = cbGetResponse( + "get", + null, + this._uri, + this._credentials, + this._extra + ); + } + return this.response; + }, + }; + + // The hawk client. + function MockedHawkClient() {} + MockedHawkClient.prototype = new HawkClient("http://mockedserver:9999"); + MockedHawkClient.prototype.constructor = MockedHawkClient; + MockedHawkClient.prototype.newHAWKAuthenticatedRESTRequest = function ( + uri, + credentials, + extra + ) { + return new MockRESTRequest(uri, credentials, extra); + }; + // Arrange for the same observerPrefix as FxAccountsClient uses + MockedHawkClient.prototype.observerPrefix = "FxA:hawk"; + + // tie it all together - configureFxAccountIdentity isn't useful here :( + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = new MockedHawkClient(); + let internal = { + fxAccountsClient: fxaClient, + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(config.fxaccount.user); + return new AccountState(storageManager); + }, + }; + let fxa = new FxAccounts(internal); + + globalSyncAuthManager._fxaService = fxa; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + // TODO: Ideally this should have a specific check for an error. + () => true, + "expecting rejection due to hawk error" + ); +} + +function getTimestamp(hawkAuthHeader) { + return parseInt(/ts="(\d+)"/.exec(hawkAuthHeader)[1], 10) * SECOND_MS; +} + +function getTimestampDelta(hawkAuthHeader, now = Date.now()) { + return Math.abs(getTimestamp(hawkAuthHeader) - now); +} + +function mockTokenServer(func) { + let requestLog = Log.repository.getLogger("testing.mock-rest"); + if (!requestLog.appenders.length) { + // might as well see what it says :) + requestLog.addAppender(new Log.DumpAppender()); + requestLog.level = Log.Level.Trace; + } + function MockRESTRequest(url) {} + MockRESTRequest.prototype = { + _log: requestLog, + setHeader() {}, + async get() { + this.response = func(); + return this.response; + }, + }; + // The mocked TokenServer client which will get the response. + function MockTSC() {} + MockTSC.prototype = new TokenServerClient(); + MockTSC.prototype.constructor = MockTSC; + MockTSC.prototype.newRESTRequest = function (url) { + return new MockRESTRequest(url); + }; + // Arrange for the same observerPrefix as sync_auth uses. + MockTSC.prototype.observerPrefix = "weave:service"; + return new MockTSC(); +} diff --git a/services/sync/tests/unit/test_syncedtabs.js b/services/sync/tests/unit/test_syncedtabs.js new file mode 100644 index 0000000000..79ab3e0686 --- /dev/null +++ b/services/sync/tests/unit/test_syncedtabs.js @@ -0,0 +1,342 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim:set ts=2 sw=2 sts=2 et: + */ +"use strict"; + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { SyncedTabs } = ChromeUtils.importESModule( + "resource://services-sync/SyncedTabs.sys.mjs" +); + +Log.repository.getLogger("Sync.RemoteTabs").addAppender(new Log.DumpAppender()); + +// A mock "Tabs" engine which the SyncedTabs module will use instead of the real +// engine. We pass a constructor that Sync creates. +function MockTabsEngine() { + this.clients = {}; // We'll set this dynamically + // Mock fxAccounts + recentDeviceList as if we hit the FxA server + this.fxAccounts = { + device: { + recentDeviceList: [ + { + id: 1, + name: "updated desktop name", + availableCommands: { + "https://identity.mozilla.com/cmd/open-uri": "baz", + }, + }, + { + id: 2, + name: "updated mobile name", + availableCommands: { + "https://identity.mozilla.com/cmd/open-uri": "boo", + }, + }, + ], + }, + }; +} + +MockTabsEngine.prototype = { + name: "tabs", + enabled: true, + + getAllClients() { + return Object.values(this.clients); + }, + + getOpenURLs() { + return new Set(); + }, +}; + +let tabsEngine; + +// A clients engine that doesn't need to be a constructor. +let MockClientsEngine = { + clientSettings: null, // Set in `configureClients`. + + isMobile(guid) { + if (!guid.endsWith("desktop") && !guid.endsWith("mobile")) { + throw new Error( + "this module expected guids to end with 'desktop' or 'mobile'" + ); + } + return guid.endsWith("mobile"); + }, + remoteClientExists(id) { + return this.clientSettings[id] !== false; + }, + getClientName(id) { + if (this.clientSettings[id]) { + return this.clientSettings[id]; + } + let client = tabsEngine.clients[id]; + let fxaDevice = tabsEngine.fxAccounts.device.recentDeviceList.find( + device => device.id === client.fxaDeviceId + ); + return fxaDevice ? fxaDevice.name : client.clientName; + }, + + getClientFxaDeviceId(id) { + if (this.clientSettings[id]) { + return this.clientSettings[id]; + } + return tabsEngine.clients[id].fxaDeviceId; + }, + + getClientType(id) { + return "desktop"; + }, +}; + +function configureClients(clients, clientSettings = {}) { + // each client record is expected to have an id. + for (let [guid, client] of Object.entries(clients)) { + client.id = guid; + } + tabsEngine.clients = clients; + // Apply clients collection overrides. + MockClientsEngine.clientSettings = clientSettings; + // Send an observer that pretends the engine just finished a sync. + Services.obs.notifyObservers(null, "weave:engine:sync:finish", "tabs"); +} + +add_task(async function setup() { + await Weave.Service.promiseInitialized; + // Configure Sync with our mock tabs engine and force it to become initialized. + await Weave.Service.engineManager.unregister("tabs"); + await Weave.Service.engineManager.register(MockTabsEngine); + Weave.Service.clientsEngine = MockClientsEngine; + tabsEngine = Weave.Service.engineManager.get("tabs"); + + // Tell the Sync XPCOM service it is initialized. + let weaveXPCService = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + weaveXPCService.ready = true; +}); + +// The tests. +add_task(async function test_noClients() { + // no clients, can't be tabs. + await configureClients({}); + + let tabs = await SyncedTabs.getTabClients(); + equal(Object.keys(tabs).length, 0); +}); + +add_task(async function test_clientWithTabs() { + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + lastUsed: 1655745700, // Mon, 20 Jun 2022 17:21:40 GMT + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[0].tabs[0].icon, "http://foo.com/favicon"); + equal(clients[0].tabs[0].lastUsed, 1655745700); + // second client has no tabs. + equal(clients[1].tabs.length, 0); +}); + +add_task(async function test_staleClientWithTabs() { + await configureClients( + { + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + lastUsed: 1655745750, + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [], + }, + guid_stale_mobile: { + clientName: "My Deleted Phone", + tabs: [], + }, + guid_stale_desktop: { + clientName: "My Deleted Laptop", + tabs: [ + { + urlHistory: ["https://bar.com/"], + icon: "https://bar.com/favicon", + lastUsed: 1655745700, + }, + ], + }, + guid_stale_name_desktop: { + clientName: "My Generic Device", + tabs: [ + { + urlHistory: ["https://example.edu/"], + icon: "https://example.edu/favicon", + lastUsed: 1655745800, + }, + ], + }, + }, + { + guid_stale_mobile: false, + guid_stale_desktop: false, + // We should always use the device name from the clients collection, instead + // of the possibly stale tabs collection. + guid_stale_name_desktop: "My Laptop", + } + ); + let clients = await SyncedTabs.getTabClients(); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients.length, 3); + equal(clients[0].name, "My Desktop"); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[0].tabs[0].lastUsed, 1655745750); + equal(clients[1].name, "My Laptop"); + equal(clients[1].tabs.length, 1); + equal(clients[1].tabs[0].url, "https://example.edu/"); + equal(clients[1].tabs[0].lastUsed, 1655745800); + equal(clients[2].name, "My Phone"); + equal(clients[2].tabs.length, 0); +}); + +add_task(async function test_clientWithTabsIconsDisabled() { + Services.prefs.setBoolPref("services.sync.syncedTabs.showRemoteIcons", false); + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 1); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + // Expect the default favicon due to the pref being false. + equal(clients[0].tabs[0].icon, "page-icon:http://foo.com/"); + Services.prefs.clearUserPref("services.sync.syncedTabs.showRemoteIcons"); +}); + +add_task(async function test_filter() { + // Nothing matches. + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + { + urlHistory: ["http://bar.com/"], + title: "Another page.", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients("foo"); + equal(clients.length, 1); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + // check it matches the title. + clients = await SyncedTabs.getTabClients("test"); + equal(clients.length, 1); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); +}); + +add_task(async function test_duplicatesTabsAcrossClients() { + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + equal(clients[0].tabs.length, 1); + equal(clients[1].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[1].tabs[0].url, "http://foo.com/"); +}); + +add_task(async function test_clientsTabUpdatedName() { + // See the "fxAccounts" object in the MockEngine above for the device list + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + }, + ], + fxaDeviceId: 1, + }, + guid_mobile: { + clientName: "My Phone", + tabs: [ + { + urlHistory: ["http://bar.com/"], + icon: "http://bar.com/favicon", + }, + ], + fxaDeviceId: 2, + }, + }); + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + equal(clients[0].name, "updated desktop name"); + equal(clients[1].name, "updated mobile name"); +}); diff --git a/services/sync/tests/unit/test_syncengine.js b/services/sync/tests/unit/test_syncengine.js new file mode 100644 index 0000000000..50995a4e40 --- /dev/null +++ b/services/sync/tests/unit/test_syncengine.js @@ -0,0 +1,302 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function makeSteamEngine() { + let engine = new SyncEngine("Steam", Service); + await engine.initialize(); + return engine; +} + +function guidSetOfSize(length) { + return new SerializableSet(Array.from({ length }, () => Utils.makeGUID())); +} + +function assertSetsEqual(a, b) { + // Assert.deepEqual doesn't understand Set. + Assert.deepEqual(Array.from(a).sort(), Array.from(b).sort()); +} + +async function testSteamEngineStorage(test) { + try { + let setupEngine = await makeSteamEngine(); + + if (test.setup) { + await test.setup(setupEngine); + } + + // Finalize the engine to flush the backlog and previous failed to disk. + await setupEngine.finalize(); + + if (test.beforeCheck) { + await test.beforeCheck(); + } + + let checkEngine = await makeSteamEngine(); + await test.check(checkEngine); + + await checkEngine.resetClient(); + await checkEngine.finalize(); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +} + +let server; + +add_task(async function setup() { + server = httpd_setup({}); +}); + +add_task(async function test_url_attributes() { + _("SyncEngine url attributes"); + await SyncTestingInfrastructure(server); + Service.clusterURL = "https://cluster/1.1/foo/"; + let engine = await makeSteamEngine(); + try { + Assert.equal(engine.storageURL, "https://cluster/1.1/foo/storage/"); + Assert.equal(engine.engineURL, "https://cluster/1.1/foo/storage/steam"); + Assert.equal(engine.metaURL, "https://cluster/1.1/foo/storage/meta/global"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_syncID() { + _("SyncEngine.syncID corresponds to preference"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal( + Svc.PrefBranch.getPrefType("steam.syncID"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.equal(await engine.getSyncID(), ""); + + // Performing the first get on the attribute will generate a new GUID. + Assert.equal(await engine.resetLocalSyncID(), "fake-guid-00"); + Assert.equal(Svc.PrefBranch.getStringPref("steam.syncID"), "fake-guid-00"); + + Svc.PrefBranch.setStringPref("steam.syncID", Utils.makeGUID()); + Assert.equal(Svc.PrefBranch.getStringPref("steam.syncID"), "fake-guid-01"); + Assert.equal(await engine.getSyncID(), "fake-guid-01"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_lastSync() { + _("SyncEngine.lastSync corresponds to preferences"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal( + Svc.PrefBranch.getPrefType("steam.lastSync"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.equal(await engine.getLastSync(), 0); + + // Floats are properly stored as floats and synced with the preference + await engine.setLastSync(123.45); + Assert.equal(await engine.getLastSync(), 123.45); + Assert.equal(Svc.PrefBranch.getStringPref("steam.lastSync"), "123.45"); + + // Integer is properly stored + await engine.setLastSync(67890); + Assert.equal(await engine.getLastSync(), 67890); + Assert.equal(Svc.PrefBranch.getStringPref("steam.lastSync"), "67890"); + + // resetLastSync() resets the value (and preference) to 0 + await engine.resetLastSync(); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(Svc.PrefBranch.getStringPref("steam.lastSync"), "0"); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_toFetch() { + _("SyncEngine.toFetch corresponds to file on disk"); + await SyncTestingInfrastructure(server); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(3), + setup(engine) { + // Ensure pristine environment + Assert.equal(engine.toFetch.size, 0); + + // Write file to disk + engine.toFetch = this.toFetch; + Assert.equal(engine.toFetch, this.toFetch); + }, + check(engine) { + // toFetch is written asynchronously + assertSetsEqual(engine.toFetch, this.toFetch); + }, + }); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(4), + toFetch2: guidSetOfSize(5), + setup(engine) { + // Make sure it work for consecutive writes before the callback is executed. + engine.toFetch = this.toFetch; + Assert.equal(engine.toFetch, this.toFetch); + + engine.toFetch = this.toFetch2; + Assert.equal(engine.toFetch, this.toFetch2); + }, + check(engine) { + assertSetsEqual(engine.toFetch, this.toFetch2); + }, + }); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(2), + async beforeCheck() { + let toFetchPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "toFetch", + "steam.json" + ); + await IOUtils.writeJSON(toFetchPath, this.toFetch, { + tmpPath: toFetchPath + ".tmp", + }); + }, + check(engine) { + // Read file from disk + assertSetsEqual(engine.toFetch, this.toFetch); + }, + }); +}); + +add_task(async function test_previousFailed() { + _("SyncEngine.previousFailed corresponds to file on disk"); + await SyncTestingInfrastructure(server); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(3), + setup(engine) { + // Ensure pristine environment + Assert.equal(engine.previousFailed.size, 0); + + // Write file to disk + engine.previousFailed = this.previousFailed; + Assert.equal(engine.previousFailed, this.previousFailed); + }, + check(engine) { + // previousFailed is written asynchronously + assertSetsEqual(engine.previousFailed, this.previousFailed); + }, + }); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(4), + previousFailed2: guidSetOfSize(5), + setup(engine) { + // Make sure it work for consecutive writes before the callback is executed. + engine.previousFailed = this.previousFailed; + Assert.equal(engine.previousFailed, this.previousFailed); + + engine.previousFailed = this.previousFailed2; + Assert.equal(engine.previousFailed, this.previousFailed2); + }, + check(engine) { + assertSetsEqual(engine.previousFailed, this.previousFailed2); + }, + }); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(2), + async beforeCheck() { + let previousFailedPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "failed", + "steam.json" + ); + await IOUtils.writeJSON(previousFailedPath, this.previousFailed, { + tmpPath: previousFailedPath + ".tmp", + }); + }, + check(engine) { + // Read file from disk + assertSetsEqual(engine.previousFailed, this.previousFailed); + }, + }); +}); + +add_task(async function test_resetClient() { + _("SyncEngine.resetClient resets lastSync and toFetch"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal( + Svc.PrefBranch.getPrefType("steam.lastSync"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.equal(engine.toFetch.size, 0); + + await engine.setLastSync(123.45); + engine.toFetch = guidSetOfSize(4); + engine.previousFailed = guidSetOfSize(3); + + await engine.resetClient(); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + } finally { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function test_wipeServer() { + _("SyncEngine.wipeServer deletes server data and resets the client."); + let engine = await makeSteamEngine(); + + const PAYLOAD = 42; + let steamCollection = new ServerWBO("steam", PAYLOAD); + let steamServer = httpd_setup({ + "/1.1/foo/storage/steam": steamCollection.handler(), + }); + await SyncTestingInfrastructure(steamServer); + do_test_pending(); + + try { + // Some data to reset. + await engine.setLastSync(123.45); + engine.toFetch = guidSetOfSize(3); + + _("Wipe server data and reset client."); + await engine.wipeServer(); + Assert.equal(steamCollection.payload, undefined); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + } finally { + steamServer.stop(do_test_finished); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + } +}); + +add_task(async function finish() { + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_syncengine_sync.js b/services/sync/tests/unit/test_syncengine_sync.js new file mode 100644 index 0000000000..efd061d5bc --- /dev/null +++ b/services/sync/tests/unit/test_syncengine_sync.js @@ -0,0 +1,1781 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +function makeRotaryEngine() { + return new RotaryEngine(Service); +} + +async function clean(engine) { + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + Svc.PrefBranch.setStringPref("log.logger.engine.rotary", "Trace"); + Service.recordManager.clearCache(); + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +} + +async function cleanAndGo(engine, server) { + await clean(engine); + await promiseStopServer(server); +} + +async function promiseClean(engine, server) { + await clean(engine); + await promiseStopServer(server); +} + +async function createServerAndConfigureClient() { + let engine = new RotaryEngine(Service); + let syncID = await engine.resetLocalSyncID(); + + let contents = { + meta: { + global: { engines: { rotary: { version: engine.version, syncID } } }, + }, + crypto: {}, + rotary: {}, + }; + + const USER = "foo"; + let server = new SyncServer(); + server.registerUser(USER, "password"); + server.createContents(USER, contents); + server.start(); + + await SyncTestingInfrastructure(server, USER); + Service._updateCachedURLs(); + + return [engine, server, USER]; +} + +/* + * Tests + * + * SyncEngine._sync() is divided into four rather independent steps: + * + * - _syncStartup() + * - _processIncoming() + * - _uploadOutgoing() + * - _syncFinish() + * + * In the spirit of unit testing, these are tested individually for + * different scenarios below. + */ + +add_task(async function setup() { + await generateNewKeys(Service.collectionKeys); + Svc.PrefBranch.setStringPref("log.logger.engine.rotary", "Trace"); +}); + +add_task(async function test_syncStartup_emptyOrOutdatedGlobalsResetsSync() { + _( + "SyncEngine._syncStartup resets sync and wipes server data if there's no or an outdated global record" + ); + + // Some server side data that's going to be wiped + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { rekolok: "Rekonstruktionslokomotive" }; + try { + // Confirm initial environment + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.rekolok, undefined); + let metaGlobal = await Service.recordManager.get(engine.metaURL); + Assert.equal(metaGlobal.payload.engines, undefined); + Assert.ok(!!collection.payload("flying")); + Assert.ok(!!collection.payload("scotsman")); + + await engine.setLastSync(Date.now() / 1000); + + // Trying to prompt a wipe -- we no longer track CryptoMeta per engine, + // so it has nothing to check. + await engine._syncStartup(); + + // The meta/global WBO has been filled with data about the engine + let engineData = metaGlobal.payload.engines.rotary; + Assert.equal(engineData.version, engine.version); + Assert.equal(engineData.syncID, await engine.getSyncID()); + + // Sync was reset and server data was wiped + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncStartup_serverHasNewerVersion() { + _("SyncEngine._syncStartup "); + + let global = new ServerWBO("global", { + engines: { rotary: { version: 23456 } }, + }); + let server = httpd_setup({ + "/1.1/foo/storage/meta/global": global.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // The server has a newer version of the data and our engine can + // handle. That should give us an exception. + let error; + try { + await engine._syncStartup(); + } catch (ex) { + error = ex; + } + Assert.equal(error.failureCode, VERSION_OUT_OF_DATE); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncStartup_syncIDMismatchResetsClient() { + _("SyncEngine._syncStartup resets sync if syncIDs don't match"); + + let server = sync_httpd_setup({}); + + await SyncTestingInfrastructure(server); + + // global record with a different syncID than our engine has + let engine = makeRotaryEngine(); + let global = new ServerWBO("global", { + engines: { rotary: { version: engine.version, syncID: "foobar" } }, + }); + server.registerPathHandler("/1.1/foo/storage/meta/global", global.handler()); + + try { + // Confirm initial environment + Assert.equal(await engine.getSyncID(), ""); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.rekolok, undefined); + + await engine.setLastSync(Date.now() / 1000); + await engine._syncStartup(); + + // The engine has assumed the server's syncID + Assert.equal(await engine.getSyncID(), "foobar"); + + // Sync was reset + Assert.equal(await engine.getLastSync(), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_emptyServer() { + _("SyncEngine._processIncoming working with an empty server backend"); + + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // Merely ensure that this code path is run without any errors + await engine._processIncoming(); + Assert.equal(await engine.getLastSync(), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_createFromServer() { + _("SyncEngine._processIncoming creates new records from server data"); + + // Some server records that will be downloaded + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + + // Two pathological cases involving relative URIs gone wrong. + let pathologicalPayload = encryptPayload({ + id: "../pathological", + denomination: "Pathological Case", + }); + collection.insert("../pathological", pathologicalPayload); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.lastModified, null); + Assert.equal(engine._store.items.flying, undefined); + Assert.equal(engine._store.items.scotsman, undefined); + Assert.equal(engine._store.items["../pathological"], undefined); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Timestamps of last sync and last server modification are set. + Assert.ok((await engine.getLastSync()) > 0); + Assert.ok(engine.lastModified > 0); + + // Local records have been created from the server data. + Assert.equal(engine._store.items.flying, "LNER Class A3 4472"); + Assert.equal(engine._store.items.scotsman, "Flying Scotsman"); + Assert.equal(engine._store.items["../pathological"], "Pathological Case"); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_reconcile() { + _("SyncEngine._processIncoming updates local records"); + + let collection = new ServerCollection(); + + // This server record is newer than the corresponding client one, + // so it'll update its data. + collection.insert( + "newrecord", + encryptPayload({ id: "newrecord", denomination: "New stuff..." }) + ); + + // This server record is newer than the corresponding client one, + // so it'll update its data. + collection.insert( + "newerserver", + encryptPayload({ id: "newerserver", denomination: "New data!" }) + ); + + // This server record is 2 mins older than the client counterpart + // but identical to it, so we're expecting the client record's + // changedID to be reset. + collection.insert( + "olderidentical", + encryptPayload({ + id: "olderidentical", + denomination: "Older but identical", + }) + ); + collection._wbos.olderidentical.modified -= 120; + + // This item simply has different data than the corresponding client + // record (which is unmodified), so it will update the client as well + collection.insert( + "updateclient", + encryptPayload({ id: "updateclient", denomination: "Get this!" }) + ); + + // This is a dupe of 'original'. + collection.insert( + "duplication", + encryptPayload({ id: "duplication", denomination: "Original Entry" }) + ); + + // This record is marked as deleted, so we're expecting the client + // record to be removed. + collection.insert( + "nukeme", + encryptPayload({ id: "nukeme", denomination: "Nuke me!", deleted: true }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { + newerserver: "New data, but not as new as server!", + olderidentical: "Older but identical", + updateclient: "Got data?", + original: "Original Entry", + long_original: "Long Original Entry", + nukeme: "Nuke me!", + }; + // Make this record 1 min old, thus older than the one on the server + await engine._tracker.addChangedID("newerserver", Date.now() / 1000 - 60); + // This record has been changed 2 mins later than the one on the server + await engine._tracker.addChangedID("olderidentical", Date.now() / 1000); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(engine._store.items.newrecord, undefined); + Assert.equal( + engine._store.items.newerserver, + "New data, but not as new as server!" + ); + Assert.equal(engine._store.items.olderidentical, "Older but identical"); + Assert.equal(engine._store.items.updateclient, "Got data?"); + Assert.equal(engine._store.items.nukeme, "Nuke me!"); + let changes = await engine._tracker.getChangedIDs(); + Assert.ok(changes.olderidentical > 0); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Timestamps of last sync and last server modification are set. + Assert.ok((await engine.getLastSync()) > 0); + Assert.ok(engine.lastModified > 0); + + // The new record is created. + Assert.equal(engine._store.items.newrecord, "New stuff..."); + + // The 'newerserver' record is updated since the server data is newer. + Assert.equal(engine._store.items.newerserver, "New data!"); + + // The data for 'olderidentical' is identical on the server, so + // it's no longer marked as changed anymore. + Assert.equal(engine._store.items.olderidentical, "Older but identical"); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.olderidentical, undefined); + + // Updated with server data. + Assert.equal(engine._store.items.updateclient, "Get this!"); + + // The incoming ID is preferred. + Assert.equal(engine._store.items.original, undefined); + Assert.equal(engine._store.items.duplication, "Original Entry"); + Assert.notEqual(engine._delete.ids.indexOf("original"), -1); + + // The 'nukeme' record marked as deleted is removed. + Assert.equal(engine._store.items.nukeme, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_reconcile_local_deleted() { + _("Ensure local, duplicate ID is deleted on server."); + + // When a duplicate is resolved, the local ID (which is never taken) should + // be deleted on the server. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + record = encryptPayload({ id: "DUPE_LOCAL", denomination: "local" }); + wbo = new ServerWBO("DUPE_LOCAL", record, now - 1); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + await engine._sync(); + + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + Assert.notEqual(undefined, collection.wbo("DUPE_INCOMING")); + + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_reconcile_equivalent() { + _("Ensure proper handling of incoming records that match local."); + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ id: "entry", denomination: "denomination" }); + let wbo = new ServerWBO("entry", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + engine._store.items = { entry: "denomination" }; + Assert.ok(await engine._store.itemExists("entry")); + + await engine._sync(); + + do_check_attribute_count(engine._store.items, 1); + + await cleanAndGo(engine, server); +}); + +add_task( + async function test_processIncoming_reconcile_locally_deleted_dupe_new() { + _( + "Ensure locally deleted duplicate record newer than incoming is handled." + ); + + // This is a somewhat complicated test. It ensures that if a client receives + // a modified record for an item that is deleted locally but with a different + // ID that the incoming record is ignored. This is a corner case for record + // handling, but it needs to be supported. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + // Simulate a locally-deleted item. + engine._store.items = {}; + await engine._tracker.addChangedID("DUPE_LOCAL", now + 3); + Assert.equal(false, await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal(false, await engine._store.itemExists("DUPE_INCOMING")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // After the sync, the server's payload for the original ID should be marked + // as deleted. + do_check_empty(engine._store.items); + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(null, wbo); + let payload = wbo.getCleartext(); + Assert.ok(payload.deleted); + + await cleanAndGo(engine, server); + } +); + +add_task( + async function test_processIncoming_reconcile_locally_deleted_dupe_old() { + _( + "Ensure locally deleted duplicate record older than incoming is restored." + ); + + // This is similar to the above test except it tests the condition where the + // incoming record is newer than the local deletion, therefore overriding it. + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + // Simulate a locally-deleted item. + engine._store.items = {}; + await engine._tracker.addChangedID("DUPE_LOCAL", now + 1); + Assert.equal(false, await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal(false, await engine._store.itemExists("DUPE_INCOMING")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + await engine._sync(); + + // Since the remote change is newer, the incoming item should exist locally. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + Assert.equal("incoming", engine._store.items.DUPE_INCOMING); + + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + let payload = wbo.getCleartext(); + Assert.equal("incoming", payload.denomination); + + await cleanAndGo(engine, server); + } +); + +add_task(async function test_processIncoming_reconcile_changed_dupe() { + _("Ensure that locally changed duplicate record is handled properly."); + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + // The local record is newer than the incoming one, so it should be retained. + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + await engine._tracker.addChangedID("DUPE_LOCAL", now + 3); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // The ID should have been changed to incoming. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + // On the server, the local ID should be deleted and the incoming ID should + // have its payload set to what was in the local record. + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(undefined, wbo); + let payload = wbo.getCleartext(); + Assert.equal("local", payload.denomination); + + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_reconcile_changed_dupe_new() { + _("Ensure locally changed duplicate record older than incoming is ignored."); + + // This test is similar to the above except the incoming record is younger + // than the local record. The incoming record should be authoritative. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + await engine._tracker.addChangedID("DUPE_LOCAL", now + 1); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // The ID should have been changed to incoming. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + // On the server, the local ID should be deleted and the incoming ID should + // have its payload retained. + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(undefined, wbo); + let payload = wbo.getCleartext(); + Assert.equal("incoming", payload.denomination); + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_resume_toFetch() { + _( + "toFetch and previousFailed items left over from previous syncs are fetched on the next sync, along with new items." + ); + + const LASTSYNC = Date.now() / 1000; + + // Server records that will be downloaded + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection.insert( + "rekolok", + encryptPayload({ id: "rekolok", denomination: "Rekonstruktionslokomotive" }) + ); + for (let i = 0; i < 3; i++) { + let id = "failed" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + let wbo = new ServerWBO(id, payload); + wbo.modified = LASTSYNC - 10; + collection.insertWBO(wbo); + } + + collection.wbo("flying").modified = collection.wbo("scotsman").modified = + LASTSYNC - 10; + collection._wbos.rekolok.modified = LASTSYNC + 10; + + // Time travel 10 seconds into the future but still download the above WBOs. + let engine = makeRotaryEngine(); + await engine.setLastSync(LASTSYNC); + engine.toFetch = new SerializableSet(["flying", "scotsman"]); + engine.previousFailed = new SerializableSet([ + "failed0", + "failed1", + "failed2", + ]); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment + Assert.equal(engine._store.items.flying, undefined); + Assert.equal(engine._store.items.scotsman, undefined); + Assert.equal(engine._store.items.rekolok, undefined); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Local records have been created from the server data. + Assert.equal(engine._store.items.flying, "LNER Class A3 4472"); + Assert.equal(engine._store.items.scotsman, "Flying Scotsman"); + Assert.equal(engine._store.items.rekolok, "Rekonstruktionslokomotive"); + Assert.equal(engine._store.items.failed0, "Record No. 0"); + Assert.equal(engine._store.items.failed1, "Record No. 1"); + Assert.equal(engine._store.items.failed2, "Record No. 2"); + Assert.equal(engine.previousFailed.size, 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_notify_count() { + _("Ensure that failed records are reported only once."); + + const NUMBER_OF_RECORDS = 15; + + // Engine that fails every 5 records. + let engine = makeRotaryEngine(); + engine._store._applyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = async function (records, countTelemetry) { + let sortedRecords = records.sort((a, b) => (a.id > b.id ? 1 : -1)); + let recordsToApply = [], + recordsToFail = []; + for (let i = 0; i < sortedRecords.length; i++) { + (i % 5 === 0 ? recordsToFail : recordsToApply).push(sortedRecords[i]); + } + recordsToFail.forEach(() => { + countTelemetry.addIncomingFailedReason("failed message"); + }); + await engine._store._applyIncomingBatch(recordsToApply, countTelemetry); + + return recordsToFail.map(record => record.id); + }; + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i.toString(10).padStart(2, "0"); + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + } + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment. + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + let called = 0; + let counts; + function onApplied(count) { + _("Called with " + JSON.stringify(counts)); + counts = count; + called++; + } + Svc.Obs.add("weave:engine:sync:applied", onApplied); + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + // Confirm failures. + do_check_attribute_count(engine._store.items, 12); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + ["record-no-00", "record-no-05", "record-no-10"].sort() + ); + + // There are newly failed records and they are reported. + Assert.equal(called, 1); + Assert.equal(counts.failed, 3); + Assert.equal(counts.failedReasons[0].count, 3); + Assert.equal(counts.failedReasons[0].name, "failed message"); + Assert.equal(counts.applied, 15); + Assert.equal(counts.newFailed, 3); + Assert.equal(counts.succeeded, 12); + + // Sync again, 1 of the failed items are the same, the rest didn't fail. + await engine._processIncoming(); + + // Confirming removed failures. + do_check_attribute_count(engine._store.items, 14); + // After failing twice the record that failed again [record-no-00] + // should NOT be stored to try again + Assert.deepEqual(Array.from(engine.previousFailed), []); + + Assert.equal(called, 2); + Assert.equal(counts.failed, 1); + Assert.equal(counts.failedReasons[0].count, 1); + Assert.equal(counts.failedReasons[0].name, "failed message"); + Assert.equal(counts.applied, 3); + Assert.equal(counts.newFailed, 0); + Assert.equal(counts.succeeded, 2); + + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_previousFailed() { + _("Ensure that failed records are retried."); + + const NUMBER_OF_RECORDS = 14; + + // Engine that alternates between failing and applying every 2 records. + let engine = makeRotaryEngine(); + engine._store._applyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = async function (records, countTelemetry) { + let sortedRecords = records.sort((a, b) => (a.id > b.id ? 1 : -1)); + let recordsToApply = [], + recordsToFail = []; + let chunks = Array.from(PlacesUtils.chunkArray(sortedRecords, 2)); + for (let i = 0; i < chunks.length; i++) { + (i % 2 === 0 ? recordsToFail : recordsToApply).push(...chunks[i]); + } + await engine._store._applyIncomingBatch(recordsToApply, countTelemetry); + return recordsToFail.map(record => record.id); + }; + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i.toString(10).padStart(2, "0"); + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + collection.insert(id, payload); + } + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment. + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + // Initial failed items in previousFailed to be reset. + let previousFailed = new SerializableSet([ + Utils.makeGUID(), + Utils.makeGUID(), + Utils.makeGUID(), + ]); + engine.previousFailed = previousFailed; + Assert.equal(engine.previousFailed, previousFailed); + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + // Expected result: 4 sync batches with 2 failures each => 8 failures + do_check_attribute_count(engine._store.items, 6); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + [ + "record-no-00", + "record-no-01", + "record-no-04", + "record-no-05", + "record-no-08", + "record-no-09", + "record-no-12", + "record-no-13", + ].sort() + ); + + // Sync again with the same failed items (records 0, 1, 8, 9). + await engine._processIncoming(); + + do_check_attribute_count(engine._store.items, 10); + // A second sync with the same failed items should NOT add the same items again. + // Items that did not fail a second time should no longer be in previousFailed. + Assert.deepEqual(Array.from(engine.previousFailed).sort(), []); + + // Refetched items that didn't fail the second time are in engine._store.items. + Assert.equal(engine._store.items["record-no-04"], "Record No. 4"); + Assert.equal(engine._store.items["record-no-05"], "Record No. 5"); + Assert.equal(engine._store.items["record-no-12"], "Record No. 12"); + Assert.equal(engine._store.items["record-no-13"], "Record No. 13"); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_failed_records() { + _( + "Ensure that failed records from _reconcile and applyIncomingBatch are refetched." + ); + + // Let's create three and a bit batches worth of server side records. + let APPLY_BATCH_SIZE = 50; + let collection = new ServerCollection(); + const NUMBER_OF_RECORDS = APPLY_BATCH_SIZE * 3 + 5; + for (let i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + let wbo = new ServerWBO(id, payload); + wbo.modified = Date.now() / 1000 + 60 * (i - APPLY_BATCH_SIZE * 3); + collection.insertWBO(wbo); + } + + // Engine that batches but likes to throw on a couple of records, + // two in each batch: the even ones fail in reconcile, the odd ones + // in applyIncoming. + const BOGUS_RECORDS = [ + "record-no-" + 42, + "record-no-" + 23, + "record-no-" + (42 + APPLY_BATCH_SIZE), + "record-no-" + (23 + APPLY_BATCH_SIZE), + "record-no-" + (42 + APPLY_BATCH_SIZE * 2), + "record-no-" + (23 + APPLY_BATCH_SIZE * 2), + "record-no-" + (2 + APPLY_BATCH_SIZE * 3), + "record-no-" + (1 + APPLY_BATCH_SIZE * 3), + ]; + let engine = makeRotaryEngine(); + + engine.__reconcile = engine._reconcile; + engine._reconcile = async function _reconcile(record) { + if (BOGUS_RECORDS.indexOf(record.id) % 2 == 0) { + throw new Error("I don't like this record! Baaaaaah!"); + } + return this.__reconcile.apply(this, arguments); + }; + engine._store._applyIncoming = engine._store.applyIncoming; + engine._store.applyIncoming = async function (record) { + if (BOGUS_RECORDS.indexOf(record.id) % 2 == 1) { + throw new Error("I don't like this record! Baaaaaah!"); + } + return this._applyIncoming.apply(this, arguments); + }; + + // Keep track of requests made of a collection. + let count = 0; + let uris = []; + function recording_handler(recordedCollection) { + let h = recordedCollection.handler(); + return function (req, res) { + ++count; + uris.push(req.path + "?" + req.queryString); + return h(req, res); + }; + } + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": recording_handler(collection), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + let observerSubject; + let observerData; + Svc.Obs.add("weave:engine:sync:applied", function onApplied(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + observerSubject = subject; + observerData = data; + }); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Ensure that all records but the bogus 4 have been applied. + do_check_attribute_count( + engine._store.items, + NUMBER_OF_RECORDS - BOGUS_RECORDS.length + ); + + // Ensure that the bogus records will be fetched again on the next sync. + Assert.equal(engine.previousFailed.size, BOGUS_RECORDS.length); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + BOGUS_RECORDS.sort() + ); + + // Ensure the observer was notified + Assert.equal(observerData, engine.name); + Assert.equal(observerSubject.failed, BOGUS_RECORDS.length); + Assert.equal(observerSubject.newFailed, BOGUS_RECORDS.length); + + // Testing batching of failed item fetches. + // Try to sync again. Ensure that we split the request into chunks to avoid + // URI length limitations. + async function batchDownload(batchSize) { + count = 0; + uris = []; + engine.guidFetchBatchSize = batchSize; + await engine._processIncoming(); + _("Tried again. Requests: " + count + "; URIs: " + JSON.stringify(uris)); + return count; + } + + // There are 8 bad records, so this needs 3 fetches. + _("Test batching with ID batch size 3, normal mobile batch size."); + Assert.equal(await batchDownload(3), 3); + + // Since there the previous batch failed again, there should be + // no more records to fetch + _("Test that the second time a record failed to sync, gets ignored"); + Assert.equal(await batchDownload(BOGUS_RECORDS.length), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_decrypt_failed() { + _("Ensure that records failing to decrypt are either replaced or refetched."); + + // Some good and some bogus records. One doesn't contain valid JSON, + // the other will throw during decrypt. + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection._wbos.nojson = new ServerWBO("nojson", "This is invalid JSON"); + collection._wbos.nojson2 = new ServerWBO("nojson2", "This is invalid JSON"); + collection._wbos.scotsman = new ServerWBO( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection._wbos.nodecrypt = new ServerWBO("nodecrypt", "Decrypt this!"); + collection._wbos.nodecrypt2 = new ServerWBO("nodecrypt2", "Decrypt this!"); + + // Patch the fake crypto service to throw on the record above. + Weave.Crypto._decrypt = Weave.Crypto.decrypt; + Weave.Crypto.decrypt = function (ciphertext) { + if (ciphertext == "Decrypt this!") { + throw new Error( + "Derp! Cipher finalized failed. Im ur crypto destroyin ur recordz." + ); + } + return this._decrypt.apply(this, arguments); + }; + + // Some broken records also exist locally. + let engine = makeRotaryEngine(); + engine.enabled = true; + engine._store.items = { nojson: "Valid JSON", nodecrypt: "Valid ciphertext" }; + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial state + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + + let observerSubject; + let observerData; + Svc.Obs.add("weave:engine:sync:applied", function onApplied(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + observerSubject = subject; + observerData = data; + }); + + await engine.setLastSync(collection.wbo("nojson").modified - 1); + let ping = await sync_engine_and_validate_telem(engine, true); + Assert.equal(ping.engines[0].incoming.applied, 2); + Assert.equal(ping.engines[0].incoming.failed, 4); + console.log("incoming telem: ", ping.engines[0].incoming); + Assert.equal( + ping.engines[0].incoming.failedReasons[0].name, + "No ciphertext: nothing to decrypt?" + ); + // There should be 4 of the same error + Assert.equal(ping.engines[0].incoming.failedReasons[0].count, 4); + + Assert.equal(engine.previousFailed.size, 4); + Assert.ok(engine.previousFailed.has("nojson")); + Assert.ok(engine.previousFailed.has("nojson2")); + Assert.ok(engine.previousFailed.has("nodecrypt")); + Assert.ok(engine.previousFailed.has("nodecrypt2")); + + // Ensure the observer was notified + Assert.equal(observerData, engine.name); + Assert.equal(observerSubject.applied, 2); + Assert.equal(observerSubject.failed, 4); + Assert.equal(observerSubject.failedReasons[0].count, 4); + } finally { + await promiseClean(engine, server); + } +}); + +add_task(async function test_uploadOutgoing_toEmptyServer() { + _("SyncEngine._uploadOutgoing uploads new records to server"); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + // Mark one of these records as changed + await engine._tracker.addChangedID("scotsman", 0); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + + await engine._syncStartup(); + await engine._uploadOutgoing(); + + // Ensure the marked record ('scotsman') has been uploaded and is + // no longer marked. + Assert.equal(collection.payload("flying"), undefined); + Assert.ok(!!collection.payload("scotsman")); + Assert.equal(collection.cleartext("scotsman").id, "scotsman"); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.scotsman, undefined); + + // The 'flying' record wasn't marked so it wasn't uploaded + Assert.equal(collection.payload("flying"), undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +async function test_uploadOutgoing_max_record_payload_bytes( + allowSkippedRecord +) { + _( + "SyncEngine._uploadOutgoing throws when payload is bigger than max_record_payload_bytes" + ); + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = allowSkippedRecord; + engine._store.items = { flying: "a".repeat(1024 * 1024), scotsman: "abcd" }; + + await engine._tracker.addChangedID("flying", 1000); + await engine._tracker.addChangedID("scotsman", 1000); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(1); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + + await engine._syncStartup(); + await engine._uploadOutgoing(); + + if (!allowSkippedRecord) { + do_throw("should not get here"); + } + + await engine.trackRemainingChanges(); + + // Check we uploaded the other record to the server + Assert.ok(collection.payload("scotsman")); + // And that we won't try to upload the huge record next time. + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + } catch (e) { + if (allowSkippedRecord) { + do_throw("should not get here"); + } + + await engine.trackRemainingChanges(); + + // Check that we will try to upload the huge record next time + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, 1000); + } finally { + // Check we didn't upload the oversized record to the server + Assert.equal(collection.payload("flying"), undefined); + await cleanAndGo(engine, server); + } +} + +add_task( + async function test_uploadOutgoing_max_record_payload_bytes_disallowSkippedRecords() { + return test_uploadOutgoing_max_record_payload_bytes(false); + } +); + +add_task( + async function test_uploadOutgoing_max_record_payload_bytes_allowSkippedRecords() { + return test_uploadOutgoing_max_record_payload_bytes(true); + } +); + +add_task(async function test_uploadOutgoing_failed() { + _( + "SyncEngine._uploadOutgoing doesn't clear the tracker of objects that failed to upload." + ); + + let collection = new ServerCollection(); + // We only define the "flying" WBO on the server, not the "scotsman" + // and "peppercorn" ones. + collection._wbos.flying = new ServerWBO("flying"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + peppercorn: "Peppercorn Class", + }; + // Mark these records as changed + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + const PEPPERCORN_CHANGED = 34567; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + await engine._tracker.addChangedID("peppercorn", PEPPERCORN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + let changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, FLYING_CHANGED); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + Assert.equal(changes.peppercorn, PEPPERCORN_CHANGED); + + engine.enabled = true; + await sync_engine_and_validate_telem(engine, true); + + // Ensure the 'flying' record has been uploaded and is no longer marked. + Assert.ok(!!collection.payload("flying")); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + + // The 'scotsman' and 'peppercorn' records couldn't be uploaded so + // they weren't cleared from the tracker. + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + Assert.equal(changes.peppercorn, PEPPERCORN_CHANGED); + } finally { + await promiseClean(engine, server); + } +}); + +async function createRecordFailTelemetry(allowSkippedRecord) { + Services.prefs.setStringPref("services.sync.username", "foo"); + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = allowSkippedRecord; + let oldCreateRecord = engine._store.createRecord; + engine._store.createRecord = async (id, col) => { + if (id != "flying") { + throw new Error("oops"); + } + return oldCreateRecord.call(engine._store, id, col); + }; + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + // Mark these records as changed + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let ping; + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + let changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, FLYING_CHANGED); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + + engine.enabled = true; + ping = await sync_engine_and_validate_telem(engine, true, onErrorPing => { + ping = onErrorPing; + }); + + if (!allowSkippedRecord) { + do_throw("should not get here"); + } + + // Ensure the 'flying' record has been uploaded and is no longer marked. + Assert.ok(!!collection.payload("flying")); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + } catch (err) { + if (allowSkippedRecord) { + do_throw("should not get here"); + } + + // Ensure the 'flying' record has not been uploaded and is still marked + Assert.ok(!collection.payload("flying")); + const changes = await engine._tracker.getChangedIDs(); + Assert.ok(changes.flying); + } finally { + // We reported in telemetry that we failed a record + Assert.equal(ping.engines[0].outgoing[0].failed, 1); + Assert.equal(ping.engines[0].outgoing[0].failedReasons[0].name, "oops"); + + // In any case, the 'scotsman' record couldn't be created so it wasn't + // uploaded nor it was not cleared from the tracker. + Assert.ok(!collection.payload("scotsman")); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + + engine._store.createRecord = oldCreateRecord; + await promiseClean(engine, server); + } +} + +add_task( + async function test_uploadOutgoing_createRecord_throws_reported_telemetry() { + _( + "SyncEngine._uploadOutgoing reports a failed record to telemetry if createRecord throws" + ); + await createRecordFailTelemetry(true); + } +); + +add_task( + async function test_uploadOutgoing_createRecord_throws_dontAllowSkipRecord() { + _( + "SyncEngine._uploadOutgoing will throw if createRecord throws and allowSkipRecord is set to false" + ); + await createRecordFailTelemetry(false); + } +); + +add_task(async function test_uploadOutgoing_largeRecords() { + _( + "SyncEngine._uploadOutgoing throws on records larger than the max record payload size" + ); + + let collection = new ServerCollection(); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = false; + engine._store.items["large-item"] = "Y".repeat( + Service.getMaxRecordPayloadSize() * 2 + ); + await engine._tracker.addChangedID("large-item", 0); + collection.insert("large-item"); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + try { + await engine._syncStartup(); + let error = null; + try { + await engine._uploadOutgoing(); + } catch (e) { + error = e; + } + ok(!!error); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncFinish_deleteByIds() { + _( + "SyncEngine._syncFinish deletes server records slated for deletion (list of record IDs)." + ); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection._wbos.scotsman = new ServerWBO( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection._wbos.rekolok = new ServerWBO( + "rekolok", + encryptPayload({ id: "rekolok", denomination: "Rekonstruktionslokomotive" }) + ); + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + engine._delete = { ids: ["flying", "rekolok"] }; + await engine._syncFinish(); + + // The 'flying' and 'rekolok' records were deleted while the + // 'scotsman' one wasn't. + Assert.equal(collection.payload("flying"), undefined); + Assert.ok(!!collection.payload("scotsman")); + Assert.equal(collection.payload("rekolok"), undefined); + + // The deletion todo list has been reset. + Assert.equal(engine._delete.ids, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncFinish_deleteLotsInBatches() { + _( + "SyncEngine._syncFinish deletes server records in batches of 100 (list of record IDs)." + ); + + let collection = new ServerCollection(); + + // Let's count how many times the client does a DELETE request to the server + var noOfUploads = 0; + collection.delete = (function (orig) { + return function () { + noOfUploads++; + return orig.apply(this, arguments); + }; + })(collection.delete); + + // Create a bunch of records on the server + let now = Date.now(); + for (var i = 0; i < 234; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + let wbo = new ServerWBO(id, payload); + wbo.modified = now / 1000 - 60 * (i + 110); + collection.insertWBO(wbo); + } + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // Confirm initial environment + Assert.equal(noOfUploads, 0); + + // Declare what we want to have deleted: all records no. 100 and + // up and all records that are less than 200 mins old (which are + // records 0 thru 90). + engine._delete = { ids: [], newer: now / 1000 - 60 * 200.5 }; + for (i = 100; i < 234; i++) { + engine._delete.ids.push("record-no-" + i); + } + + await engine._syncFinish(); + + // Ensure that the appropriate server data has been wiped while + // preserving records 90 thru 200. + for (i = 0; i < 234; i++) { + let id = "record-no-" + i; + if (i <= 90 || i >= 100) { + Assert.equal(collection.payload(id), undefined); + } else { + Assert.ok(!!collection.payload(id)); + } + } + + // The deletion was done in batches + Assert.equal(noOfUploads, 2 + 1); + + // The deletion todo list has been reset. + Assert.equal(engine._delete.ids, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_sync_partialUpload() { + _("SyncEngine.sync() keeps changedIDs that couldn't be uploaded."); + + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + let oldServerConfiguration = Service.serverConfiguration; + Service.serverConfiguration = { + max_post_records: 100, + }; + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + + // Let the third upload fail completely + var noOfUploads = 0; + collection.post = (function (orig) { + return function () { + if (noOfUploads == 2) { + throw new Error("FAIL!"); + } + noOfUploads++; + return orig.apply(this, arguments); + }; + })(collection.post); + + // Create a bunch of records (and server side handlers) + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + engine._store.items[id] = "Record No. " + i; + await engine._tracker.addChangedID(id, i); + // Let two items in the first upload batch fail. + if (i != 23 && i != 42) { + collection.insert(id); + } + } + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + engine.enabled = true; + let error; + try { + await sync_engine_and_validate_telem(engine, true); + } catch (ex) { + error = ex; + } + + ok(!!error); + + const changes = await engine._tracker.getChangedIDs(); + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + // Ensure failed records are back in the tracker: + // * records no. 23 and 42 were rejected by the server, + // * records after the third batch and higher couldn't be uploaded because + // we failed hard on the 3rd upload. + if (i == 23 || i == 42 || i >= 200) { + Assert.equal(changes[id], i); + } else { + Assert.equal(false, id in changes); + } + } + } finally { + Service.serverConfiguration = oldServerConfiguration; + await promiseClean(engine, server); + } +}); + +add_task(async function test_canDecrypt_noCryptoKeys() { + _( + "SyncEngine.canDecrypt returns false if the engine fails to decrypt items on the server, e.g. due to a missing crypto key collection." + ); + + // Wipe collection keys so we can test the desired scenario. + Service.collectionKeys.clear(); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + let engine = makeRotaryEngine(); + try { + Assert.equal(false, await engine.canDecrypt()); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_canDecrypt_true() { + _( + "SyncEngine.canDecrypt returns true if the engine can decrypt the items on the server." + ); + + await generateNewKeys(Service.collectionKeys); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + let engine = makeRotaryEngine(); + try { + Assert.ok(await engine.canDecrypt()); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncapplied_observer() { + const NUMBER_OF_RECORDS = 10; + + let engine = makeRotaryEngine(); + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + } + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let numApplyCalls = 0; + let engine_name; + let count; + function onApplied(subject, data) { + numApplyCalls++; + engine_name = data; + count = subject; + } + + Svc.Obs.add("weave:engine:sync:applied", onApplied); + + try { + Service.scheduler.hasIncomingItems = false; + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + do_check_attribute_count(engine._store.items, 10); + + Assert.equal(numApplyCalls, 1); + Assert.equal(engine_name, "rotary"); + Assert.equal(count.applied, 10); + + Assert.ok(Service.scheduler.hasIncomingItems); + } finally { + await cleanAndGo(engine, server); + Service.scheduler.hasIncomingItems = false; + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + } +}); diff --git a/services/sync/tests/unit/test_syncscheduler.js b/services/sync/tests/unit/test_syncscheduler.js new file mode 100644 index 0000000000..98b7937da3 --- /dev/null +++ b/services/sync/tests/unit/test_syncscheduler.js @@ -0,0 +1,1195 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FxAccounts } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); +const { SyncScheduler } = ChromeUtils.importESModule( + "resource://services-sync/policies.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); +} +CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + throw this.exception; + }, +}; +Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + +var scheduler = new SyncScheduler(Service); +let clientsEngine; + +async function sync_httpd_setup() { + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + return httpd_setup({ + "/1.1/johndoe@mozilla.com/storage/meta/global": upd( + "meta", + global.handler() + ), + "/1.1/johndoe@mozilla.com/info/collections": collectionsHelper.handler, + "/1.1/johndoe@mozilla.com/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe@mozilla.com/storage/clients": upd( + "clients", + clientsColl.handler() + ), + }); +} + +async function setUp(server) { + await configureIdentity({ username: "johndoe@mozilla.com" }, server); + + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let result = ( + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)) + ).success; + return result; +} + +async function cleanUpAndGo(server) { + await Async.promiseYield(); + await clientsEngine._store.wipe(); + await Service.startOver(); + // Re-enable logging, which we just disabled. + syncTestLogging(); + if (server) { + await promiseStopServer(server); + } +} + +add_task(async function setup() { + await Service.promiseInitialized; + clientsEngine = Service.clientsEngine; + // Don't remove stale clients when syncing. This is a test-only workaround + // that lets us add clients directly to the store, without losing them on + // the next sync. + clientsEngine._removeRemoteClient = async id => {}; + await Service.engineManager.clear(); + + validate_all_future_pings(); + + scheduler.setDefaults(); + + await Service.engineManager.register(CatapultEngine); +}); + +add_test(function test_prefAttributes() { + _("Test various attributes corresponding to preferences."); + + const INTERVAL = 42 * 60 * 1000; // 42 minutes + const THRESHOLD = 3142; + const SCORE = 2718; + const TIMESTAMP1 = 1275493471649; + + _( + "The 'nextSync' attribute stores a millisecond timestamp rounded down to the nearest second." + ); + Assert.equal(scheduler.nextSync, 0); + scheduler.nextSync = TIMESTAMP1; + Assert.equal(scheduler.nextSync, Math.floor(TIMESTAMP1 / 1000) * 1000); + + _("'syncInterval' defaults to singleDeviceInterval."); + Assert.equal( + Svc.PrefBranch.getPrefType("syncInterval"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _("'syncInterval' corresponds to a preference setting."); + scheduler.syncInterval = INTERVAL; + Assert.equal(scheduler.syncInterval, INTERVAL); + Assert.equal(Svc.PrefBranch.getIntPref("syncInterval"), INTERVAL); + + _( + "'syncThreshold' corresponds to preference, defaults to SINGLE_USER_THRESHOLD" + ); + Assert.equal( + Svc.PrefBranch.getPrefType("syncThreshold"), + Ci.nsIPrefBranch.PREF_INVALID + ); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + scheduler.syncThreshold = THRESHOLD; + Assert.equal(scheduler.syncThreshold, THRESHOLD); + + _("'globalScore' corresponds to preference, defaults to zero."); + Assert.equal(Svc.PrefBranch.getIntPref("globalScore"), 0); + Assert.equal(scheduler.globalScore, 0); + scheduler.globalScore = SCORE; + Assert.equal(scheduler.globalScore, SCORE); + Assert.equal(Svc.PrefBranch.getIntPref("globalScore"), SCORE); + + _("Intervals correspond to default preferences."); + Assert.equal( + scheduler.singleDeviceInterval, + Svc.PrefBranch.getIntPref("scheduler.fxa.singleDeviceInterval") * 1000 + ); + Assert.equal( + scheduler.idleInterval, + Svc.PrefBranch.getIntPref("scheduler.idleInterval") * 1000 + ); + Assert.equal( + scheduler.activeInterval, + Svc.PrefBranch.getIntPref("scheduler.activeInterval") * 1000 + ); + Assert.equal( + scheduler.immediateInterval, + Svc.PrefBranch.getIntPref("scheduler.immediateInterval") * 1000 + ); + + _("Custom values for prefs will take effect after a restart."); + Svc.PrefBranch.setIntPref("scheduler.fxa.singleDeviceInterval", 420); + Svc.PrefBranch.setIntPref("scheduler.idleInterval", 230); + Svc.PrefBranch.setIntPref("scheduler.activeInterval", 180); + Svc.PrefBranch.setIntPref("scheduler.immediateInterval", 31415); + scheduler.setDefaults(); + Assert.equal(scheduler.idleInterval, 230000); + Assert.equal(scheduler.singleDeviceInterval, 420000); + Assert.equal(scheduler.activeInterval, 180000); + Assert.equal(scheduler.immediateInterval, 31415000); + + _("Custom values for interval prefs can't be less than 60 seconds."); + Svc.PrefBranch.setIntPref("scheduler.fxa.singleDeviceInterval", 42); + Svc.PrefBranch.setIntPref("scheduler.idleInterval", 50); + Svc.PrefBranch.setIntPref("scheduler.activeInterval", 50); + Svc.PrefBranch.setIntPref("scheduler.immediateInterval", 10); + scheduler.setDefaults(); + Assert.equal(scheduler.idleInterval, 60000); + Assert.equal(scheduler.singleDeviceInterval, 60000); + Assert.equal(scheduler.activeInterval, 60000); + Assert.equal(scheduler.immediateInterval, 60000); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + scheduler.setDefaults(); + run_next_test(); +}); + +add_task(async function test_sync_skipped_low_score_no_resync() { + enableValidationPrefs(); + let server = await sync_httpd_setup(); + + function SkipEngine() { + SyncEngine.call(this, "Skip", Service); + this.syncs = 0; + } + + SkipEngine.prototype = { + _sync() { + do_throw("Should have been skipped"); + }, + shouldSkipSync() { + return true; + }, + }; + Object.setPrototypeOf(SkipEngine.prototype, SyncEngine.prototype); + await Service.engineManager.register(SkipEngine); + + let engine = Service.engineManager.get("skip"); + engine.enabled = true; + engine._tracker._score = 30; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + let resyncDoneObserver = promiseOneObserver("weave:service:resyncs-finished"); + + let synced = false; + function onSyncStarted() { + Assert.ok(!synced, "Only should sync once"); + synced = true; + } + + await Service.sync(); + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Svc.Obs.add("weave:service:sync:start", onSyncStarted); + await resyncDoneObserver; + + Svc.Obs.remove("weave:service:sync:start", onSyncStarted); + engine._tracker._store = 0; + await cleanUpAndGo(server); +}); + +add_task(async function test_updateClientMode() { + _( + "Test updateClientMode adjusts scheduling attributes based on # of clients appropriately" + ); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Trigger a change in interval & threshold by noting there are multiple clients. + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.updateClientMode(); + + Assert.equal(scheduler.syncThreshold, MULTI_DEVICE_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Resets the number of clients to 0. + await clientsEngine.resetClient(); + Svc.PrefBranch.clearUserPref("clients.devices.mobile"); + scheduler.updateClientMode(); + + // Goes back to single user if # clients is 1. + Assert.equal(scheduler.numClients, 1); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + await cleanUpAndGo(); +}); + +add_task(async function test_masterpassword_locked_retry_interval() { + enableValidationPrefs(); + + _( + "Test Status.login = MASTER_PASSWORD_LOCKED results in reschedule at MASTER_PASSWORD interval" + ); + let loginFailed = false; + Svc.Obs.add("weave:service:login:error", function onLoginError() { + Svc.Obs.remove("weave:service:login:error", onLoginError); + loginFailed = true; + }); + + let rescheduleInterval = false; + + let oldScheduleAtInterval = SyncScheduler.prototype.scheduleAtInterval; + SyncScheduler.prototype.scheduleAtInterval = function (interval) { + rescheduleInterval = true; + Assert.equal(interval, MASTER_PASSWORD_LOCKED_RETRY_INTERVAL); + }; + + let oldVerifyLogin = Service.verifyLogin; + Service.verifyLogin = async function () { + Status.login = MASTER_PASSWORD_LOCKED; + return false; + }; + + let server = await sync_httpd_setup(); + await setUp(server); + + await Service.sync(); + + Assert.ok(loginFailed); + Assert.equal(Status.login, MASTER_PASSWORD_LOCKED); + Assert.ok(rescheduleInterval); + + Service.verifyLogin = oldVerifyLogin; + SyncScheduler.prototype.scheduleAtInterval = oldScheduleAtInterval; + + await cleanUpAndGo(server); +}); + +add_task(async function test_calculateBackoff() { + Assert.equal(Status.backoffInterval, 0); + + // Test no interval larger than the maximum backoff is used if + // Status.backoffInterval is smaller. + Status.backoffInterval = 5; + let backoffInterval = Utils.calculateBackoff( + 50, + MAXIMUM_BACKOFF_INTERVAL, + Status.backoffInterval + ); + + Assert.equal(backoffInterval, MAXIMUM_BACKOFF_INTERVAL); + + // Test Status.backoffInterval is used if it is + // larger than MAXIMUM_BACKOFF_INTERVAL. + Status.backoffInterval = MAXIMUM_BACKOFF_INTERVAL + 10; + backoffInterval = Utils.calculateBackoff( + 50, + MAXIMUM_BACKOFF_INTERVAL, + Status.backoffInterval + ); + + Assert.equal(backoffInterval, MAXIMUM_BACKOFF_INTERVAL + 10); + + await cleanUpAndGo(); +}); + +add_task(async function test_scheduleNextSync_nowOrPast() { + enableValidationPrefs(); + + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + + let server = await sync_httpd_setup(); + await setUp(server); + + // We're late for a sync... + scheduler.scheduleNextSync(-1); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_scheduleNextSync_future_noBackoff() { + enableValidationPrefs(); + + _( + "scheduleNextSync() uses the current syncInterval if no interval is provided." + ); + // Test backoffInterval is 0 as expected. + Assert.equal(Status.backoffInterval, 0); + + _("Test setting sync interval when nextSync == 0"); + scheduler.nextSync = 0; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= scheduler.syncInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.syncInterval); + + _("Test setting sync interval when nextSync != 0"); + scheduler.nextSync = Date.now() + scheduler.singleDeviceInterval; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= scheduler.syncInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.syncInterval); + + _( + "Scheduling requests for intervals larger than the current one will be ignored." + ); + // Request a sync at a longer interval. The sync that's already scheduled + // for sooner takes precedence. + let nextSync = scheduler.nextSync; + let timerDelay = scheduler.syncTimer.delay; + let requestedInterval = scheduler.syncInterval * 10; + scheduler.scheduleNextSync(requestedInterval); + Assert.equal(scheduler.nextSync, nextSync); + Assert.equal(scheduler.syncTimer.delay, timerDelay); + + // We can schedule anything we want if there isn't a sync scheduled. + scheduler.nextSync = 0; + scheduler.scheduleNextSync(requestedInterval); + Assert.ok(scheduler.nextSync <= Date.now() + requestedInterval); + Assert.equal(scheduler.syncTimer.delay, requestedInterval); + + // Request a sync at the smallest possible interval (0 triggers now). + scheduler.scheduleNextSync(1); + Assert.ok(scheduler.nextSync <= Date.now() + 1); + Assert.equal(scheduler.syncTimer.delay, 1); + + await cleanUpAndGo(); +}); + +add_task(async function test_scheduleNextSync_future_backoff() { + enableValidationPrefs(); + + _("scheduleNextSync() will honour backoff in all scheduling requests."); + // Let's take a backoff interval that's bigger than the default sync interval. + const BACKOFF = 7337; + Status.backoffInterval = scheduler.syncInterval + BACKOFF; + + _("Test setting sync interval when nextSync == 0"); + scheduler.nextSync = 0; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + _("Test setting sync interval when nextSync != 0"); + scheduler.nextSync = Date.now() + scheduler.singleDeviceInterval; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= Status.backoffInterval); + Assert.ok(scheduler.syncTimer.delay <= Status.backoffInterval); + + // Request a sync at a longer interval. The sync that's already scheduled + // for sooner takes precedence. + let nextSync = scheduler.nextSync; + let timerDelay = scheduler.syncTimer.delay; + let requestedInterval = scheduler.syncInterval * 10; + Assert.ok(requestedInterval > Status.backoffInterval); + scheduler.scheduleNextSync(requestedInterval); + Assert.equal(scheduler.nextSync, nextSync); + Assert.equal(scheduler.syncTimer.delay, timerDelay); + + // We can schedule anything we want if there isn't a sync scheduled. + scheduler.nextSync = 0; + scheduler.scheduleNextSync(requestedInterval); + Assert.ok(scheduler.nextSync <= Date.now() + requestedInterval); + Assert.equal(scheduler.syncTimer.delay, requestedInterval); + + // Request a sync at the smallest possible interval (0 triggers now). + scheduler.scheduleNextSync(1); + Assert.ok(scheduler.nextSync <= Date.now() + Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_handleSyncError() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Force sync to fail. + Svc.PrefBranch.setStringPref("firstSync", "notReady"); + + _("Ensure expected initial environment."); + Assert.equal(scheduler._syncErrors, 0); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(Status.backoffInterval, 0); + + // Trigger sync with an error several times & observe + // functionality of handleSyncError() + _("Test first error calls scheduleNextSync on default interval"); + await Service.sync(); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + Assert.equal(scheduler._syncErrors, 1); + Assert.ok(!Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Test second error still calls scheduleNextSync on default interval"); + await Service.sync(); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + Assert.equal(scheduler._syncErrors, 2); + Assert.ok(!Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Test third error sets Status.enforceBackoff and calls scheduleAtInterval"); + await Service.sync(); + let maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.equal(Status.backoffInterval, 0); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + Assert.equal(scheduler._syncErrors, 3); + Assert.ok(Status.enforceBackoff); + + // Status.enforceBackoff is false but there are still errors. + Status.resetBackoff(); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 3); + scheduler.syncTimer.clear(); + + _( + "Test fourth error still calls scheduleAtInterval even if enforceBackoff was reset" + ); + await Service.sync(); + maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + Assert.equal(scheduler._syncErrors, 4); + Assert.ok(Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Arrange for a successful sync to reset the scheduler error count"); + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + Svc.PrefBranch.setStringPref("firstSync", "wipeRemote"); + scheduler.scheduleNextSync(-1); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_client_sync_finish_updateClientMode() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults. + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.idle); + + // Trigger a change in interval & threshold by adding a client. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + Assert.equal(false, scheduler.numClients > 1); + scheduler.updateClientMode(); + await Service.sync(); + + Assert.equal(scheduler.syncThreshold, MULTI_DEVICE_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Resets the number of clients to 0. + await clientsEngine.resetClient(); + // Also re-init the server, or we suck our "foo" client back down. + await setUp(server); + + await Service.sync(); + + // Goes back to single user if # clients is 1. + Assert.equal(scheduler.numClients, 1); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + await cleanUpAndGo(server); +}); + +add_task(async function test_autoconnect_nextSync_past() { + enableValidationPrefs(); + + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + // nextSync will be 0 by default, so it's way in the past. + + let server = await sync_httpd_setup(); + await setUp(server); + + scheduler.autoConnect(); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_autoconnect_nextSync_future() { + enableValidationPrefs(); + + let previousSync = Date.now() + scheduler.syncInterval / 2; + scheduler.nextSync = previousSync; + // nextSync rounds to the nearest second. + let expectedSync = scheduler.nextSync; + let expectedInterval = expectedSync - Date.now() - 1000; + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + await configureIdentity({ username: "johndoe@mozilla.com" }); + scheduler.autoConnect(); + await promiseZeroTimer(); + + Assert.equal(scheduler.nextSync, expectedSync); + Assert.ok(scheduler.syncTimer.delay >= expectedInterval); + + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(); +}); + +add_task(async function test_autoconnect_mp_locked() { + let server = await sync_httpd_setup(); + await setUp(server); + + // Pretend user did not unlock master password. + let origLocked = Utils.mpLocked; + Utils.mpLocked = () => true; + + let origEnsureMPUnlocked = Utils.ensureMPUnlocked; + Utils.ensureMPUnlocked = () => { + _("Faking Master Password entry cancelation."); + return false; + }; + let origFxA = Service.identity._fxaService; + Service.identity._fxaService = new FxAccounts({ + currentAccountState: { + getUserAccountData(...args) { + return origFxA._internal.currentAccountState.getUserAccountData( + ...args + ); + }, + }, + keys: { + canGetKeyForScope() { + return false; + }, + }, + }); + // A locked master password will still trigger a sync, but then we'll hit + // MASTER_PASSWORD_LOCKED and hence MASTER_PASSWORD_LOCKED_RETRY_INTERVAL. + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + scheduler.autoConnect(); + await promiseObserved; + + await Async.promiseYield(); + + Assert.equal(Status.login, MASTER_PASSWORD_LOCKED); + + Utils.mpLocked = origLocked; + Utils.ensureMPUnlocked = origEnsureMPUnlocked; + Service.identity._fxaService = origFxA; + + await cleanUpAndGo(server); +}); + +add_task(async function test_no_autoconnect_during_wizard() { + let server = await sync_httpd_setup(); + await setUp(server); + + // Simulate the Sync setup wizard. + Svc.PrefBranch.setStringPref("firstSync", "notReady"); + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + scheduler.autoConnect(0); + await promiseZeroTimer(); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(server); +}); + +add_task(async function test_no_autoconnect_status_not_ok() { + let server = await sync_httpd_setup(); + Status.__authManager = Service.identity = new SyncAuthManager(); + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + scheduler.autoConnect(); + await promiseZeroTimer(); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + + Assert.equal(Status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Status.login, LOGIN_FAILED_NO_USERNAME); + + await cleanUpAndGo(server); +}); + +add_task(async function test_idle_adjustSyncInterval() { + // Confirm defaults. + Assert.equal(scheduler.idle, false); + + // Single device: nothing changes. + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.equal(scheduler.idle, true); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // Multiple devices: switch to idle interval. + scheduler.idle = false; + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.updateClientMode(); + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.equal(scheduler.idle, true); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_back_triggersSync() { + // Confirm defaults. + Assert.ok(!scheduler.idle); + Assert.equal(Status.backoffInterval, 0); + + // Set up: Define 2 clients and put the system in idle. + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.ok(scheduler.idle); + + // We don't actually expect the sync (or the login, for that matter) to + // succeed. We just want to ensure that it was attempted. + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Send an 'active' event to trigger sync soonish. + scheduler.observe( + null, + "active", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + await promiseObserved; + await cleanUpAndGo(); +}); + +add_task(async function test_active_triggersSync_observesBackoff() { + // Confirm defaults. + Assert.ok(!scheduler.idle); + + // Set up: Set backoff, define 2 clients and put the system in idle. + const BACKOFF = 7337; + Status.backoffInterval = scheduler.idleInterval + BACKOFF; + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.equal(scheduler.idle, true); + + function onLoginStart() { + do_throw("Shouldn't have kicked off a sync!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + let promiseTimer = promiseNamedTimer( + IDLE_OBSERVER_BACK_DELAY * 1.5, + {}, + "timer" + ); + + // Send an 'active' event to try to trigger sync soonish. + scheduler.observe( + null, + "active", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + await promiseTimer; + Svc.Obs.remove("weave:service:login:start", onLoginStart); + + Assert.ok(scheduler.nextSync <= Date.now() + Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_back_debouncing() { + _( + "Ensure spurious back-then-idle events, as observed on OS X, don't trigger a sync." + ); + + // Confirm defaults. + Assert.equal(scheduler.idle, false); + + // Set up: Define 2 clients and put the system in idle. + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + Assert.equal(scheduler.idle, true); + + function onLoginStart() { + do_throw("Shouldn't have kicked off a sync!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + // Create spurious back-then-idle events as observed on OS X: + scheduler.observe( + null, + "active", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + scheduler.observe( + null, + "idle", + Svc.PrefBranch.getIntPref("scheduler.idleTime") + ); + + await promiseNamedTimer(IDLE_OBSERVER_BACK_DELAY * 1.5, {}, "timer"); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(); +}); + +add_task(async function test_no_sync_node() { + enableValidationPrefs(); + + // Test when Status.sync == NO_SYNC_NODE_FOUND + // it is not overwritten on sync:finish + let server = await sync_httpd_setup(); + await setUp(server); + + let oldfc = Service.identity._findCluster; + Service.identity._findCluster = () => null; + Service.clusterURL = ""; + try { + await Service.sync(); + Assert.equal(Status.sync, NO_SYNC_NODE_FOUND); + Assert.equal(scheduler.syncTimer.delay, NO_SYNC_NODE_INTERVAL); + + await cleanUpAndGo(server); + } finally { + Service.identity._findCluster = oldfc; + } +}); + +add_task(async function test_sync_failed_partial_500s() { + enableValidationPrefs(); + + _("Test a 5xx status calls handleSyncError."); + scheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF; + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 500 }; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + let maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.equal(Status.backoffInterval, 0); + Assert.ok(Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 4); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_failed_partial_noresync() { + enableValidationPrefs(); + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = "Bad news"; + engine._tracker._score = MULTI_DEVICE_THRESHOLD + 1; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + let resyncDoneObserver = promiseOneObserver("weave:service:resyncs-finished"); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + function onSyncStarted() { + do_throw("Should not start resync when previous sync failed"); + } + + Svc.Obs.add("weave:service:sync:start", onSyncStarted); + await resyncDoneObserver; + + Svc.Obs.remove("weave:service:sync:start", onSyncStarted); + engine._tracker._store = 0; + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_failed_partial_400s() { + enableValidationPrefs(); + + _("Test a non-5xx status doesn't call handleSyncError."); + scheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF; + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 400 }; + + // Have multiple devices for an active interval. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + Assert.equal(Status.backoffInterval, 0); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 0); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.activeInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_X_Weave_Backoff() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Use an odd value on purpose so that it doesn't happen to coincide with one + // of the sync intervals. + const BACKOFF = 7337; + + // Extend info/collections so that we can put it into server maintenance mode. + const INFO_COLLECTIONS = "/1.1/johndoe@mozilla.com/info/collections"; + let infoColl = server._handler._overridePaths[INFO_COLLECTIONS]; + let serverBackoff = false; + function infoCollWithBackoff(request, response) { + if (serverBackoff) { + response.setHeader("X-Weave-Backoff", "" + BACKOFF); + } + infoColl(request, response); + } + server.registerPathHandler(INFO_COLLECTIONS, infoCollWithBackoff); + + // Pretend we have two clients so that the regular sync interval is + // sufficiently low. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + let rec = await clientsEngine._store.createRecord("foo", "clients"); + await rec.encrypt(Service.collectionKeys.keyForCollection("clients")); + await rec.upload(Service.resource(clientsEngine.engineURL + rec.id)); + + // Sync once to log in and get everything set up. Let's verify our initial + // values. + await Service.sync(); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.syncInterval); + // Sanity check that we picked the right value for BACKOFF: + Assert.ok(scheduler.syncInterval < BACKOFF * 1000); + + // Turn on server maintenance and sync again. + serverBackoff = true; + await Service.sync(); + + Assert.ok(Status.backoffInterval >= BACKOFF * 1000); + // Allowing 20 seconds worth of of leeway between when Status.minimumNextSync + // was set and when this line gets executed. + let minimumExpectedDelay = (BACKOFF - 20) * 1000; + Assert.ok(Status.minimumNextSync >= Date.now() + minimumExpectedDelay); + + // Verify that the next sync is actually going to wait that long. + Assert.ok(scheduler.nextSync >= Date.now() + minimumExpectedDelay); + Assert.ok(scheduler.syncTimer.delay >= minimumExpectedDelay); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_503_Retry_After() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Use an odd value on purpose so that it doesn't happen to coincide with one + // of the sync intervals. + const BACKOFF = 7337; + + // Extend info/collections so that we can put it into server maintenance mode. + const INFO_COLLECTIONS = "/1.1/johndoe@mozilla.com/info/collections"; + let infoColl = server._handler._overridePaths[INFO_COLLECTIONS]; + let serverMaintenance = false; + function infoCollWithMaintenance(request, response) { + if (!serverMaintenance) { + infoColl(request, response); + return; + } + response.setHeader("Retry-After", "" + BACKOFF); + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + } + server.registerPathHandler(INFO_COLLECTIONS, infoCollWithMaintenance); + + // Pretend we have two clients so that the regular sync interval is + // sufficiently low. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + let rec = await clientsEngine._store.createRecord("foo", "clients"); + await rec.encrypt(Service.collectionKeys.keyForCollection("clients")); + await rec.upload(Service.resource(clientsEngine.engineURL + rec.id)); + + // Sync once to log in and get everything set up. Let's verify our initial + // values. + await Service.sync(); + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.syncInterval); + // Sanity check that we picked the right value for BACKOFF: + Assert.ok(scheduler.syncInterval < BACKOFF * 1000); + + // Turn on server maintenance and sync again. + serverMaintenance = true; + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.ok(Status.backoffInterval >= BACKOFF * 1000); + // Allowing 3 seconds worth of of leeway between when Status.minimumNextSync + // was set and when this line gets executed. + let minimumExpectedDelay = (BACKOFF - 3) * 1000; + Assert.ok(Status.minimumNextSync >= Date.now() + minimumExpectedDelay); + + // Verify that the next sync is actually going to wait that long. + Assert.ok(scheduler.nextSync >= Date.now() + minimumExpectedDelay); + Assert.ok(scheduler.syncTimer.delay >= minimumExpectedDelay); + + await cleanUpAndGo(server); +}); + +add_task(async function test_loginError_recoverable_reschedules() { + _("Verify that a recoverable login error schedules a new sync."); + await configureIdentity({ username: "johndoe@mozilla.com" }); + Service.clusterURL = "http://localhost:1234/"; + Status.resetSync(); // reset Status.login + + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Let's set it up so that a sync is overdue, both in terms of previously + // scheduled syncs and the global score. We still do not expect an immediate + // sync because we just tried (duh). + scheduler.nextSync = Date.now() - 100000; + scheduler.globalScore = SINGLE_USER_THRESHOLD + 1; + function onSyncStart() { + do_throw("Shouldn't have started a sync!"); + } + Svc.Obs.add("weave:service:sync:start", onSyncStart); + + // Sanity check. + Assert.equal(scheduler.syncTimer, null); + Assert.equal(Status.checkSetup(), STATUS_OK); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + + scheduler.scheduleNextSync(0); + await promiseObserved; + await Async.promiseYield(); + + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + let expectedNextSync = Date.now() + scheduler.syncInterval; + Assert.ok(scheduler.nextSync > Date.now()); + Assert.ok(scheduler.nextSync <= expectedNextSync); + Assert.ok(scheduler.syncTimer.delay > 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.syncInterval); + + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + await cleanUpAndGo(); +}); + +add_task(async function test_loginError_fatal_clearsTriggers() { + _("Verify that a fatal login error clears sync triggers."); + await configureIdentity({ username: "johndoe@mozilla.com" }); + + let server = httpd_setup({ + "/1.1/johndoe@mozilla.com/info/collections": httpd_handler( + 401, + "Unauthorized" + ), + }); + + Service.clusterURL = server.baseURI + "/"; + Status.resetSync(); // reset Status.login + + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Sanity check. + Assert.equal(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer, null); + Assert.equal(Status.checkSetup(), STATUS_OK); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + + scheduler.scheduleNextSync(0); + await promiseObserved; + await Async.promiseYield(); + + // For the FxA identity, a 401 on info/collections means a transient + // error, probably due to an inability to fetch a token. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + // syncs should still be scheduled. + Assert.ok(scheduler.nextSync > Date.now()); + Assert.ok(scheduler.syncTimer.delay > 0); + + await cleanUpAndGo(server); +}); + +add_task(async function test_proper_interval_on_only_failing() { + _("Ensure proper behavior when only failed records are applied."); + + // If an engine reports that no records succeeded, we shouldn't decrease the + // sync interval. + Assert.ok(!scheduler.hasIncomingItems); + const INTERVAL = 10000000; + scheduler.syncInterval = INTERVAL; + + Svc.Obs.notify("weave:service:sync:applied", { + applied: 2, + succeeded: 0, + failed: 2, + newFailed: 2, + reconciled: 0, + }); + + await Async.promiseYield(); + scheduler.adjustSyncInterval(); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); +}); + +add_task(async function test_link_status_change() { + _("Check that we only attempt to sync when link status is up"); + try { + sinon.spy(scheduler, "scheduleNextSync"); + + Svc.Obs.notify("network:link-status-changed", null, "down"); + equal(scheduler.scheduleNextSync.callCount, 0); + + Svc.Obs.notify("network:link-status-changed", null, "change"); + equal(scheduler.scheduleNextSync.callCount, 0); + + Svc.Obs.notify("network:link-status-changed", null, "up"); + equal(scheduler.scheduleNextSync.callCount, 1); + + Svc.Obs.notify("network:link-status-changed", null, "change"); + equal(scheduler.scheduleNextSync.callCount, 1); + } finally { + scheduler.scheduleNextSync.restore(); + } +}); diff --git a/services/sync/tests/unit/test_tab_engine.js b/services/sync/tests/unit/test_tab_engine.js new file mode 100644 index 0000000000..5b1e61871e --- /dev/null +++ b/services/sync/tests/unit/test_tab_engine.js @@ -0,0 +1,226 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +// We'll need the clients engine for testing as tabs is closely related +let clientsEngine; + +async function syncClientsEngine(server) { + clientsEngine._lastFxADevicesFetch = 0; + clientsEngine.lastModified = server.getCollection("foo", "clients").timestamp; + await clientsEngine._sync(); +} + +async function makeRemoteClients() { + let server = await serverForFoo(clientsEngine); + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId, + fxaDeviceName: "Fxa - Remote client", + protocols: ["1.5"], + }); + + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId2, + fxaDeviceName: "Fxa - Remote client 2", + protocols: ["1.5"], + }); + + let fxAccounts = clientsEngine.fxAccounts; + clientsEngine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteId, name: "remote device" }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + await syncClientsEngine(server); +} + +add_task(async function setup() { + clientsEngine = Service.clientsEngine; + // Make some clients to test with + await makeRemoteClients(); + + // Make the tabs engine for all the tests to use + engine = Service.engineManager.get("tabs"); + await engine.initialize(); + + // Since these are xpcshell tests, we'll need to mock this + TabProvider.shouldSkipWindow = mockShouldSkipWindow; +}); + +add_task(async function test_tab_engine_skips_incoming_local_record() { + _("Ensure incoming records that match local client ID are never applied."); + + let localID = clientsEngine.localID; + let collection = new ServerCollection(); + + _("Creating remote tab record with local client ID"); + let localRecord = encryptPayload({ + id: localID, + clientName: "local", + tabs: [ + { + title: "title", + urlHistory: ["http://foo.com/"], + icon: "", + lastUsed: 2000, + }, + ], + }); + collection.insert(localID, localRecord); + + _("Creating remote tab record with a different client ID"); + let remoteID = "fake-guid-00"; // remote should match one of the test clients + let remoteRecord = encryptPayload({ + id: remoteID, + clientName: "not local", + tabs: [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + lastUsed: 3000, + }, + ], + }); + collection.insert(remoteID, remoteRecord); + + _("Setting up Sync server"); + let server = sync_httpd_setup({ + "/1.1/foo/storage/tabs": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { + tabs: { version: engine.version, syncID }, + }; + + await generateNewKeys(Service.collectionKeys); + + let promiseFinished = new Promise(resolve => { + let syncFinish = engine._syncFinish; + engine._syncFinish = async function () { + let remoteTabs = await engine._rustStore.getAll(); + equal( + remoteTabs.length, + 1, + "Remote client record was applied and local wasn't" + ); + let record = remoteTabs[0]; + equal(record.clientId, remoteID, "Remote client ID matches"); + + _("Ensure getAllClients returns the correct shape"); + let clients = await engine.getAllClients(); + equal(clients.length, 1); + let client = clients[0]; + equal(client.id, "fake-guid-00"); + equal(client.name, "Remote client"); + equal(client.type, "desktop"); + Assert.ok(client.lastModified); // lastModified should be filled in once serverModified is populated from the server + deepEqual(client.tabs, [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + inactive: false, + lastUsed: 3000, + }, + ]); + await syncFinish.call(engine); + resolve(); + }; + }); + + _("Start sync"); + Service.scheduler.hasIncomingItems = false; + await engine._sync(); + await promiseFinished; + // Bug 1800185 - we don't want the sync scheduler to see these records as incoming. + Assert.ok(!Service.scheduler.hasIncomingItems); +}); + +// Ensure we trim tabs in the case of going past the max payload size allowed +add_task(async function test_too_many_tabs() { + let a_lot_of_tabs = []; + + for (let i = 0; i < 4000; ++i) { + a_lot_of_tabs.push( + `http://example${i}.com/some-super-long-url-chain-to-help-with-bytes` + ); + } + + TabProvider.getWindowEnumerator = mockGetWindowEnumerator.bind( + this, + a_lot_of_tabs + ); + + let encoder = Utils.utf8Encoder; + // see tryfitItems(..) in util.js + const computeSerializedSize = records => + encoder.encode(JSON.stringify(records)).byteLength; + + const maxPayloadSize = Service.getMaxRecordPayloadSize(); + const maxSerializedSize = (maxPayloadSize / 4) * 3 - 1500; + // We are over max payload size + Assert.ok(computeSerializedSize(a_lot_of_tabs) > maxSerializedSize); + let tabs = await engine.getTabsWithinPayloadSize(); + // We are now under max payload size + Assert.ok(computeSerializedSize(tabs) < maxSerializedSize); +}); diff --git a/services/sync/tests/unit/test_tab_provider.js b/services/sync/tests/unit/test_tab_provider.js new file mode 100644 index 0000000000..bbf68dea33 --- /dev/null +++ b/services/sync/tests/unit/test_tab_provider.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); + +add_task(async function test_getAllTabs() { + let provider = TabProvider; + provider.shouldSkipWindow = mockShouldSkipWindow; + + let tabs; + + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://bar.com", + ]); + + _("Get all tabs."); + tabs = await provider.getAllTabsWithEstimatedMax( + false, + Number.MAX_SAFE_INTEGER + ); + _("Tabs: " + JSON.stringify(tabs)); + equal(tabs.length, 1); + equal(tabs[0].title, "title"); + equal(tabs[0].urlHistory.length, 1); + equal(tabs[0].urlHistory[0], "http://bar.com/"); + equal(tabs[0].icon, ""); + equal(tabs[0].lastUsed, 2); // windowenumerator returns in ms but the getAllTabs..() returns in seconds + + _("Get all tabs, and check that filtering works."); + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + "about:foo", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + _("Filtered: " + JSON.stringify(tabs)); + equal(tabs.length, 1); + + _("Get all tabs, and check that they are properly sorted"); + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + "http://bar.com", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + _("Ordered: " + JSON.stringify(tabs)); + equal(tabs[0].lastUsed > tabs[1].lastUsed, true); + + // reader mode URLs are provided. + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "about:reader?url=http%3A%2F%2Ffoo.com%2F", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + equal(tabs[0].urlHistory[0], "http://foo.com/"); +}); diff --git a/services/sync/tests/unit/test_tab_quickwrite.js b/services/sync/tests/unit/test_tab_quickwrite.js new file mode 100644 index 0000000000..2a1c75c8c6 --- /dev/null +++ b/services/sync/tests/unit/test_tab_quickwrite.js @@ -0,0 +1,204 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.importESModule("resource://services-sync/engines/tabs.sys.mjs"); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); + +const FAR_FUTURE = 4102405200000; // 2100/01/01 + +add_task(async function setup() { + // Since these are xpcshell tests, we'll need to mock ui features + TabProvider.shouldSkipWindow = mockShouldSkipWindow; + TabProvider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + ]); +}); + +async function prepareServer() { + _("Setting up Sync server"); + Service.serverConfiguration = { + max_post_records: 100, + }; + + let server = new SyncServer(); + server.start(); + await SyncTestingInfrastructure(server, "username"); + server.registerUser("username"); + + let collection = server.createCollection("username", "tabs"); + await generateNewKeys(Service.collectionKeys); + + let engine = Service.engineManager.get("tabs"); + await engine.initialize(); + + return { server, collection, engine }; +} + +async function withPatchedValue(object, name, patchedVal, fn) { + _(`patching ${name}=${patchedVal}`); + let old = object[name]; + object[name] = patchedVal; + try { + await fn(); + } finally { + object[name] = old; + } +} + +add_task(async function test_tab_quickwrite_works() { + _("Ensure a simple quickWrite works."); + let { server, collection, engine } = await prepareServer(); + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + Assert.ok(await engine.quickWrite()); + // Validate we didn't bork lastSync + let lastSync = await engine.getLastSync(); + Assert.ok(lastSync < FAR_FUTURE); + Assert.equal(collection.count(), 1, "tab record was written"); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_bad_status() { + _("Ensure quickWrite silently aborts when we aren't setup correctly."); + let { server, engine } = await prepareServer(); + // Store the original lock to reset it back after this test + let lock = engine.lock; + // Arrange for this test to fail if it tries to take the lock. + engine.lock = function () { + throw new Error("this test should abort syncing before locking"); + }; + let quickWrite = engine.quickWrite.bind(engine); // lol javascript. + + await withPatchedValue(engine, "enabled", false, quickWrite); + await withPatchedValue(Service, "serverConfiguration", null, quickWrite); + + Services.prefs.clearUserPref("services.sync.username"); + await quickWrite(); + // Validate we didn't bork lastSync + let lastSync = await engine.getLastSync(); + Assert.ok(lastSync < FAR_FUTURE); + Service.status.resetSync(); + engine.lock = lock; + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickwrite_lock() { + _("Ensure we fail to quickWrite if the engine is locked."); + let { server, collection, engine } = await prepareServer(); + + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + engine.lock(); + Assert.ok(!(await engine.quickWrite())); + Assert.equal(collection.count(), 0, "didn't sync due to being locked"); + engine.unlock(); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickwrite_keeps_old_tabs() { + _("Ensure we don't delete other tabs on quickWrite (bug 1801295)."); + let { server, engine } = await prepareServer(); + + // need a first sync to ensure everything is setup correctly. + await Service.sync({ engines: ["tabs"] }); + + const id = "fake-guid-99"; + let remoteRecord = encryptPayload({ + id, + clientName: "not local", + tabs: [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + lastUsed: 3000, + }, + ], + }); + + let collection = server.getCollection("username", "tabs"); + collection.insert(id, remoteRecord); + + await Service.sync({ engines: ["tabs"] }); + + // collection should now have 2 records - ours and the pretend remote one we inserted. + Assert.equal(collection.count(), 2, "starting with 2 tab records"); + + // So fxAccounts.device.recentDeviceList is not null. + engine.service.clientsEngine.fxAccounts.device._deviceListCache = { + devices: [], + }; + // trick the clients engine into thinking it has a remote client with the same guid. + engine.service.clientsEngine._store._remoteClients = {}; + engine.service.clientsEngine._store._remoteClients[id] = { + id, + fxaDeviceId: id, + }; + + let clients = await engine.getAllClients(); + Assert.equal(clients.length, 1); + + _("Doing a quick-write"); + Assert.ok(await engine.quickWrite()); + + // Should still have our client after a quickWrite. + _("Grabbing clients after the quick-write"); + clients = await engine.getAllClients(); + Assert.equal(clients.length, 1); + + engine.service.clientsEngine._store._remoteClients = {}; + + await promiseStopServer(server); +}); + +add_task(async function test_tab_lastSync() { + _("Ensure we restore the lastSync timestamp after a quick-write."); + let { server, collection, engine } = await prepareServer(); + + await engine.initialize(); + await engine.service.clientsEngine.initialize(); + + let origLastSync = engine.lastSync; + Assert.ok(await engine.quickWrite()); + Assert.equal(engine.lastSync, origLastSync); + Assert.equal(collection.count(), 1, "successful sync"); + engine.unlock(); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickWrite_telemetry() { + _("Ensure we record the telemetry we expect."); + // hook into telemetry + let telem = get_sync_test_telemetry(); + telem.payloads = []; + let oldSubmit = telem.submit; + let submitPromise = new Promise((resolve, reject) => { + telem.submit = function (ping) { + telem.submit = oldSubmit; + resolve(ping); + }; + }); + + let { server, collection, engine } = await prepareServer(); + + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + Assert.ok(await engine.quickWrite()); + Assert.equal(collection.count(), 1, "tab record was written"); + + let ping = await submitPromise; + let syncs = ping.syncs; + Assert.equal(syncs.length, 1); + let sync = syncs[0]; + Assert.equal(sync.why, "quick-write"); + Assert.equal(sync.engines.length, 1); + Assert.equal(sync.engines[0].name, "tabs"); + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_tab_tracker.js b/services/sync/tests/unit/test_tab_tracker.js new file mode 100644 index 0000000000..8bb71a898a --- /dev/null +++ b/services/sync/tests/unit/test_tab_tracker.js @@ -0,0 +1,371 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.importESModule("resource://services-sync/engines/tabs.sys.mjs"); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { SyncScheduler } = ChromeUtils.importESModule( + "resource://services-sync/policies.sys.mjs" +); + +var scheduler = new SyncScheduler(Service); +let clientsEngine; + +add_task(async function setup() { + await Service.promiseInitialized; + clientsEngine = Service.clientsEngine; + + scheduler.setDefaults(); +}); + +function fakeSvcWinMediator() { + // actions on windows are captured in logs + let logs = []; + delete Services.wm; + + function getNext() { + let elt = { addTopics: [], remTopics: [], numAPL: 0, numRPL: 0 }; + logs.push(elt); + return { + addEventListener(topic) { + elt.addTopics.push(topic); + }, + removeEventListener(topic) { + elt.remTopics.push(topic); + }, + gBrowser: { + addProgressListener() { + elt.numAPL++; + }, + removeProgressListener() { + elt.numRPL++; + }, + }, + }; + } + + Services.wm = { + getEnumerator() { + return [getNext(), getNext()]; + }, + }; + return logs; +} + +function fakeGetTabState(tab) { + return tab; +} + +function clearQuickWriteTimer(tracker) { + if (tracker.tabsQuickWriteTimer) { + tracker.tabsQuickWriteTimer.clear(); + } +} + +add_task(async function run_test() { + let engine = Service.engineManager.get("tabs"); + await engine.initialize(); + _("We assume that tabs have changed at startup."); + let tracker = engine._tracker; + tracker.getTabState = fakeGetTabState; + + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + let logs; + + _("Test listeners are registered on windows"); + logs = fakeSvcWinMediator(); + tracker.start(); + Assert.equal(logs.length, 2); + for (let log of logs) { + Assert.equal(log.addTopics.length, 3); + Assert.ok(log.addTopics.includes("TabOpen")); + Assert.ok(log.addTopics.includes("TabClose")); + Assert.ok(log.addTopics.includes("unload")); + Assert.equal(log.remTopics.length, 0); + Assert.equal(log.numAPL, 1, "Added 1 progress listener"); + Assert.equal(log.numRPL, 0, "Didn't remove a progress listener"); + } + + _("Test listeners are unregistered on windows"); + logs = fakeSvcWinMediator(); + await tracker.stop(); + Assert.equal(logs.length, 2); + for (let log of logs) { + Assert.equal(log.addTopics.length, 0); + Assert.equal(log.remTopics.length, 3); + Assert.ok(log.remTopics.includes("TabOpen")); + Assert.ok(log.remTopics.includes("TabClose")); + Assert.ok(log.remTopics.includes("unload")); + Assert.equal(log.numAPL, 0, "Didn't add a progress listener"); + Assert.equal(log.numRPL, 1, "Removed 1 progress listener"); + } + + _("Test tab listener"); + for (let evttype of ["TabOpen", "TabClose"]) { + // Pretend we just synced. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + } + + // Pretend we just synced. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + + tracker.onTab({ + type: "TabOpen", + originalTarget: "TabOpen", + target: { entries: [], currentURI: "about:config" }, + }); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + // Pretend we just synced and saw some progress listeners. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + tracker.onLocationChange({ isTopLevel: false }, undefined, undefined, 0); + Assert.ok(!tracker.modified, "non-toplevel request didn't flag as modified"); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org"), + Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT + ); + Assert.ok( + tracker.modified, + "location change within the same document request did flag as modified" + ); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org") + ); + Assert.ok( + tracker.modified, + "location change for a new top-level document flagged as modified" + ); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); +}); + +add_task(async function run_sync_on_tab_change_test() { + let testPrefDelay = 20000; + + // This is the pref that determines sync delay after tab change + Svc.PrefBranch.setIntPref( + "syncedTabs.syncDelayAfterTabChange", + testPrefDelay + ); + // We should only be syncing on tab change if + // the user has > 1 client + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 2); + + let engine = Service.engineManager.get("tabs"); + + _("We assume that tabs have changed at startup."); + let tracker = engine._tracker; + tracker.getTabState = fakeGetTabState; + + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + _("Test sync is scheduled after a tab change"); + for (let evttype of ["TabOpen", "TabClose"]) { + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be more delayed at or more than what the pref is set at + let nextSchedule = tracker.tabsQuickWriteTimer.delay; + Assert.ok(nextSchedule >= testPrefDelay); + } + + _("Test sync is NOT scheduled after an unsupported tab open"); + for (let evttype of ["TabOpen"]) { + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: ["about:newtab"], currentURI: null }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be scheduling <= pref value + Assert.ok(scheduler.nextSync - Date.now() <= testPrefDelay); + } + + _("Test navigating within the same tab does NOT trigger a sync"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org"), + Ci.nsIWebProgressListener.LOCATION_CHANGE_RELOAD + ); + Assert.ok( + !tracker.modified, + "location change for reloading doesn't trigger a sync" + ); + Assert.ok(!tracker.tabsQuickWriteTimer, "reload does not trigger a sync"); + + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + _("Test navigating to an about page does trigger sync"); + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("about:config") + ); + Assert.ok(tracker.modified, "about page does not trigger a tab modified"); + Assert.ok( + tracker.tabsQuickWriteTimer, + "about schema should trigger a sync happening soon" + ); + + _("Test adjusting the filterScheme pref works"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + Svc.PrefBranch.setStringPref( + "engine.tabs.filteredSchemes", + // Removing the about scheme for this test + "resource|chrome|file|blob|moz-extension" + ); + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("about:config") + ); + Assert.ok( + tracker.modified, + "about page triggers a modified after we changed the pref" + ); + Assert.ok( + tracker.tabsQuickWriteTimer, + "about page should schedule a quickWrite sync soon after we changed the pref" + ); + + _("Test no sync after tab change for accounts with <= 1 clients"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + // Setting clients to only 1 so we don't sync after a tab change + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 0); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 1); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org") + ); + Assert.ok( + tracker.modified, + "location change for a new top-level document flagged as modified" + ); + Assert.ok( + !tracker.tabsQuickWriteTimer, + "We should NOT be syncing shortly because there is only one client" + ); + + _("Changing the pref adjusts the sync schedule"); + Svc.PrefBranch.setIntPref("syncedTabs.syncDelayAfterTabChange", 10000); // 10seconds + let delayPref = Svc.PrefBranch.getIntPref( + "syncedTabs.syncDelayAfterTabChange" + ); + let evttype = "TabOpen"; + Assert.equal(delayPref, 10000); // ensure our pref is at 10s + // Only have task continuity if we have more than 1 device + Svc.PrefBranch.setIntPref("clients.devices.desktop", 1); + Svc.PrefBranch.setIntPref("clients.devices.mobile", 1); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 2); + clearQuickWriteTimer(tracker); + + // Fire ontab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be scheduling <= preference value + Assert.equal(tracker.tabsQuickWriteTimer.delay, delayPref); + + _("We should not have a sync scheduled if pref is at 0"); + + Svc.PrefBranch.setIntPref("syncedTabs.syncDelayAfterTabChange", 0); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + // Fire ontab event + evttype = "TabOpen"; + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + + // We should NOT be scheduled for a sync soon + Assert.ok(!tracker.tabsQuickWriteTimer); + + scheduler.setDefaults(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } +}); diff --git a/services/sync/tests/unit/test_telemetry.js b/services/sync/tests/unit/test_telemetry.js new file mode 100644 index 0000000000..961e96a01b --- /dev/null +++ b/services/sync/tests/unit/test_telemetry.js @@ -0,0 +1,1462 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); +const { getFxAccountsSingleton } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const fxAccounts = getFxAccountsSingleton(); + +function SteamStore(engine) { + Store.call(this, "Steam", engine); +} +Object.setPrototypeOf(SteamStore.prototype, Store.prototype); + +function SteamTracker(name, engine) { + LegacyTracker.call(this, name || "Steam", engine); +} +Object.setPrototypeOf(SteamTracker.prototype, LegacyTracker.prototype); + +function SteamEngine(service) { + SyncEngine.call(this, "steam", service); +} + +SteamEngine.prototype = { + _storeObj: SteamStore, + _trackerObj: SteamTracker, + _errToThrow: null, + problemsToReport: null, + async _sync() { + if (this._errToThrow) { + throw this._errToThrow; + } + }, + getValidator() { + return new SteamValidator(); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function BogusEngine(service) { + SyncEngine.call(this, "bogus", service); +} + +BogusEngine.prototype = Object.create(SteamEngine.prototype); + +class SteamValidator { + async canValidate() { + return true; + } + + async validate(engine) { + return { + problems: new SteamValidationProblemData(engine.problemsToReport), + version: 1, + duration: 0, + recordCount: 0, + }; + } +} + +class SteamValidationProblemData { + constructor(problemsToReport = []) { + this.problemsToReport = problemsToReport; + } + + getSummary() { + return this.problemsToReport; + } +} + +async function cleanAndGo(engine, server) { + await engine._tracker.clearChangedIDs(); + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + syncTestLogging(); + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function setup() { + // Avoid addon manager complaining about not being initialized + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); +}); + +add_task(async function test_basic() { + enableValidationPrefs(); + + let helper = track_collections_helper(); + let upd = helper.with_updated_collection; + + let handlers = { + "/1.1/johndoe/info/collections": helper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + }; + + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = upd( + coll, + new ServerCollection({}, true).handler() + ); + } + + let server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + + let ping = await wait_for_ping(() => Service.sync(), true, true); + + // Check the "os" block - we can't really check specific values, but can + // check it smells sane. + ok(ping.os, "there is an OS block"); + ok("name" in ping.os, "there is an OS name"); + ok("version" in ping.os, "there is an OS version"); + ok("locale" in ping.os, "there is an OS locale"); + + for (const pref of Svc.PrefBranch.getChildList("")) { + Svc.PrefBranch.clearUserPref(pref); + } + await promiseStopServer(server); +}); + +add_task(async function test_processIncoming_error() { + let engine = Service.engineManager.get("bookmarks"); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + try { + // Create a bogus record that when synced down will provoke a + // network error which in turn provokes an exception in _processIncoming. + const BOGUS_GUID = "zzzzzzzzzzzz"; + let bogus_record = collection.insert(BOGUS_GUID, "I'm a bogus record!"); + bogus_record.get = function get() { + throw new Error("Sync this!"); + }; + // Make the 10 minutes old so it will only be synced in the toFetch phase. + bogus_record.modified = Date.now() / 1000 - 60 * 10; + await engine.setLastSync(Date.now() / 1000 - 60); + engine.toFetch = new SerializableSet([BOGUS_GUID]); + + let error, pingPayload, fullPing; + try { + await sync_engine_and_validate_telem( + engine, + true, + (errPing, fullErrPing) => { + pingPayload = errPing; + fullPing = fullErrPing; + } + ); + } catch (ex) { + error = ex; + } + ok(!!error); + ok(!!pingPayload); + + equal(fullPing.uid, "f".repeat(32)); // as setup by SyncTestingInfrastructure + deepEqual(pingPayload.failureReason, { + name: "httperror", + code: 500, + }); + + equal(pingPayload.engines.length, 1); + + equal(pingPayload.engines[0].name, "bookmarks-buffered"); + deepEqual(pingPayload.engines[0].failureReason, { + name: "httperror", + code: 500, + }); + } finally { + await store.wipe(); + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_uploading() { + let engine = Service.engineManager.get("bookmarks"); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let bmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + + try { + let ping = await sync_engine_and_validate_telem(engine, false); + ok(!!ping); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "bookmarks-buffered"); + ok(!!ping.engines[0].outgoing); + greater(ping.engines[0].outgoing[0].sent, 0); + ok(!ping.engines[0].incoming); + + await PlacesUtils.bookmarks.update({ + guid: bmk.guid, + title: "New Title", + }); + + await store.wipe(); + await engine.resetClient(); + // We don't sync via the service, so don't re-hit info/collections, so + // lastModified remaning at zero breaks things subtly... + engine.lastModified = null; + + ping = await sync_engine_and_validate_telem(engine, false); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "bookmarks-buffered"); + equal(ping.engines[0].outgoing.length, 1); + ok(!!ping.engines[0].incoming); + } finally { + // Clean up. + await store.wipe(); + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_upload_failed() { + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + await configureIdentity({ username: "foo" }, server); + + let engine = new RotaryEngine(Service); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + peppercorn: "Peppercorn Class", + }; + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + const PEPPERCORN_CHANGED = 34567; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + await engine._tracker.addChangedID("peppercorn", PEPPERCORN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + let changes = await engine._tracker.getChangedIDs(); + _( + `test_upload_failed: Rotary tracker contents at first sync: ${JSON.stringify( + changes + )}` + ); + engine.enabled = true; + let ping = await sync_engine_and_validate_telem(engine, true); + ok(!!ping); + equal(ping.engines.length, 1); + equal(ping.engines[0].incoming, null); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 3, + failed: 2, + failedReasons: [ + { name: "scotsman", count: 1 }, + { name: "peppercorn", count: 1 }, + ], + }, + ]); + await engine.setLastSync(123); + + changes = await engine._tracker.getChangedIDs(); + _( + `test_upload_failed: Rotary tracker contents at second sync: ${JSON.stringify( + changes + )}` + ); + ping = await sync_engine_and_validate_telem(engine, true); + ok(!!ping); + equal(ping.engines.length, 1); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 2, + failed: 2, + failedReasons: [ + { name: "scotsman", count: 1 }, + { name: "peppercorn", count: 1 }, + ], + }, + ]); + } finally { + await cleanAndGo(engine, server); + await engine.finalize(); + } +}); + +add_task(async function test_sync_partialUpload() { + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = new RotaryEngine(Service); + await engine.setLastSync(123); + + // Create a bunch of records (and server side handlers) + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + engine._store.items[id] = "Record No. " + i; + await engine._tracker.addChangedID(id, i); + // Let two items in the first upload batch fail. + if (i != 23 && i != 42) { + collection.insert(id); + } + } + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + let changes = await engine._tracker.getChangedIDs(); + _( + `test_sync_partialUpload: Rotary tracker contents at first sync: ${JSON.stringify( + changes + )}` + ); + engine.enabled = true; + let ping = await sync_engine_and_validate_telem(engine, true); + + ok(!!ping); + ok(!ping.failureReason); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "rotary"); + ok(!ping.engines[0].incoming); + ok(!ping.engines[0].failureReason); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 234, + failed: 2, + failedReasons: [ + { name: "record-no-23", count: 1 }, + { name: "record-no-42", count: 1 }, + ], + }, + ]); + collection.post = function () { + throw new Error("Failure"); + }; + + engine._store.items["record-no-1000"] = "Record No. 1000"; + await engine._tracker.addChangedID("record-no-1000", 1000); + collection.insert("record-no-1000", 1000); + + await engine.setLastSync(123); + ping = null; + + changes = await engine._tracker.getChangedIDs(); + _( + `test_sync_partialUpload: Rotary tracker contents at second sync: ${JSON.stringify( + changes + )}` + ); + try { + // should throw + await sync_engine_and_validate_telem( + engine, + true, + errPing => (ping = errPing) + ); + } catch (e) {} + // It would be nice if we had a more descriptive error for this... + let uploadFailureError = { + name: "httperror", + code: 500, + }; + + ok(!!ping); + deepEqual(ping.failureReason, uploadFailureError); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "rotary"); + deepEqual(ping.engines[0].incoming, { + failed: 1, + failedReasons: [{ name: "No ciphertext: nothing to decrypt?", count: 1 }], + }); + ok(!ping.engines[0].outgoing); + deepEqual(ping.engines[0].failureReason, uploadFailureError); + } finally { + await cleanAndGo(engine, server); + await engine.finalize(); + } +}); + +add_task(async function test_generic_engine_fail() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let e = new Error("generic failure message"); + engine._errToThrow = e; + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_generic_engine_fail: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: String(e), + }); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_engine_fail_weird_errors() { + enableValidationPrefs(); + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + try { + let msg = "Bad things happened!"; + engine._errToThrow = { message: msg }; + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: "Bad things happened!", + }); + }); + let e = { msg }; + engine._errToThrow = e; + await sync_and_validate_telem(ping => { + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: JSON.stringify(e), + }); + }); + } finally { + await cleanAndGo(engine, server); + Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_overrideTelemetryName() { + enableValidationPrefs(["steam"]); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.overrideTelemetryName = "steam-but-better"; + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const problemsToReport = [ + { name: "someProblem", count: 123 }, + { name: "anotherProblem", count: 456 }, + ]; + + try { + info("Sync with validation problems"); + engine.problemsToReport = problemsToReport; + await sync_and_validate_telem(ping => { + let enginePing = ping.engines.find(e => e.name === "steam-but-better"); + ok(enginePing); + ok(!ping.engines.find(e => e.name === "steam")); + deepEqual( + enginePing.validation, + { + version: 1, + checked: 0, + problems: problemsToReport, + }, + "Should include validation report with overridden name" + ); + }); + + info("Sync without validation problems"); + engine.problemsToReport = null; + await sync_and_validate_telem(ping => { + let enginePing = ping.engines.find(e => e.name === "steam-but-better"); + ok(enginePing); + ok(!ping.engines.find(e => e.name === "steam")); + ok( + !enginePing.validation, + "Should not include validation report when there are no problems" + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_engine_fail_ioerror() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + // create an IOError to re-throw as part of Sync. + try { + // (Note that fakeservices.js has replaced Utils.jsonMove etc, but for + // this test we need the real one so we get real exceptions from the + // filesystem.) + await Utils._real_jsonMove("file-does-not-exist", "anything", {}); + } catch (ex) { + engine._errToThrow = ex; + } + ok(engine._errToThrow, "expecting exception"); + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_engine_fail_ioerror: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + // ensure the profile dir in the exception message has been stripped. + ok( + !failureReason.error.includes(PathUtils.profileDir), + failureReason.error + ); + ok(failureReason.error.includes("[profileDir]"), failureReason.error); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_error_detections() { + let telem = get_sync_test_telemetry(); + + // Non-network NS_ERROR_ codes get their own category. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_FAILURE)), + { name: "nserror", code: Cr.NS_ERROR_FAILURE } + ); + + // Some NS_ERROR_ code in the "network" module are treated as http errors. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_UNKNOWN_HOST)), + { name: "httperror", code: Cr.NS_ERROR_UNKNOWN_HOST } + ); + // Some NS_ERROR_ABORT is treated as network by our telemetry. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_ABORT)), + { name: "httperror", code: Cr.NS_ERROR_ABORT } + ); +}); + +add_task(async function test_clean_urls() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + engine._errToThrow = new TypeError( + "http://www.google .com is not a valid URL." + ); + + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_clean_urls: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal(failureReason.error, "<URL> is not a valid URL."); + }); + // Handle other errors that include urls. + engine._errToThrow = + "Other error message that includes some:url/foo/bar/ in it."; + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal( + failureReason.error, + "Other error message that includes <URL> in it." + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +// Test sanitizing guid-related errors with the pattern of <guid: {guid}> +add_task(async function test_sanitize_bookmarks_guid() { + let { ErrorSanitizer } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + + for (let [original, expected] of [ + [ + "Can't insert Bookmark <guid: sknD84IdnSY2> into Folder <guid: odfninDdi93_3>", + "Can't insert Bookmark <GUID> into Folder <GUID>", + ], + [ + "Merge Error: Item <guid: H6fmPA16gZs9> can't contain itself", + "Merge Error: Item <GUID> can't contain itself", + ], + ]) { + const sanitized = ErrorSanitizer.cleanErrorMessage(original); + Assert.equal(sanitized, expected); + } +}); + +// Test sanitization of some hard-coded error strings. +add_task(async function test_clean_errors() { + let { ErrorSanitizer } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + + for (let [message, name, expected] of [ + [ + `Could not open the file at ${PathUtils.join( + PathUtils.profileDir, + "weave", + "addonsreconciler.json" + )} for writing`, + "NotFoundError", + "OS error [File/Path not found] Could not open the file at [profileDir]/weave/addonsreconciler.json for writing", + ], + [ + `Could not get info for the file at ${PathUtils.join( + PathUtils.profileDir, + "weave", + "addonsreconciler.json" + )}`, + "NotAllowedError", + "OS error [Permission denied] Could not get info for the file at [profileDir]/weave/addonsreconciler.json", + ], + ]) { + const error = new DOMException(message, name); + const sanitized = ErrorSanitizer.cleanErrorMessage(message, error); + Assert.equal(sanitized, expected); + } +}); + +// Arrange for a sync to hit a "real" OS error during a sync and make sure it's sanitized. +add_task(async function test_clean_real_os_error() { + enableValidationPrefs(); + + // Simulate a real error. + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let path = PathUtils.join(PathUtils.profileDir, "no", "such", "path.json"); + try { + await IOUtils.readJSON(path); + throw new Error("should fail to read the file"); + } catch (ex) { + engine._errToThrow = ex; + } + + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_clean_urls: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal( + failureReason.error, + "OS error [File/Path not found] Could not open the file at [profileDir]/no/such/path.json" + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_initial_sync_engines() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + // These are the only ones who actually have things to sync at startup. + let telemetryEngineNames = ["clients", "prefs", "tabs", "bookmarks-buffered"]; + let server = await serverForEnginesWithKeys( + { foo: "password" }, + ["bookmarks", "prefs", "tabs"].map(name => Service.engineManager.get(name)) + ); + await SyncTestingInfrastructure(server); + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_initial_sync_engines: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + let ping = await wait_for_ping(() => Service.sync(), true); + + equal(ping.engines.find(e => e.name === "clients").outgoing[0].sent, 1); + equal(ping.engines.find(e => e.name === "tabs").outgoing[0].sent, 1); + + // for the rest we don't care about specifics + for (let e of ping.engines) { + if (!telemetryEngineNames.includes(engine.name)) { + continue; + } + greaterOrEqual(e.took, 1); + ok(!!e.outgoing); + equal(e.outgoing.length, 1); + notEqual(e.outgoing[0].sent, undefined); + equal(e.outgoing[0].failed, undefined); + equal(e.outgoing[0].failedReasons, undefined); + } + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_nserror() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + engine._errToThrow = Components.Exception( + "NS_ERROR_UNKNOWN_HOST", + Cr.NS_ERROR_UNKNOWN_HOST + ); + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_nserror: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + deepEqual(ping.status, { + service: SYNC_FAILED_PARTIAL, + sync: LOGIN_FAILED_NETWORK_ERROR, + }); + let enginePing = ping.engines.find(e => e.name === "steam"); + deepEqual(enginePing.failureReason, { + name: "httperror", + code: Cr.NS_ERROR_UNKNOWN_HOST, + }); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_sync_why() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let e = new Error("generic failure message"); + engine._errToThrow = e; + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_generic_engine_fail: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + let ping = await wait_for_ping( + () => Service.sync({ why: "user" }), + true, + false + ); + _(JSON.stringify(ping)); + equal(ping.why, "user"); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_discarding() { + enableValidationPrefs(); + + let helper = track_collections_helper(); + let upd = helper.with_updated_collection; + let telem = get_sync_test_telemetry(); + telem.maxPayloadCount = 2; + telem.submissionInterval = Infinity; + let oldSubmit = telem.submit; + + let server; + try { + let handlers = { + "/1.1/johndoe/info/collections": helper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + }; + + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = upd( + coll, + new ServerCollection({}, true).handler() + ); + } + + server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + telem.submit = p => + ok( + false, + "Submitted telemetry ping when we should not have" + JSON.stringify(p) + ); + + for (let i = 0; i < 5; ++i) { + await Service.sync(); + } + telem.submit = oldSubmit; + telem.submissionInterval = -1; + let ping = await wait_for_ping(() => Service.sync(), true, true); // with this we've synced 6 times + equal(ping.syncs.length, 2); + equal(ping.discarded, 4); + } finally { + telem.maxPayloadCount = 500; + telem.submissionInterval = -1; + telem.submit = oldSubmit; + if (server) { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_submit_interval() { + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + let numSubmissions = 0; + telem.submit = function () { + numSubmissions += 1; + }; + + function notify(what, data = null) { + Svc.Obs.notify(what, JSON.stringify(data)); + } + + try { + // submissionInterval is set such that each sync should submit + notify("weave:service:sync:start", { why: "testing" }); + notify("weave:service:sync:finish"); + Assert.equal(numSubmissions, 1, "should submit this ping due to interval"); + + // As should each event outside of a sync. + Service.recordTelemetryEvent("object", "method"); + Assert.equal(numSubmissions, 2); + + // But events while we are syncing should not. + notify("weave:service:sync:start", { why: "testing" }); + Service.recordTelemetryEvent("object", "method"); + Assert.equal(numSubmissions, 2, "no submission for this event"); + notify("weave:service:sync:finish"); + Assert.equal(numSubmissions, 3, "was submitted after sync finish"); + } finally { + telem.submit = oldSubmit; + } +}); + +add_task(async function test_no_foreign_engines_in_error_ping() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + engine._errToThrow = new Error("Oh no!"); + await SyncTestingInfrastructure(server); + try { + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + ok(ping.engines.every(e => e.name !== "bogus")); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_no_foreign_engines_in_success_ping() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + await sync_and_validate_telem(ping => { + ok(ping.engines.every(e => e.name !== "bogus")); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_events() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + + let telem = get_sync_test_telemetry(); + telem.submissionInterval = Infinity; + + try { + let serverTime = Resource.serverTime; + Service.recordTelemetryEvent("object", "method", "value", { foo: "bar" }); + let ping = await wait_for_ping(() => Service.sync(), true, true); + equal(ping.events.length, 1); + let [timestamp, category, method, object, value, extra] = ping.events[0]; + ok(typeof timestamp == "number" && timestamp > 0); // timestamp. + equal(category, "sync"); + equal(method, "method"); + equal(object, "object"); + equal(value, "value"); + deepEqual(extra, { foo: "bar", serverTime: String(serverTime) }); + ping = await wait_for_ping( + () => { + // Test with optional values. + Service.recordTelemetryEvent("object", "method"); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 4); + + ping = await wait_for_ping( + () => { + Service.recordTelemetryEvent("object", "method", "extra"); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 5); + + ping = await wait_for_ping( + () => { + Service.recordTelemetryEvent("object", "method", undefined, { + foo: "bar", + }); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 6); + [timestamp, category, method, object, value, extra] = ping.events[0]; + equal(value, null); + + // Fake a submission due to shutdown. + ping = await wait_for_ping( + () => { + telem.submissionInterval = Infinity; + Service.recordTelemetryEvent("object", "method", undefined, { + foo: "bar", + }); + telem.finish("shutdown"); + }, + false, + true + ); + equal(ping.syncs.length, 0); + equal(ping.events.length, 1); + equal(ping.events[0].length, 6); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_histograms() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + let histId = "TELEMETRY_TEST_LINEAR"; + Services.obs.notifyObservers(null, "weave:telemetry:histogram", histId); + let ping = await wait_for_ping(() => Service.sync(), true, true); + equal(Object.keys(ping.histograms).length, 1); + equal(ping.histograms[histId].sum, 0); + equal(ping.histograms[histId].histogram_type, 1); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_invalid_events() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + async function checkNotRecorded(...args) { + Service.recordTelemetryEvent.call(args); + let ping = await wait_for_ping(() => Service.sync(), false, true); + equal(ping.events, undefined); + } + + await SyncTestingInfrastructure(server); + try { + let long21 = "l".repeat(21); + let long81 = "l".repeat(81); + let long86 = "l".repeat(86); + await checkNotRecorded("object"); + await checkNotRecorded("object", 2); + await checkNotRecorded(2, "method"); + await checkNotRecorded("object", "method", 2); + await checkNotRecorded("object", "method", "value", 2); + await checkNotRecorded("object", "method", "value", { foo: 2 }); + await checkNotRecorded(long21, "method", "value"); + await checkNotRecorded("object", long21, "value"); + await checkNotRecorded("object", "method", long81); + let badextra = {}; + badextra[long21] = "x"; + await checkNotRecorded("object", "method", "value", badextra); + badextra = { x: long86 }; + await checkNotRecorded("object", "method", "value", badextra); + for (let i = 0; i < 10; i++) { + badextra["name" + i] = "x"; + } + await checkNotRecorded("object", "method", "value", badextra); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_no_ping_for_self_hosters() { + enableValidationPrefs(); + + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + let submitPromise = new Promise(resolve => { + telem.submit = function () { + let result = oldSubmit.apply(this, arguments); + resolve(result); + }; + }); + await Service.sync(); + let pingSubmitted = await submitPromise; + // The Sync testing infrastructure already sets up a custom token server, + // so we don't need to do anything to simulate a self-hosted user. + ok(!pingSubmitted, "Should not submit ping with custom token server URL"); + } finally { + telem.submit = oldSubmit; + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_fxa_device_telem() { + let t = get_sync_test_telemetry(); + let syncEnabled = true; + let oldGetClientsEngineRecords = t.getClientsEngineRecords; + let oldGetFxaDevices = t.getFxaDevices; + let oldSyncIsEnabled = t.syncIsEnabled; + let oldSanitizeFxaDeviceId = t.sanitizeFxaDeviceId; + t.syncIsEnabled = () => syncEnabled; + t.sanitizeFxaDeviceId = id => `So clean: ${id}`; + try { + let keep0 = Utils.makeGUID(); + let keep1 = Utils.makeGUID(); + let keep2 = Utils.makeGUID(); + let curdev = Utils.makeGUID(); + + let keep1Sync = Utils.makeGUID(); + let keep2Sync = Utils.makeGUID(); + let curdevSync = Utils.makeGUID(); + let fxaDevices = [ + { + id: curdev, + isCurrentDevice: true, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "current device", + }, + { + id: keep0, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 10, + pushEndpointExpired: false, + type: "mobile", + name: "dupe", + }, + // Valid 2 + { + id: keep1, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "valid2", + }, + // Valid 3 + { + id: keep2, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 5, + pushEndpointExpired: false, + type: "desktop", + name: "valid3", + }, + ]; + let clientInfo = [ + { + id: keep1Sync, + fxaDeviceId: keep1, + os: "Windows 30", + version: "Firefox 1 million", + }, + { + id: keep2Sync, + fxaDeviceId: keep2, + os: "firefox, but an os", + verison: "twelve", + }, + { + id: Utils.makeGUID(), + fxaDeviceId: null, + os: "apparently ios used to keep write these IDs as null.", + version: "Doesn't seem to anymore", + }, + { + id: curdevSync, + fxaDeviceId: curdev, + os: "emacs", + version: "22", + }, + { + id: Utils.makeGUID(), + fxaDeviceId: Utils.makeGUID(), + os: "not part of the fxa device set at all", + version: "foo bar baz", + }, + // keep0 intententionally omitted. + ]; + t.getClientsEngineRecords = () => clientInfo; + let devInfo = t.updateFxaDevices(fxaDevices); + equal(devInfo.deviceID, t.sanitizeFxaDeviceId(curdev)); + for (let d of devInfo.devices) { + ok(d.id.startsWith("So clean:")); + if (d.syncID) { + ok(d.syncID.startsWith("So clean:")); + } + } + equal(devInfo.devices.length, 4); + let k0 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep0)); + let k1 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep1)); + let k2 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep2)); + + deepEqual(k0, { + id: t.sanitizeFxaDeviceId(keep0), + type: "mobile", + os: undefined, + version: undefined, + syncID: undefined, + }); + deepEqual(k1, { + id: t.sanitizeFxaDeviceId(keep1), + type: "desktop", + os: clientInfo[0].os, + version: clientInfo[0].version, + syncID: t.sanitizeFxaDeviceId(keep1Sync), + }); + deepEqual(k2, { + id: t.sanitizeFxaDeviceId(keep2), + type: "desktop", + os: clientInfo[1].os, + version: clientInfo[1].version, + syncID: t.sanitizeFxaDeviceId(keep2Sync), + }); + let newCurId = Utils.makeGUID(); + // Update the ID + fxaDevices[0].id = newCurId; + + let keep3 = Utils.makeGUID(); + fxaDevices.push({ + id: keep3, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "valid 4", + }); + devInfo = t.updateFxaDevices(fxaDevices); + + let afterSubmit = [keep0, keep1, keep2, keep3, newCurId] + .map(id => t.sanitizeFxaDeviceId(id)) + .sort(); + deepEqual(devInfo.devices.map(d => d.id).sort(), afterSubmit); + + // Reset this, as our override doesn't check for sync being enabled. + t.sanitizeFxaDeviceId = oldSanitizeFxaDeviceId; + syncEnabled = false; + fxAccounts.telemetry._setHashedUID(false); + devInfo = t.updateFxaDevices(fxaDevices); + equal(devInfo.deviceID, undefined); + equal(devInfo.devices.length, 5); + for (let d of devInfo.devices) { + equal(d.os, undefined); + equal(d.version, undefined); + equal(d.syncID, undefined); + // Type should still be present. + notEqual(d.type, undefined); + } + } finally { + t.getClientsEngineRecords = oldGetClientsEngineRecords; + t.getFxaDevices = oldGetFxaDevices; + t.syncIsEnabled = oldSyncIsEnabled; + t.sanitizeFxaDeviceId = oldSanitizeFxaDeviceId; + } +}); + +add_task(async function test_sanitize_fxa_device_id() { + let t = get_sync_test_telemetry(); + fxAccounts.telemetry._setHashedUID(false); + sinon.stub(t, "syncIsEnabled").callsFake(() => true); + const rawDeviceId = "raw one two three"; + try { + equal(t.sanitizeFxaDeviceId(rawDeviceId), null); + fxAccounts.telemetry._setHashedUID("mock uid"); + const sanitizedDeviceId = t.sanitizeFxaDeviceId(rawDeviceId); + ok(sanitizedDeviceId); + notEqual(sanitizedDeviceId, rawDeviceId); + } finally { + t.syncIsEnabled.restore(); + fxAccounts.telemetry._setHashedUID(false); + } +}); + +add_task(async function test_no_node_type() { + let server = sync_httpd_setup({}); + await configureIdentity(null, server); + + await sync_and_validate_telem(ping => { + ok(ping.syncNodeType === undefined); + }, true); + await promiseStopServer(server); +}); + +add_task(async function test_node_type() { + Service.identity.logout(); + let server = sync_httpd_setup({}); + await configureIdentity({ node_type: "the-node-type" }, server); + + await sync_and_validate_telem(ping => { + equal(ping.syncNodeType, "the-node-type"); + }, true); + await promiseStopServer(server); +}); + +add_task(async function test_node_type_change() { + let pingPromise = wait_for_pings(2); + + Service.identity.logout(); + let server = sync_httpd_setup({}); + await configureIdentity({ node_type: "first-node-type" }, server); + // Default to submitting each hour - we should still submit on node change. + let telem = get_sync_test_telemetry(); + telem.submissionInterval = 60 * 60 * 1000; + // reset the node type from previous test or our first sync will submit. + telem.lastSyncNodeType = null; + // do 2 syncs with the same node type. + await Service.sync(); + await Service.sync(); + // then another with a different node type. + Service.identity.logout(); + await configureIdentity({ node_type: "second-node-type" }, server); + await Service.sync(); + telem.finish(); + + let pings = await pingPromise; + equal(pings.length, 2); + equal(pings[0].syncs.length, 2, "2 syncs in first ping"); + equal(pings[0].syncNodeType, "first-node-type"); + equal(pings[1].syncs.length, 1, "1 sync in second ping"); + equal(pings[1].syncNodeType, "second-node-type"); + await promiseStopServer(server); +}); + +add_task(async function test_ids() { + let telem = get_sync_test_telemetry(); + Assert.ok(!telem._shouldSubmitForDataChange()); + fxAccounts.telemetry._setHashedUID("new_uid"); + Assert.ok(telem._shouldSubmitForDataChange()); + telem.maybeSubmitForDataChange(); + // now it's been submitted the new uid is current. + Assert.ok(!telem._shouldSubmitForDataChange()); +}); + +add_task(async function test_deletion_request_ping() { + async function assertRecordedSyncDeviceID(expected) { + // The scalar gets updated asynchronously, so wait a tick before checking. + await Promise.resolve(); + const scalars = + Services.telemetry.getSnapshotForScalars("deletion-request").parent || {}; + equal(scalars["deletion.request.sync_device_id"], expected); + } + + const MOCK_HASHED_UID = "00112233445566778899aabbccddeeff"; + const MOCK_DEVICE_ID1 = "ffeeddccbbaa99887766554433221100"; + const MOCK_DEVICE_ID2 = "aabbccddeeff99887766554433221100"; + + // Calculated by hand using SHA256(DEVICE_ID + HASHED_UID)[:32] + const SANITIZED_DEVICE_ID1 = "dd7c845006df9baa1c6d756926519c8c"; + const SANITIZED_DEVICE_ID2 = "0d06919a736fc029007e1786a091882c"; + + let currentDeviceID = null; + sinon.stub(fxAccounts.device, "getLocalId").callsFake(() => { + return Promise.resolve(currentDeviceID); + }); + let telem = get_sync_test_telemetry(); + sinon.stub(telem, "isProductionSyncUser").callsFake(() => true); + fxAccounts.telemetry._setHashedUID(false); + try { + // The scalar should start out undefined, since no user is actually logged in. + await assertRecordedSyncDeviceID(undefined); + + // If we start up without knowing the hashed UID, it should stay undefined. + telem.observe(null, "weave:service:ready"); + await assertRecordedSyncDeviceID(undefined); + + // But now let's say we've discovered the hashed UID from the server. + fxAccounts.telemetry._setHashedUID(MOCK_HASHED_UID); + currentDeviceID = MOCK_DEVICE_ID1; + + // Now when we load up, we'll record the sync device id. + telem.observe(null, "weave:service:ready"); + await assertRecordedSyncDeviceID(SANITIZED_DEVICE_ID1); + + // When the device-id changes we'll update it. + currentDeviceID = MOCK_DEVICE_ID2; + telem.observe(null, "fxaccounts:new_device_id"); + await assertRecordedSyncDeviceID(SANITIZED_DEVICE_ID2); + + // When the user signs out we'll clear it. + telem.observe(null, "fxaccounts:onlogout"); + await assertRecordedSyncDeviceID(""); + } finally { + fxAccounts.telemetry._setHashedUID(false); + telem.isProductionSyncUser.restore(); + fxAccounts.device.getLocalId.restore(); + } +}); diff --git a/services/sync/tests/unit/test_tracker_addChanged.js b/services/sync/tests/unit/test_tracker_addChanged.js new file mode 100644 index 0000000000..7f510794fc --- /dev/null +++ b/services/sync/tests/unit/test_tracker_addChanged.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_tracker_basics() { + let tracker = new LegacyTracker("Tracker", Service); + + let id = "the_id!"; + + _("Make sure nothing exists yet.."); + let changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], null); + + _("Make sure adding of time 0 works"); + await tracker.addChangedID(id, 0); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 0); + + _("A newer time will replace the old 0"); + await tracker.addChangedID(id, 10); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 10); + + _("An older time will not replace the newer 10"); + await tracker.addChangedID(id, 5); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 10); + + _("Adding without time defaults to current time"); + await tracker.addChangedID(id); + changes = await tracker.getChangedIDs(); + Assert.ok(changes[id] > 10); +}); + +add_task(async function test_tracker_persistence() { + let tracker = new LegacyTracker("Tracker", Service); + let id = "abcdef"; + + let promiseSave = new Promise((resolve, reject) => { + let save = tracker._storage._save; + tracker._storage._save = function () { + save.call(tracker._storage).then(resolve, reject); + }; + }); + + await tracker.addChangedID(id, 5); + + await promiseSave; + + _("IDs saved."); + const changes = await tracker.getChangedIDs(); + Assert.equal(5, changes[id]); + + let json = await Utils.jsonLoad(["changes", "tracker"], tracker); + Assert.equal(5, json[id]); +}); diff --git a/services/sync/tests/unit/test_uistate.js b/services/sync/tests/unit/test_uistate.js new file mode 100644 index 0000000000..cb1ff1979e --- /dev/null +++ b/services/sync/tests/unit/test_uistate.js @@ -0,0 +1,324 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { UIState } = ChromeUtils.importESModule( + "resource://services-sync/UIState.sys.mjs" +); + +const UIStateInternal = UIState._internal; + +add_task(async function test_isReady_unconfigured() { + UIState.reset(); + + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + // On the first call, returns false + // Does trigger a refresh of the state - even though services.sync.username + // is undefined we still need to check the account state. + ok(!UIState.isReady()); + // resfreshState is called when idle - so only check after idle. + await new Promise(resolve => { + Services.tm.idleDispatchToMainThread(resolve); + }); + ok(refreshState.called); + refreshState.resetHistory(); + + // On subsequent calls, only return true + ok(UIState.isReady()); + ok(!refreshState.called); + + refreshState.restore(); +}); + +add_task(async function test_isReady_signedin() { + UIState.reset(); + Services.prefs.setStringPref("services.sync.username", "foo"); + + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + // On the first call, returns false and triggers a refresh of the state + ok(!UIState.isReady()); + await new Promise(resolve => { + Services.tm.idleDispatchToMainThread(resolve); + }); + ok(refreshState.calledOnce); + refreshState.resetHistory(); + + // On subsequent calls, only return true + ok(UIState.isReady()); + ok(!refreshState.called); + + refreshState.restore(); +}); + +add_task(async function test_refreshState_signedin() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setStringPref("services.sync.lastSync", now); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ + verified: true, + uid: "123", + email: "foo@bar.com", + displayName: "Foo Bar", + avatar: "https://foo/bar", + }), + hasLocalSession: () => Promise.resolve(true), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_SIGNED_IN); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, "Foo Bar"); + equal(state.avatarURL, "https://foo/bar"); + equal(state.lastSync, now); + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_syncButNoFxA() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setStringPref("services.sync.lastSync", now); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => Promise.resolve(null), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, undefined); + equal(state.email, "test@test.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); // only set when STATUS_SIGNED_IN. + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; + Services.prefs.clearUserPref("services.sync.lastSync"); + Services.prefs.clearUserPref("services.sync.username"); +}); + +add_task(async function test_refreshState_signedin_profile_unavailable() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setStringPref("services.sync.lastSync", now); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + _internal: { + profile: { + getProfile: () => { + return Promise.reject(new Error("Profile unavailable")); + }, + }, + }, + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_SIGNED_IN); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, now); + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; + Services.prefs.clearUserPref("services.sync.lastSync"); + Services.prefs.clearUserPref("services.sync.username"); +}); + +add_task(async function test_refreshState_unverified() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: false, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_NOT_VERIFIED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_unverified_nosession() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: false, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(false), + }; + + let state = await UIState.refresh(); + + // No session should "win" over the unverified state. + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_loginFailed() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + let loginFailed = sinon.stub(UIStateInternal, "_loginFailed"); + loginFailed.returns(true); + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + loginFailed.restore(); + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_observer_refreshState() { + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + let shouldRefresh = [ + "weave:service:login:got-hashed-id", + "weave:service:login:error", + "weave:service:ready", + "fxaccounts:onverified", + "fxaccounts:onlogin", + "fxaccounts:onlogout", + "fxaccounts:profilechange", + ]; + + for (let topic of shouldRefresh) { + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, topic); + await uiUpdateObserved; + ok(refreshState.calledOnce); + refreshState.resetHistory(); + } + + refreshState.restore(); +}); + +// Drive the UIState in a configured state. +async function configureUIState(syncing, lastSync = new Date()) { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal._syncing = syncing; + Services.prefs.setStringPref("services.sync.lastSync", lastSync.toString()); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + }; + await UIState.refresh(); + UIStateInternal.fxAccounts = fxAccountsOrig; +} + +add_task(async function test_syncStarted() { + await configureUIState(false); + + const oldState = Object.assign({}, UIState.get()); + ok(!oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, "weave:service:sync:start"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(newState.syncing); +}); + +add_task(async function test_syncFinished() { + let yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + await configureUIState(true, yesterday); + + const oldState = Object.assign({}, UIState.get()); + ok(oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.prefs.setStringPref("services.sync.lastSync", new Date().toString()); + Services.obs.notifyObservers(null, "weave:service:sync:finish"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(!newState.syncing); + ok(new Date(newState.lastSync) > new Date(oldState.lastSync)); +}); + +add_task(async function test_syncError() { + let yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + await configureUIState(true, yesterday); + + const oldState = Object.assign({}, UIState.get()); + ok(oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, "weave:service:sync:error"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(!newState.syncing); + deepEqual(newState.lastSync, oldState.lastSync); +}); + +function observeUIUpdate() { + return new Promise(resolve => { + let obs = (aSubject, aTopic, aData) => { + Services.obs.removeObserver(obs, aTopic); + const state = UIState.get(); + resolve(state); + }; + Services.obs.addObserver(obs, UIState.ON_UPDATE); + }); +} diff --git a/services/sync/tests/unit/test_utils_catch.js b/services/sync/tests/unit/test_utils_catch.js new file mode 100644 index 0000000000..590d04527f --- /dev/null +++ b/services/sync/tests/unit/test_utils_catch.js @@ -0,0 +1,119 @@ +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + _("Make sure catch when copied to an object will correctly catch stuff"); + let ret, rightThis, didCall, didThrow, wasCovfefe, wasLocked; + let obj = { + _catch: Utils.catch, + _log: { + debug(str) { + didThrow = str.search(/^Exception/) == 0; + }, + info(str) { + wasLocked = str.indexOf("Cannot start sync: already syncing?") == 0; + }, + }, + + func() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + })(); + }, + + callbacky() { + return this._catch( + async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + }, + async function (ex) { + wasCovfefe = ex && ex.message == "covfefe"; + } + )(); + }, + + lockedy() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + Utils.throwLockException(null); + })(); + }, + + lockedy_chained() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + Utils.throwLockException(null); + })(); + }, + }; + + _("Make sure a normal call will call and return"); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(!didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(!wasLocked); + + _( + "Make sure catch/throw results in debug call and caller doesn't need to handle exception" + ); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.throwy(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(!wasLocked); + + _("Test callback for exception testing."); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.callbacky(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.ok(wasCovfefe); + Assert.ok(!wasLocked); + + _("Test the lock-aware catch that Service uses."); + obj._catch = Service._catch; + rightThis = didCall = didThrow = wasLocked = false; + wasCovfefe = undefined; + ret = await obj.lockedy(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(wasLocked); + + _("Test the lock-aware catch that Service uses with a chained promise."); + rightThis = didCall = didThrow = wasLocked = false; + wasCovfefe = undefined; + ret = await obj.lockedy_chained(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(wasLocked); +}); diff --git a/services/sync/tests/unit/test_utils_deepEquals.js b/services/sync/tests/unit/test_utils_deepEquals.js new file mode 100644 index 0000000000..218cc21b72 --- /dev/null +++ b/services/sync/tests/unit/test_utils_deepEquals.js @@ -0,0 +1,51 @@ +_("Make sure Utils.deepEquals correctly finds items that are deeply equal"); + +Services.prefs.setBoolPref("security.allow_eval_with_system_principal", true); +registerCleanupFunction(() => { + Services.prefs.clearUserPref("security.allow_eval_with_system_principal"); +}); + +function run_test() { + let data = + '[NaN, undefined, null, true, false, Infinity, 0, 1, "a", "b", {a: 1}, {a: "a"}, [{a: 1}], [{a: true}], {a: 1, b: 2}, [1, 2], [1, 2, 3]]'; + _("Generating two copies of data:", data); + /* eslint-disable no-eval */ + let d1 = eval(data); + let d2 = eval(data); + /* eslint-enable no-eval */ + + d1.forEach(function (a) { + _("Testing", a, typeof a, JSON.stringify([a])); + let numMatch = 0; + + d2.forEach(function (b) { + if (Utils.deepEquals(a, b)) { + numMatch++; + _("Found a match", b, typeof b, JSON.stringify([b])); + } + }); + + let expect = 1; + if (isNaN(a) && typeof a == "number") { + expect = 0; + _("Checking NaN should result in no matches"); + } + + _("Making sure we found the correct # match:", expect); + _("Actual matches:", numMatch); + Assert.equal(numMatch, expect); + }); + + _("Make sure adding undefined properties doesn't affect equalness"); + let a = {}; + let b = { a: undefined }; + Assert.ok(Utils.deepEquals(a, b)); + a.b = 5; + Assert.ok(!Utils.deepEquals(a, b)); + b.b = 5; + Assert.ok(Utils.deepEquals(a, b)); + a.c = undefined; + Assert.ok(Utils.deepEquals(a, b)); + b.d = undefined; + Assert.ok(Utils.deepEquals(a, b)); +} diff --git a/services/sync/tests/unit/test_utils_deferGetSet.js b/services/sync/tests/unit/test_utils_deferGetSet.js new file mode 100644 index 0000000000..6db812b844 --- /dev/null +++ b/services/sync/tests/unit/test_utils_deferGetSet.js @@ -0,0 +1,50 @@ +_( + "Make sure various combinations of deferGetSet arguments correctly defer getting/setting properties to another object" +); + +function run_test() { + let base = function () {}; + base.prototype = { + dst: {}, + + get a() { + return "a"; + }, + set b(val) { + this.dst.b = val + "!!!"; + }, + }; + let src = new base(); + + _("get/set a single property"); + Utils.deferGetSet(base, "dst", "foo"); + src.foo = "bar"; + Assert.equal(src.dst.foo, "bar"); + Assert.equal(src.foo, "bar"); + + _("editing the target also updates the source"); + src.dst.foo = "baz"; + Assert.equal(src.dst.foo, "baz"); + Assert.equal(src.foo, "baz"); + + _("handle multiple properties"); + Utils.deferGetSet(base, "dst", ["p1", "p2"]); + src.p1 = "v1"; + src.p2 = "v2"; + Assert.equal(src.p1, "v1"); + Assert.equal(src.dst.p1, "v1"); + Assert.equal(src.p2, "v2"); + Assert.equal(src.dst.p2, "v2"); + + _("make sure existing getter keeps its functionality"); + Utils.deferGetSet(base, "dst", "a"); + src.a = "not a"; + Assert.equal(src.dst.a, "not a"); + Assert.equal(src.a, "a"); + + _("make sure existing setter keeps its functionality"); + Utils.deferGetSet(base, "dst", "b"); + src.b = "b"; + Assert.equal(src.dst.b, "b!!!"); + Assert.equal(src.b, "b!!!"); +} diff --git a/services/sync/tests/unit/test_utils_json.js b/services/sync/tests/unit/test_utils_json.js new file mode 100644 index 0000000000..5bf26b2361 --- /dev/null +++ b/services/sync/tests/unit/test_utils_json.js @@ -0,0 +1,95 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +add_task(async function test_roundtrip() { + _("Do a simple write of an array to json and read"); + await Utils.jsonSave("foo", {}, ["v1", "v2"]); + + let foo = await Utils.jsonLoad("foo", {}); + Assert.equal(typeof foo, "object"); + Assert.equal(foo.length, 2); + Assert.equal(foo[0], "v1"); + Assert.equal(foo[1], "v2"); +}); + +add_task(async function test_string() { + _("Try saving simple strings"); + await Utils.jsonSave("str", {}, "hi"); + + let str = await Utils.jsonLoad("str", {}); + Assert.equal(typeof str, "string"); + Assert.equal(str.length, 2); + Assert.equal(str[0], "h"); + Assert.equal(str[1], "i"); +}); + +add_task(async function test_number() { + _("Try saving a number"); + await Utils.jsonSave("num", {}, 42); + + let num = await Utils.jsonLoad("num", {}); + Assert.equal(typeof num, "number"); + Assert.equal(num, 42); +}); + +add_task(async function test_nonexistent_file() { + _("Try loading a non-existent file."); + let val = await Utils.jsonLoad("non-existent", {}); + Assert.equal(val, undefined); +}); + +add_task(async function test_save_logging() { + _("Verify that writes are logged."); + let trace; + await Utils.jsonSave( + "log", + { + _log: { + trace(msg) { + trace = msg; + }, + }, + }, + "hi" + ); + Assert.ok(!!trace); +}); + +add_task(async function test_load_logging() { + _("Verify that reads and read errors are logged."); + + // Write a file with some invalid JSON + let file = await IOUtils.getFile(PathUtils.profileDir, "weave", "log.json"); + let fos = Cc["@mozilla.org/network/file-output-stream;1"].createInstance( + Ci.nsIFileOutputStream + ); + let flags = + FileUtils.MODE_WRONLY | FileUtils.MODE_CREATE | FileUtils.MODE_TRUNCATE; + fos.init(file, flags, FileUtils.PERMS_FILE, fos.DEFER_OPEN); + let stream = Cc["@mozilla.org/intl/converter-output-stream;1"].createInstance( + Ci.nsIConverterOutputStream + ); + stream.init(fos, "UTF-8"); + stream.writeString("invalid json!"); + stream.close(); + + let trace, debug; + let obj = { + _log: { + trace(msg) { + trace = msg; + }, + debug(msg) { + debug = msg; + }, + }, + }; + let val = await Utils.jsonLoad("log", obj); + Assert.ok(!val); + Assert.ok(!!trace); + Assert.ok(!!debug); +}); diff --git a/services/sync/tests/unit/test_utils_keyEncoding.js b/services/sync/tests/unit/test_utils_keyEncoding.js new file mode 100644 index 0000000000..30a8a4f2aa --- /dev/null +++ b/services/sync/tests/unit/test_utils_keyEncoding.js @@ -0,0 +1,23 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +function run_test() { + Assert.equal( + Utils.encodeKeyBase32("foobarbafoobarba"), + "mzxw6ytb9jrgcztpn5rgc4tcme" + ); + Assert.equal( + Utils.decodeKeyBase32("mzxw6ytb9jrgcztpn5rgc4tcme"), + "foobarbafoobarba" + ); + Assert.equal( + Utils.encodeKeyBase32( + "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + ), + "aeaqcaibaeaqcaibaeaqcaibae" + ); + Assert.equal( + Utils.decodeKeyBase32("aeaqcaibaeaqcaibaeaqcaibae"), + "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + ); +} diff --git a/services/sync/tests/unit/test_utils_lock.js b/services/sync/tests/unit/test_utils_lock.js new file mode 100644 index 0000000000..71d6486ff9 --- /dev/null +++ b/services/sync/tests/unit/test_utils_lock.js @@ -0,0 +1,76 @@ +_("Make sure lock prevents calling with a shared lock"); + +// Utility that we only use here. + +function do_check_begins(thing, startsWith) { + if (!(thing && thing.indexOf && thing.indexOf(startsWith) == 0)) { + do_throw(thing + " doesn't begin with " + startsWith); + } +} + +add_task(async function run_test() { + let ret, rightThis, didCall; + let state, lockState, lockedState, unlockState; + let obj = { + _lock: Utils.lock, + lock() { + lockState = ++state; + if (this._locked) { + lockedState = ++state; + return false; + } + this._locked = true; + return true; + }, + unlock() { + unlockState = ++state; + this._locked = false; + }, + + func() { + return this._lock("Test utils lock", async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this._lock("Test utils lock throwy", async function () { + rightThis = this == obj; + didCall = true; + return this.throwy(); + })(); + }, + }; + + _("Make sure a normal call will call and return"); + rightThis = didCall = false; + state = 0; + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.equal(lockState, 1); + Assert.equal(unlockState, 2); + Assert.equal(state, 2); + + _("Make sure code that calls locked code throws"); + ret = null; + rightThis = didCall = false; + try { + ret = await obj.throwy(); + do_throw("throwy internal call should have thrown!"); + } catch (ex) { + // Should throw an Error, not a string. + do_check_begins(ex.message, "Could not acquire lock"); + } + Assert.equal(ret, null); + Assert.ok(rightThis); + Assert.ok(didCall); + _("Lock should be called twice so state 3 is skipped"); + Assert.equal(lockState, 4); + Assert.equal(lockedState, 5); + Assert.equal(unlockState, 6); + Assert.equal(state, 6); +}); diff --git a/services/sync/tests/unit/test_utils_makeGUID.js b/services/sync/tests/unit/test_utils_makeGUID.js new file mode 100644 index 0000000000..b1104c1114 --- /dev/null +++ b/services/sync/tests/unit/test_utils_makeGUID.js @@ -0,0 +1,44 @@ +const base64url = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; + +function run_test() { + _("Make sure makeGUID makes guids of the right length/characters"); + _("Create a bunch of guids to make sure they don't conflict"); + let guids = []; + for (let i = 0; i < 1000; i++) { + let newGuid = Utils.makeGUID(); + _("Generated " + newGuid); + + // Verify that the GUID's length is correct, even when it's URL encoded. + Assert.equal(newGuid.length, 12); + Assert.equal(encodeURIComponent(newGuid).length, 12); + + // Verify that the GUID only contains base64url characters + Assert.ok( + Array.prototype.every.call(newGuid, function (chr) { + return base64url.includes(chr); + }) + ); + + // Verify that Utils.checkGUID() correctly identifies them as valid. + Assert.ok(Utils.checkGUID(newGuid)); + + // Verify uniqueness within our sample of 1000. This could cause random + // failures, but they should be extremely rare. Otherwise we'd have a + // problem with GUID collisions. + Assert.ok( + guids.every(function (g) { + return g != newGuid; + }) + ); + guids.push(newGuid); + } + + _("Make sure checkGUID fails for invalid GUIDs"); + Assert.ok(!Utils.checkGUID(undefined)); + Assert.ok(!Utils.checkGUID(null)); + Assert.ok(!Utils.checkGUID("")); + Assert.ok(!Utils.checkGUID("blergh")); + Assert.ok(!Utils.checkGUID("ThisGUIDisWayTooLong")); + Assert.ok(!Utils.checkGUID("Invalid!!!!!")); +} diff --git a/services/sync/tests/unit/test_utils_notify.js b/services/sync/tests/unit/test_utils_notify.js new file mode 100644 index 0000000000..5c0c3702a6 --- /dev/null +++ b/services/sync/tests/unit/test_utils_notify.js @@ -0,0 +1,97 @@ +_("Make sure notify sends out the right notifications"); +add_task(async function run_test() { + let ret, rightThis, didCall; + let obj = { + notify: Utils.notify("foo:"), + _log: { + trace() {}, + }, + + func() { + return this.notify("bar", "baz", async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this.notify("bad", "one", async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + })(); + }, + }; + + let state = 0; + let makeObs = function (topic) { + let obj2 = { + observe(subject, obsTopic, data) { + this.state = ++state; + this.subject = subject; + this.topic = obsTopic; + this.data = data; + }, + }; + + Svc.Obs.add(topic, obj2); + return obj2; + }; + + _("Make sure a normal call will call and return with notifications"); + rightThis = didCall = false; + let fs = makeObs("foo:bar:start"); + let ff = makeObs("foo:bar:finish"); + let fe = makeObs("foo:bar:error"); + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + + Assert.equal(fs.state, 1); + Assert.equal(fs.subject, undefined); + Assert.equal(fs.topic, "foo:bar:start"); + Assert.equal(fs.data, "baz"); + + Assert.equal(ff.state, 2); + Assert.equal(ff.subject, 5); + Assert.equal(ff.topic, "foo:bar:finish"); + Assert.equal(ff.data, "baz"); + + Assert.equal(fe.state, undefined); + Assert.equal(fe.subject, undefined); + Assert.equal(fe.topic, undefined); + Assert.equal(fe.data, undefined); + + _("Make sure a throwy call will call and throw with notifications"); + ret = null; + rightThis = didCall = false; + let ts = makeObs("foo:bad:start"); + let tf = makeObs("foo:bad:finish"); + let te = makeObs("foo:bad:error"); + try { + ret = await obj.throwy(); + do_throw("throwy should have thrown!"); + } catch (ex) { + Assert.equal(ex.message, "covfefe"); + } + Assert.equal(ret, null); + Assert.ok(rightThis); + Assert.ok(didCall); + + Assert.equal(ts.state, 3); + Assert.equal(ts.subject, undefined); + Assert.equal(ts.topic, "foo:bad:start"); + Assert.equal(ts.data, "one"); + + Assert.equal(tf.state, undefined); + Assert.equal(tf.subject, undefined); + Assert.equal(tf.topic, undefined); + Assert.equal(tf.data, undefined); + + Assert.equal(te.state, 4); + Assert.equal(te.subject.message, "covfefe"); + Assert.equal(te.topic, "foo:bad:error"); + Assert.equal(te.data, "one"); +}); diff --git a/services/sync/tests/unit/test_utils_passphrase.js b/services/sync/tests/unit/test_utils_passphrase.js new file mode 100644 index 0000000000..fa58086113 --- /dev/null +++ b/services/sync/tests/unit/test_utils_passphrase.js @@ -0,0 +1,45 @@ +/* eslint no-tabs:"off" */ + +function run_test() { + _("Normalize passphrase recognizes hyphens."); + const pp = "26ect2thczm599m2ffqarbicjq"; + const hyphenated = "2-6ect2-thczm-599m2-ffqar-bicjq"; + Assert.equal(Utils.normalizePassphrase(hyphenated), pp); + + _("Skip whitespace."); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase("aaaaaaaaaaaaaaaaaaaaaaaaaa ") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa ") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" a-aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ") + ); + Assert.ok(Utils.isPassphrase("aaaaaaaaaaaaaaaaaaaaaaaaaa ")); + Assert.ok(Utils.isPassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa")); + Assert.ok(Utils.isPassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa ")); + Assert.ok(Utils.isPassphrase(" a-aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ")); + Assert.ok(!Utils.isPassphrase(" -aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ")); + + _("Normalizing 20-char passphrases."); + Assert.equal( + Utils.normalizePassphrase("abcde-abcde-abcde-abcde"), + "abcdeabcdeabcdeabcde" + ); + Assert.equal( + Utils.normalizePassphrase("a-bcde-abcde-abcde-abcde"), + "a-bcde-abcde-abcde-abcde" + ); + Assert.equal( + Utils.normalizePassphrase(" abcde-abcde-abcde-abcde "), + "abcdeabcdeabcdeabcde" + ); +} diff --git a/services/sync/tests/unit/xpcshell.toml b/services/sync/tests/unit/xpcshell.toml new file mode 100644 index 0000000000..e958c8a738 --- /dev/null +++ b/services/sync/tests/unit/xpcshell.toml @@ -0,0 +1,304 @@ +[DEFAULT] +head = "head_appinfo.js ../../../common/tests/unit/head_helpers.js head_helpers.js head_http_server.js head_errorhandler_common.js" +firefox-appdir = "browser" +prefs = ["identity.fxaccounts.enabled=true"] +support-files = [ + "addon1-search.json", + "bootstrap1-search.json", + "missing-sourceuri.json", + "missing-xpi-search.json", + "rewrite-search.json", + "sync_ping_schema.json", + "systemaddon-search.json", + "!/services/common/tests/unit/head_helpers.js", + "!/toolkit/components/extensions/test/xpcshell/head_sync.js", +] + +# The manifest is roughly ordered from low-level to high-level. When making +# systemic sweeping changes, this makes it easier to identify errors closer to +# the source. + +# Ensure we can import everything. + +["test_412.js"] + +["test_addon_utils.js"] +run-sequentially = "Restarts server, can't change pref." +tags = "addons" + +["test_addons_engine.js"] +run-sequentially = "Frequent timeouts, bug 1395148" +tags = "addons" + +["test_addons_reconciler.js"] +skip-if = ["appname == 'thunderbird'"] +tags = "addons" + +["test_addons_store.js"] +run-sequentially = "Frequent timeouts, bug 1395148" +tags = "addons" + +["test_addons_tracker.js"] +tags = "addons" + +["test_addons_validator.js"] +tags = "addons" + +["test_bookmark_batch_fail.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_decline_undecline.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_engine.js"] +skip-if = [ + "appname == 'thunderbird'", + "tsan", # Runs unreasonably slow on TSan, bug 1612707 +] + +["test_bookmark_order.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_places_query_rewriting.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_record.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_store.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_bookmark_tracker.js"] +skip-if = [ + "appname == 'thunderbird'", + "tsan", # Runs unreasonably slow on TSan, bug 1612707 +] +requesttimeoutfactor = 4 + +["test_bridged_engine.js"] + +["test_clients_engine.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_clients_escape.js"] + +["test_collection_getBatched.js"] + +["test_collections_recovery.js"] + +["test_corrupt_keys.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_declined.js"] + +["test_disconnect_shutdown.js"] + +["test_engine.js"] + +["test_engine_abort.js"] + +["test_engine_changes_during_sync.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_enginemanager.js"] + +["test_errorhandler_1.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_errorhandler_2.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_errorhandler_filelog.js"] + +["test_errorhandler_sync_checkServerError.js"] + +["test_extension_storage_engine.js"] +skip-if = ["appname == 'thunderbird'"] +run-sequentially = "extension-storage migration happens only once, and must be tested first." + +["test_extension_storage_engine_kinto.js"] +skip-if = ["appname == 'thunderbird'"] +run-sequentially = "extension-storage migration happens only once, and must be tested first." + +["test_extension_storage_migration_telem.js"] +skip-if = ["appname == 'thunderbird'"] +run-sequentially = "extension-storage migration happens only once, and must be tested first." + +["test_extension_storage_tracker_kinto.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_form_validator.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_forms_store.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_forms_tracker.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_fxa_node_reassignment.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_fxa_service_cluster.js"] +# Finally, we test each engine. + +["test_history_engine.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_history_store.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_history_tracker.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_hmac_error.js"] + +["test_httpd_sync_server.js"] +# HTTP layers. + +["test_interval_triggers.js"] + +["test_keys.js"] + +["test_load_modules.js"] +# util contains a bunch of functionality used throughout. + +["test_node_reassignment.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_password_engine.js"] + +["test_password_store.js"] + +["test_password_tracker.js"] + +["test_password_validator.js"] + +["test_postqueue.js"] +# Synced tabs. + +["test_prefs_engine.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_prefs_store.js"] +skip-if = ["appname == 'thunderbird'"] +support-files = ["prefs_test_prefs_store.js"] + +["test_prefs_tracker.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_records_crypto.js"] + +["test_records_wbo.js"] + +["test_resource.js"] + +["test_resource_header.js"] + +["test_resource_ua.js"] +# Generic Sync types. + +["test_score_triggers.js"] + +["test_service_attributes.js"] +# Bug 752243: Profile cleanup frequently fails +skip-if = [ + "os == 'mac'", + "os == 'linux'", +] + +["test_service_cluster.js"] + +["test_service_detect_upgrade.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_service_login.js"] + +["test_service_startOver.js"] + +["test_service_startup.js"] + +["test_service_sync_401.js"] + +["test_service_sync_locked.js"] + +["test_service_sync_remoteSetup.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_service_sync_specified.js"] + +["test_service_sync_updateEnabledEngines.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_service_verifyLogin.js"] + +["test_service_wipeClient.js"] + +["test_service_wipeServer.js"] +# Bug 752243: Profile cleanup frequently fails +skip-if = [ + "os == 'mac'", + "os == 'linux'", +] + +["test_status.js"] + +["test_status_checkSetup.js"] + +["test_sync_auth_manager.js"] +# Engine APIs. + +["test_syncedtabs.js"] + +["test_syncengine.js"] + +["test_syncengine_sync.js"] +run-sequentially = "Frequent timeouts, bug 1395148" + +["test_syncscheduler.js"] +run-sequentially = "Frequent timeouts, bug 1395148" +# Firefox Accounts specific tests + +["test_tab_engine.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_tab_provider.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_tab_quickwrite.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_tab_tracker.js"] +skip-if = ["appname == 'thunderbird'"] + +["test_telemetry.js"] +skip-if = [ + "appname == 'thunderbird'", + "tsan", # Unreasonably slow, bug 1612707 +] +requesttimeoutfactor = 4 + +["test_tracker_addChanged.js"] +# Service semantics. + +["test_uistate.js"] + +["test_utils_catch.js"] + +["test_utils_deepEquals.js"] + +["test_utils_deferGetSet.js"] + +["test_utils_json.js"] + +["test_utils_keyEncoding.js"] + +["test_utils_lock.js"] + +["test_utils_makeGUID.js"] +run-sequentially = "Disproportionately slows down full test run, bug 1450316" + +["test_utils_notify.js"] + +["test_utils_passphrase.js"] +# We have a number of other libraries that are pretty much standalone. diff --git a/services/sync/tps/extensions/tps/api.js b/services/sync/tps/extensions/tps/api.js new file mode 100644 index 0000000000..0843376630 --- /dev/null +++ b/services/sync/tps/extensions/tps/api.js @@ -0,0 +1,77 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +/* globals ExtensionAPI, Services, XPCOMUtils */ + +XPCOMUtils.defineLazyServiceGetter( + this, + "resProto", + "@mozilla.org/network/protocol;1?name=resource", + "nsISubstitutingProtocolHandler" +); + +async function tpsStartup() { + try { + var { TPS } = ChromeUtils.importESModule("resource://tps/tps.sys.mjs"); + let { goQuitApplication } = ChromeUtils.importESModule( + "resource://tps/quit.sys.mjs" + ); + TPS.goQuitApplication = goQuitApplication; + + let testFile = Services.prefs.getStringPref("testing.tps.testFile", ""); + let testPhase = Services.prefs.getStringPref("testing.tps.testPhase", ""); + if (!testFile || !testPhase) { + // Note: this quits. + TPS.DumpError( + "TPS no longer takes arguments from the command line. " + + "instead you need to pass preferences `testing.tps.{testFile,testPhase}` " + + "and optionally `testing.tps.{logFile,ignoreUnusedEngines}`.\n" + ); + } + + let logFile = Services.prefs.getStringPref("testing.tps.logFile", ""); + let ignoreUnusedEngines = Services.prefs.getBoolPref( + "testing.tps.ignoreUnusedEngines", + false + ); + let options = { ignoreUnusedEngines }; + let testFileUri = Services.io.newFileURI(new FileUtils.File(testFile)).spec; + + try { + await TPS.RunTestPhase(testFileUri, testPhase, logFile, options); + } catch (err) { + TPS.DumpError("TestPhase failed", err); + } + } catch (e) { + if (typeof TPS != "undefined") { + // Note: This calls quit() under the hood + TPS.DumpError("Test initialization failed", e); + } + dump(`TPS test initialization failed: ${e} - ${e.stack}\n`); + // Try and quit right away, no reason to wait around for python + // to kill us if initialization failed. + Services.startup.quit(Ci.nsIAppStartup.eForceQuit); + } +} + +this.tps = class extends ExtensionAPI { + onStartup() { + resProto.setSubstitution( + "tps", + Services.io.newURI("resource/", null, this.extension.rootURI) + ); + /* Ignore the platform's online/offline status while running tests. */ + Services.io.manageOfflineStatus = false; + Services.io.offline = false; + tpsStartup(); + } + + onShutdown() { + resProto.setSubstitution("tps", null); + } +}; diff --git a/services/sync/tps/extensions/tps/manifest.json b/services/sync/tps/extensions/tps/manifest.json new file mode 100644 index 0000000000..c961e76506 --- /dev/null +++ b/services/sync/tps/extensions/tps/manifest.json @@ -0,0 +1,23 @@ +{ + "manifest_version": 2, + "name": "TPS", + "version": "1.0", + + "browser_specific_settings": { + "gecko": { + "id": "tps@mozilla.org" + } + }, + + "experiment_apis": { + "tps": { + "schema": "schema.json", + "parent": { + "scopes": ["addon_parent"], + "script": "api.js", + "paths": [["tps"]], + "events": ["startup"] + } + } + } +} diff --git a/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs new file mode 100644 index 0000000000..81c0fd578a --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs @@ -0,0 +1,209 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; +import { clearTimeout, setTimeout } from "resource://gre/modules/Timer.sys.mjs"; + +import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs"; + +const fxAccounts = getFxAccountsSingleton(); +import { FxAccountsClient } from "resource://gre/modules/FxAccountsClient.sys.mjs"; +import { FxAccountsConfig } from "resource://gre/modules/FxAccountsConfig.sys.mjs"; +import { Logger } from "resource://tps/logger.sys.mjs"; + +/** + * Helper object for Firefox Accounts authentication + */ +export var Authentication = { + /** + * Check if an user has been logged in + */ + async isLoggedIn() { + return !!(await this.getSignedInUser()); + }, + + async isReady() { + let user = await this.getSignedInUser(); + return user && user.verified; + }, + + _getRestmailUsername(user) { + const restmailSuffix = "@restmail.net"; + if (user.toLowerCase().endsWith(restmailSuffix)) { + return user.slice(0, -restmailSuffix.length); + } + return null; + }, + + async shortWaitForVerification(ms) { + let userData = await this.getSignedInUser(); + let timeoutID; + let timeoutPromise = new Promise(resolve => { + timeoutID = setTimeout(() => { + Logger.logInfo(`Warning: no verification after ${ms}ms.`); + resolve(); + }, ms); + }); + await Promise.race([ + fxAccounts.whenVerified(userData).finally(() => clearTimeout(timeoutID)), + timeoutPromise, + ]); + userData = await this.getSignedInUser(); + return userData && userData.verified; + }, + + async _openVerificationPage(uri) { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let newtab = mainWindow.gBrowser.addWebTab(uri); + let win = mainWindow.gBrowser.getBrowserForTab(newtab); + await new Promise(resolve => { + win.addEventListener("loadend", resolve, { once: true }); + }); + let didVerify = await this.shortWaitForVerification(10000); + mainWindow.gBrowser.removeTab(newtab); + return didVerify; + }, + + async _completeVerification(user) { + let username = this._getRestmailUsername(user); + if (!username) { + Logger.logInfo( + `Username "${user}" isn't a restmail username so can't complete verification` + ); + return false; + } + Logger.logInfo("Fetching mail (from restmail) for user " + username); + let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent( + username + )}`; + let triedAlready = new Set(); + const tries = 10; + const normalWait = 2000; + for (let i = 0; i < tries; ++i) { + let resp = await fetch(restmailURI); + let messages = await resp.json(); + // Sort so that the most recent emails are first. + messages.sort((a, b) => new Date(b.receivedAt) - new Date(a.receivedAt)); + for (let m of messages) { + // We look for a link that has a x-link that we haven't yet tried. + if (!m.headers["x-link"] || triedAlready.has(m.headers["x-link"])) { + continue; + } + let confirmLink = m.headers["x-link"]; + triedAlready.add(confirmLink); + Logger.logInfo("Trying confirmation link " + confirmLink); + try { + if (await this._openVerificationPage(confirmLink)) { + return true; + } + } catch (e) { + Logger.logInfo( + "Warning: Failed to follow confirmation link: " + + Log.exceptionStr(e) + ); + } + } + if (i === 0) { + // first time through after failing we'll do this. + await fxAccounts.resendVerificationEmail(); + } + if (await this.shortWaitForVerification(normalWait)) { + return true; + } + } + // One last try. + return this.shortWaitForVerification(normalWait); + }, + + async deleteEmail(user) { + let username = this._getRestmailUsername(user); + if (!username) { + Logger.logInfo("Not a restmail username, can't delete"); + return false; + } + Logger.logInfo("Deleting mail (from restmail) for user " + username); + let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent( + username + )}`; + try { + // Clean up after ourselves. + let deleteResult = await fetch(restmailURI, { method: "DELETE" }); + if (!deleteResult.ok) { + Logger.logInfo( + `Warning: Got non-success status ${deleteResult.status} when deleting emails` + ); + return false; + } + } catch (e) { + Logger.logInfo( + "Warning: Failed to delete old emails: " + Log.exceptionStr(e) + ); + return false; + } + return true; + }, + + /** + * Wrapper to retrieve the currently signed in user + * + * @returns Information about the currently signed in user + */ + async getSignedInUser() { + try { + return await fxAccounts.getSignedInUser(); + } catch (error) { + Logger.logError( + "getSignedInUser() failed with: " + JSON.stringify(error) + ); + throw error; + } + }, + + /** + * Wrapper to synchronize the login of a user + * + * @param account + * Account information of the user to login + * @param account.username + * The username for the account (utf8) + * @param account.password + * The user's password + */ + async signIn(account) { + Logger.AssertTrue(account.username, "Username has been found"); + Logger.AssertTrue(account.password, "Password has been found"); + + Logger.logInfo("Login user: " + account.username); + + try { + // Required here since we don't go through the real login page + await FxAccountsConfig.ensureConfigured(); + + let client = new FxAccountsClient(); + let credentials = await client.signIn( + account.username, + account.password, + true + ); + await fxAccounts._internal.setSignedInUser(credentials); + if (!credentials.verified) { + await this._completeVerification(account.username); + } + + return true; + } catch (error) { + throw new Error("signIn() failed with: " + error.message); + } + }, + + /** + * Sign out of Firefox Accounts. + */ + async signOut() { + if (await Authentication.isLoggedIn()) { + // Note: This will clean up the device ID. + await fxAccounts.signOut(); + } + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/logger.sys.mjs b/services/sync/tps/extensions/tps/resource/logger.sys.mjs new file mode 100644 index 0000000000..a1995f88b6 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/logger.sys.mjs @@ -0,0 +1,168 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", +}); + +export var Logger = { + _foStream: null, + _converter: null, + _potentialError: null, + + init(path) { + if (this._converter != null) { + // we're already open! + return; + } + + if (path) { + Services.prefs.setStringPref("tps.logfile", path); + } else { + path = Services.prefs.getStringPref("tps.logfile"); + } + + this._file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + this._file.initWithPath(path); + var exists = this._file.exists(); + + // Make a file output stream and converter to handle it. + this._foStream = Cc[ + "@mozilla.org/network/file-output-stream;1" + ].createInstance(Ci.nsIFileOutputStream); + // If the file already exists, append it, otherwise create it. + var fileflags = exists ? 0x02 | 0x08 | 0x10 : 0x02 | 0x08 | 0x20; + + this._foStream.init(this._file, fileflags, 0o666, 0); + this._converter = Cc[ + "@mozilla.org/intl/converter-output-stream;1" + ].createInstance(Ci.nsIConverterOutputStream); + this._converter.init(this._foStream, "UTF-8"); + }, + + write(data) { + if (this._converter == null) { + console.error("TPS Logger.write called with _converter == null!"); + return; + } + this._converter.writeString(data); + }, + + close() { + if (this._converter != null) { + this._converter.close(); + this._converter = null; + this._foStream = null; + } + }, + + AssertTrue(bool, msg, showPotentialError) { + if (bool) { + return; + } + + if (showPotentialError && this._potentialError) { + msg += "; " + this._potentialError; + this._potentialError = null; + } + throw new Error("ASSERTION FAILED! " + msg); + }, + + AssertFalse(bool, msg, showPotentialError) { + return this.AssertTrue(!bool, msg, showPotentialError); + }, + + AssertEqual(got, expected, msg) { + if (!lazy.ObjectUtils.deepEqual(got, expected)) { + throw new Error( + "ASSERTION FAILED! " + + msg + + "; expected " + + JSON.stringify(expected) + + ", got " + + JSON.stringify(got) + ); + } + }, + + log(msg, withoutPrefix) { + dump(msg + "\n"); + if (withoutPrefix) { + this.write(msg + "\n"); + } else { + function pad(n, len) { + let s = "0000" + n; + return s.slice(-len); + } + + let now = new Date(); + let year = pad(now.getFullYear(), 4); + let month = pad(now.getMonth() + 1, 2); + let day = pad(now.getDate(), 2); + let hour = pad(now.getHours(), 2); + let minutes = pad(now.getMinutes(), 2); + let seconds = pad(now.getSeconds(), 2); + let ms = pad(now.getMilliseconds(), 3); + + this.write( + year + + "-" + + month + + "-" + + day + + " " + + hour + + ":" + + minutes + + ":" + + seconds + + "." + + ms + + " " + + msg + + "\n" + ); + } + }, + + clearPotentialError() { + this._potentialError = null; + }, + + logPotentialError(msg) { + this._potentialError = msg; + }, + + logLastPotentialError(msg) { + var message = msg; + if (this._potentialError) { + message = this._poentialError; + this._potentialError = null; + } + this.log("CROSSWEAVE ERROR: " + message); + }, + + logError(msg) { + this.log("CROSSWEAVE ERROR: " + msg); + }, + + logInfo(msg, withoutPrefix) { + if (withoutPrefix) { + this.log(msg, true); + } else { + this.log("CROSSWEAVE INFO: " + msg); + } + }, + + logPass(msg) { + this.log("CROSSWEAVE TEST PASS: " + msg); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs new file mode 100644 index 0000000000..596f942a06 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AddonManager } from "resource://gre/modules/AddonManager.sys.mjs"; +import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs"; +import { Logger } from "resource://tps/logger.sys.mjs"; + +export const STATE_ENABLED = 1; +export const STATE_DISABLED = 2; + +export function Addon(TPS, id) { + this.TPS = TPS; + this.id = id; +} + +Addon.prototype = { + addon: null, + + async uninstall() { + // find our addon locally + let addon = await AddonManager.getAddonByID(this.id); + Logger.AssertTrue( + !!addon, + "could not find addon " + this.id + " to uninstall" + ); + await AddonUtils.uninstallAddon(addon); + }, + + async find(state) { + let addon = await AddonManager.getAddonByID(this.id); + + if (!addon) { + Logger.logInfo("Could not find add-on with ID: " + this.id); + return false; + } + + this.addon = addon; + + Logger.logInfo( + "add-on found: " + addon.id + ", enabled: " + !addon.userDisabled + ); + if (state == STATE_ENABLED) { + Logger.AssertFalse(addon.userDisabled, "add-on is disabled: " + addon.id); + return true; + } else if (state == STATE_DISABLED) { + Logger.AssertTrue(addon.userDisabled, "add-on is enabled: " + addon.id); + return true; + } else if (state) { + throw new Error("Don't know how to handle state: " + state); + } else { + // No state, so just checking that it exists. + return true; + } + }, + + async install() { + // For Install, the id parameter initially passed is really the filename + // for the addon's install .xml; we'll read the actual id from the .xml. + + const result = await AddonUtils.installAddons([ + { id: this.id, requireSecureURI: false }, + ]); + + Logger.AssertEqual( + 1, + result.installedIDs.length, + "Exactly 1 add-on was installed." + ); + Logger.AssertEqual( + this.id, + result.installedIDs[0], + "Add-on was installed successfully: " + this.id + ); + }, + + async setEnabled(flag) { + Logger.AssertTrue(await this.find(), "Add-on is available."); + + let userDisabled; + if (flag == STATE_ENABLED) { + userDisabled = false; + } else if (flag == STATE_DISABLED) { + userDisabled = true; + } else { + throw new Error("Unknown flag to setEnabled: " + flag); + } + + AddonUtils.updateUserDisabled(this.addon, userDisabled); + + return true; + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs new file mode 100644 index 0000000000..a7724c6aaa --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs @@ -0,0 +1,1063 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file was moved to tps from the main production code as it was unused +// after removal of the non-mirror bookmarks engine. +// It used to have a test before it was moved: +// https://searchfox.org/mozilla-central/rev/b1a5802e0f73bfd6d2096e5fefc2b47831a50b2d/services/sync/tests/unit/test_bookmark_validator.js + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +const QUERY_PROTOCOL = "place:"; + +function areURLsEqual(a, b) { + if (a === b) { + return true; + } + if (a.startsWith(QUERY_PROTOCOL) != b.startsWith(QUERY_PROTOCOL)) { + return false; + } + // Tag queries are special because we rewrite them to point to the + // local tag folder ID. It's expected that the folders won't match, + // but all other params should. + let aParams = new URLSearchParams(a.slice(QUERY_PROTOCOL.length)); + let aType = +aParams.get("type"); + if (aType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + return false; + } + let bParams = new URLSearchParams(b.slice(QUERY_PROTOCOL.length)); + let bType = +bParams.get("type"); + if (bType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + return false; + } + let aKeys = new Set(aParams.keys()); + let bKeys = new Set(bParams.keys()); + if (aKeys.size != bKeys.size) { + return false; + } + // Tag queries shouldn't reference multiple folders, or named folders like + // "TOOLBAR" or "BOOKMARKS_MENU". Just in case, we make sure all folder IDs + // are numeric. If they are, we ignore them when comparing the query params. + if (aKeys.has("folder") && aParams.getAll("folder").every(isFinite)) { + aKeys.delete("folder"); + } + if (bKeys.has("folder") && bParams.getAll("folder").every(isFinite)) { + bKeys.delete("folder"); + } + for (let key of aKeys) { + if (!bKeys.has(key)) { + return false; + } + if ( + !CommonUtils.arrayEqual( + aParams.getAll(key).sort(), + bParams.getAll(key).sort() + ) + ) { + return false; + } + } + for (let key of bKeys) { + if (!aKeys.has(key)) { + return false; + } + } + return true; +} + +const BOOKMARK_VALIDATOR_VERSION = 1; + +/** + * Result of bookmark validation. Contains the following fields which describe + * server-side problems unless otherwise specified. + * + * - missingIDs (number): # of objects with missing ids + * - duplicates (array of ids): ids seen more than once + * - parentChildMismatches (array of {parent: parentid, child: childid}): + * instances where the child's parentid and the parent's children array + * do not match + * - cycles (array of array of ids). List of cycles found in the server-side tree. + * - clientCycles (array of array of ids). List of cycles found in the client-side tree. + * - orphans (array of {id: string, parent: string}): List of nodes with + * either no parentid, or where the parent could not be found. + * - missingChildren (array of {parent: id, child: id}): + * List of parent/children where the child id couldn't be found + * - deletedChildren (array of { parent: id, child: id }): + * List of parent/children where child id was a deleted item (but still showed up + * in the children array) + * - multipleParents (array of {child: id, parents: array of ids}): + * List of children that were part of multiple parent arrays + * - deletedParents (array of ids) : List of records that aren't deleted but + * had deleted parents + * - childrenOnNonFolder (array of ids): list of non-folders that still have + * children arrays + * - duplicateChildren (array of ids): list of records who have the same + * child listed multiple times in their children array + * - parentNotFolder (array of ids): list of records that have parents that + * aren't folders + * - rootOnServer (boolean): true if the root came from the server + * - badClientRoots (array of ids): Contains any client-side root ids where + * the root is missing or isn't a (direct) child of the places root. + * + * - clientMissing: Array of ids on the server missing from the client + * - serverMissing: Array of ids on the client missing from the server + * - serverDeleted: Array of ids on the client that the server had marked as deleted. + * - serverUnexpected: Array of ids that appear on the server but shouldn't + * because the client attempts to never upload them. + * - differences: Array of {id: string, differences: string array} recording + * the non-structural properties that are differente between the client and server + * - structuralDifferences: As above, but contains the items where the differences were + * structural, that is, they contained childGUIDs or parentid + */ +export class BookmarkProblemData { + constructor() { + this.rootOnServer = false; + this.missingIDs = 0; + + this.duplicates = []; + this.parentChildMismatches = []; + this.cycles = []; + this.clientCycles = []; + this.orphans = []; + this.missingChildren = []; + this.deletedChildren = []; + this.multipleParents = []; + this.deletedParents = []; + this.childrenOnNonFolder = []; + this.duplicateChildren = []; + this.parentNotFolder = []; + + this.badClientRoots = []; + this.clientMissing = []; + this.serverMissing = []; + this.serverDeleted = []; + this.serverUnexpected = []; + this.differences = []; + this.structuralDifferences = []; + } + + /** + * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into + * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc. + */ + _summarizeDifferences(prefix, diffs) { + let diffCounts = new Map(); + for (let { differences } of diffs) { + for (let type of differences) { + let name = prefix + ":" + type; + let count = diffCounts.get(name) || 0; + diffCounts.set(name, count + 1); + } + } + return [...diffCounts].map(([name, count]) => ({ name, count })); + } + + /** + * Produce a list summarizing problems found. Each entry contains {name, count}, + * where name is the field name for the problem, and count is the number of times + * the problem was encountered. + * + * Validation has failed if all counts are not 0. + * + * If the `full` argument is truthy, we also include information about which + * properties we saw structural differences in. Currently, this means either + * "sdiff:parentid" and "sdiff:childGUIDS" may be present. + */ + getSummary(full) { + let result = [ + { name: "clientMissing", count: this.clientMissing.length }, + { name: "serverMissing", count: this.serverMissing.length }, + { name: "serverDeleted", count: this.serverDeleted.length }, + { name: "serverUnexpected", count: this.serverUnexpected.length }, + + { + name: "structuralDifferences", + count: this.structuralDifferences.length, + }, + { name: "differences", count: this.differences.length }, + + { name: "missingIDs", count: this.missingIDs }, + { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 }, + + { name: "duplicates", count: this.duplicates.length }, + { + name: "parentChildMismatches", + count: this.parentChildMismatches.length, + }, + { name: "cycles", count: this.cycles.length }, + { name: "clientCycles", count: this.clientCycles.length }, + { name: "badClientRoots", count: this.badClientRoots.length }, + { name: "orphans", count: this.orphans.length }, + { name: "missingChildren", count: this.missingChildren.length }, + { name: "deletedChildren", count: this.deletedChildren.length }, + { name: "multipleParents", count: this.multipleParents.length }, + { name: "deletedParents", count: this.deletedParents.length }, + { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length }, + { name: "duplicateChildren", count: this.duplicateChildren.length }, + { name: "parentNotFolder", count: this.parentNotFolder.length }, + ]; + if (full) { + let structural = this._summarizeDifferences( + "sdiff", + this.structuralDifferences + ); + result.push.apply(result, structural); + } + return result; + } +} + +// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon. +ChromeUtils.defineLazyGetter(lazy, "SYNCED_ROOTS", () => [ + lazy.PlacesUtils.bookmarks.menuGuid, + lazy.PlacesUtils.bookmarks.toolbarGuid, + lazy.PlacesUtils.bookmarks.unfiledGuid, + lazy.PlacesUtils.bookmarks.mobileGuid, +]); + +// Maps root GUIDs to their query folder names from +// toolkit/components/places/nsNavHistoryQuery.cpp. We follow queries that +// reference existing folders in the client tree, and detect cycles where a +// query references its containing folder. +ChromeUtils.defineLazyGetter(lazy, "ROOT_GUID_TO_QUERY_FOLDER_NAME", () => ({ + [lazy.PlacesUtils.bookmarks.rootGuid]: "PLACES_ROOT", + [lazy.PlacesUtils.bookmarks.menuGuid]: "BOOKMARKS_MENU", + + // Tags should never show up in our client tree, and never form cycles, but we + // report them just in case. + [lazy.PlacesUtils.bookmarks.tagsGuid]: "TAGS", + + [lazy.PlacesUtils.bookmarks.unfiledGuid]: "UNFILED_BOOKMARKS", + [lazy.PlacesUtils.bookmarks.toolbarGuid]: "TOOLBAR", + [lazy.PlacesUtils.bookmarks.mobileGuid]: "MOBILE_BOOKMARKS", +})); + +async function detectCycles(records) { + // currentPath and pathLookup contain the same data. pathLookup is faster to + // query, but currentPath gives is the order of traversal that we need in + // order to report the members of the cycles. + let pathLookup = new Set(); + let currentPath = []; + let cycles = []; + let seenEver = new Set(); + const yieldState = lazy.Async.yieldState(); + + const traverse = async node => { + if (pathLookup.has(node)) { + let cycleStart = currentPath.lastIndexOf(node); + let cyclePath = currentPath.slice(cycleStart).map(n => n.id); + cycles.push(cyclePath); + return; + } else if (seenEver.has(node)) { + // If we're checking the server, this is a problem, but it should already be reported. + // On the client, this could happen due to including `node.concrete` in the child list. + return; + } + seenEver.add(node); + let children = node.children || []; + if (node.concreteItems) { + children.push(...node.concreteItems); + } + if (children.length) { + pathLookup.add(node); + currentPath.push(node); + await lazy.Async.yieldingForEach(children, traverse, yieldState); + currentPath.pop(); + pathLookup.delete(node); + } + }; + + await lazy.Async.yieldingForEach( + records, + async record => { + if (!seenEver.has(record)) { + await traverse(record); + } + }, + yieldState + ); + + return cycles; +} + +class ServerRecordInspection { + constructor() { + this.serverRecords = null; + this.liveRecords = []; + + this.folders = []; + + this.root = null; + + this.idToRecord = new Map(); + + this.deletedIds = new Set(); + this.deletedRecords = []; + + this.problemData = new BookmarkProblemData(); + + // These are handled outside of problemData + this._orphans = new Map(); + this._multipleParents = new Map(); + + this.yieldState = lazy.Async.yieldState(); + } + + static async create(records) { + return new ServerRecordInspection().performInspection(records); + } + + async performInspection(records) { + await this._setRecords(records); + await this._linkParentIDs(); + await this._linkChildren(); + await this._findOrphans(); + await this._finish(); + return this; + } + + // We don't set orphans in this.problemData. Instead, we walk the tree at the + // end to find unreachable items. + _noteOrphan(id, parentId = undefined) { + // This probably shouldn't be called with a parentId twice, but if it + // happens we take the most recent one. + if (parentId || !this._orphans.has(id)) { + this._orphans.set(id, parentId); + } + } + + noteParent(child, parent) { + let parents = this._multipleParents.get(child); + if (!parents) { + this._multipleParents.set(child, [parent]); + } else { + parents.push(parent); + } + } + + noteMismatch(child, parent) { + let exists = this.problemData.parentChildMismatches.some( + match => match.child == child && match.parent == parent + ); + if (!exists) { + this.problemData.parentChildMismatches.push({ child, parent }); + } + } + + // - Populates `this.deletedIds`, `this.folders`, and `this.idToRecord` + // - calls `_initRoot` (thus initializing `this.root`). + async _setRecords(records) { + if (this.serverRecords) { + // In general this class is expected to be created, have + // `performInspection` called, and then only read from from that point on. + throw new Error("Bug: ServerRecordInspection can't `setRecords` twice"); + } + this.serverRecords = records; + let rootChildren = []; + + await lazy.Async.yieldingForEach( + this.serverRecords, + async record => { + if (!record.id) { + ++this.problemData.missingIDs; + return; + } + + if (record.deleted) { + this.deletedIds.add(record.id); + } + if (this.idToRecord.has(record.id)) { + this.problemData.duplicates.push(record.id); + return; + } + + this.idToRecord.set(record.id, record); + + if (!record.deleted) { + this.liveRecords.push(record); + + if (record.parentid == "places") { + rootChildren.push(record); + } + } + + if (!record.children) { + return; + } + + if (record.type != "folder") { + // Due to implementation details in engines/bookmarks.js, (Livemark + // subclassing BookmarkFolder) Livemarks will have a children array, + // but it should still be empty. + if (!record.children.length) { + return; + } + // Otherwise we mark it as an error and still try to resolve the children + this.problemData.childrenOnNonFolder.push(record.id); + } + + this.folders.push(record); + + if (new Set(record.children).size !== record.children.length) { + this.problemData.duplicateChildren.push(record.id); + } + + // After we're through with them, folder records store 3 (ugh) arrays that + // represent their folder information. The final fields looks like: + // + // - childGUIDs: The original `children` array, which is an array of + // record IDs. + // + // - unfilteredChildren: Contains more or less `childGUIDs.map(id => + // idToRecord.get(id))`, without the nulls for missing children. It will + // still have deleted, duplicate, mismatching, etc. children. + // + // - children: This is the 'cleaned' version of the child records that are + // safe to iterate over, etc.. If there are no reported problems, it should + // be identical to unfilteredChildren. + // + // The last two are left alone until later `this._linkChildren`, however. + record.childGUIDs = record.children; + + await lazy.Async.yieldingForEach( + record.childGUIDs, + id => { + this.noteParent(id, record.id); + }, + this.yieldState + ); + + record.children = []; + }, + this.yieldState + ); + + // Finish up some parts we can easily do now that we have idToRecord. + this.deletedRecords = Array.from(this.deletedIds, id => + this.idToRecord.get(id) + ); + + this._initRoot(rootChildren); + } + + _initRoot(rootChildren) { + let serverRoot = this.idToRecord.get("places"); + if (serverRoot) { + this.root = serverRoot; + this.problemData.rootOnServer = true; + return; + } + + // Fabricate a root. We want to be able to remember that it's fake, but + // would like to avoid it needing too much special casing, so we come up + // with children for it too (we just get these while we're iterating over + // the records to avoid needing two passes over a potentially large number + // of records). + + this.root = { + id: "places", + fake: true, + children: rootChildren, + childGUIDs: rootChildren.map(record => record.id), + type: "folder", + title: "", + }; + this.liveRecords.push(this.root); + this.idToRecord.set("places", this.root); + } + + // Adds `parent` to all records it can that have `parentid` + async _linkParentIDs() { + await lazy.Async.yieldingForEach( + this.idToRecord, + ([id, record]) => { + if (record == this.root || record.deleted) { + return false; + } + + // Check and update our orphan map. + let parentID = record.parentid; + let parent = this.idToRecord.get(parentID); + if (!parentID || !parent) { + this._noteOrphan(id, parentID); + return false; + } + + record.parent = parent; + + if (parent.deleted) { + this.problemData.deletedParents.push(id); + return true; + } else if (parent.type != "folder") { + this.problemData.parentNotFolder.push(record.id); + return true; + } + + if (parent.id !== "place" || this.problemData.rootOnServer) { + if (!parent.childGUIDs.includes(record.id)) { + this.noteMismatch(record.id, parent.id); + } + } + + if (parent.deleted && !record.deleted) { + this.problemData.deletedParents.push(record.id); + } + + // Note: We used to check if the parentName on the server matches the + // actual local parent name, but given this is used only for de-duping a + // record the first time it is seen and expensive to keep up-to-date, we + // decided to just stop recording it. See bug 1276969 for more. + return false; + }, + this.yieldState + ); + } + + // Build the children and unfilteredChildren arrays, (which are of record + // objects, not ids) + async _linkChildren() { + // Check that we aren't missing any children. + await lazy.Async.yieldingForEach( + this.folders, + async folder => { + folder.children = []; + folder.unfilteredChildren = []; + + let idsThisFolder = new Set(); + + await lazy.Async.yieldingForEach( + folder.childGUIDs, + childID => { + let child = this.idToRecord.get(childID); + + if (!child) { + this.problemData.missingChildren.push({ + parent: folder.id, + child: childID, + }); + return; + } + + if (child.deleted) { + this.problemData.deletedChildren.push({ + parent: folder.id, + child: childID, + }); + return; + } + + if (child.parentid != folder.id) { + this.noteMismatch(childID, folder.id); + return; + } + + if (idsThisFolder.has(childID)) { + // Already recorded earlier, we just don't want to mess up `children` + return; + } + folder.children.push(child); + }, + this.yieldState + ); + }, + this.yieldState + ); + } + + // Finds the orphans in the tree using something similar to a `mark and sweep` + // strategy. That is, we iterate over the children from the root, remembering + // which items we've seen. Then, we iterate all items, and know the ones we + // haven't seen are orphans. + async _findOrphans() { + let seen = new Set([this.root.id]); + + const inCycle = await lazy.Async.yieldingForEach( + Utils.walkTree(this.root), + ([node]) => { + if (seen.has(node.id)) { + // We're in an infloop due to a cycle. + // Return early to avoid reporting false positives for orphans. + return true; + } + seen.add(node.id); + + return false; + }, + this.yieldState + ); + + if (inCycle) { + return; + } + + await lazy.Async.yieldingForEach( + this.liveRecords, + (record, i) => { + if (!seen.has(record.id)) { + // We intentionally don't record the parentid here, since we only record + // that if the record refers to a parent that doesn't exist, which we + // have already handled (when linking parentid's). + this._noteOrphan(record.id); + } + }, + this.yieldState + ); + + await lazy.Async.yieldingForEach( + this._orphans, + ([id, parent]) => { + this.problemData.orphans.push({ id, parent }); + }, + this.yieldState + ); + } + + async _finish() { + this.problemData.cycles = await detectCycles(this.liveRecords); + + for (const [child, recordedParents] of this._multipleParents) { + let parents = new Set(recordedParents); + if (parents.size > 1) { + this.problemData.multipleParents.push({ child, parents: [...parents] }); + } + } + // Dedupe simple arrays in the problem data, so that we don't have to worry + // about it in the code + const idArrayProps = [ + "duplicates", + "deletedParents", + "childrenOnNonFolder", + "duplicateChildren", + "parentNotFolder", + ]; + for (let prop of idArrayProps) { + this.problemData[prop] = [...new Set(this.problemData[prop])]; + } + } +} + +export class BookmarkValidator { + constructor() { + this.yieldState = lazy.Async.yieldState(); + } + + async canValidate() { + return !(await lazy.PlacesSyncUtils.bookmarks.havePendingChanges()); + } + + async _followQueries(recordsByQueryId) { + await lazy.Async.yieldingForEach( + recordsByQueryId.values(), + entry => { + if ( + entry.type !== "query" && + (!entry.bmkUri || !entry.bmkUri.startsWith(QUERY_PROTOCOL)) + ) { + return; + } + let params = new URLSearchParams( + entry.bmkUri.slice(QUERY_PROTOCOL.length) + ); + // Queries with `excludeQueries` won't form cycles because they'll + // exclude all queries, including themselves, from the result set. + let excludeQueries = params.get("excludeQueries"); + if (excludeQueries === "1" || excludeQueries === "true") { + // `nsNavHistoryQuery::ParseQueryBooleanString` allows `1` and `true`. + return; + } + entry.concreteItems = []; + let queryIds = params.getAll("folder"); + for (let queryId of queryIds) { + let concreteItem = recordsByQueryId.get(queryId); + if (concreteItem) { + entry.concreteItems.push(concreteItem); + } + } + }, + this.yieldState + ); + } + + async createClientRecordsFromTree(clientTree) { + // Iterate over the treeNode, converting it to something more similar to what + // the server stores. + let records = []; + // A map of local IDs and well-known query folder names to records. Unlike + // GUIDs, local IDs aren't synced, since they're not stable across devices. + // New Places APIs use GUIDs to refer to bookmarks, but the legacy APIs + // still use local IDs. We use this mapping to parse `place:` queries that + // refer to folders via their local IDs. + let recordsByQueryId = new Map(); + let syncedRoots = lazy.SYNCED_ROOTS; + + const traverse = async (treeNode, synced) => { + if (!synced) { + synced = syncedRoots.includes(treeNode.guid); + } + let localId = treeNode.id; + let guid = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(treeNode.guid); + let itemType = "item"; + treeNode.ignored = !synced; + treeNode.id = guid; + switch (treeNode.type) { + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE: + if (treeNode.uri.startsWith(QUERY_PROTOCOL)) { + itemType = "query"; + } else { + itemType = "bookmark"; + } + break; + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: + let isLivemark = false; + if (treeNode.annos) { + for (let anno of treeNode.annos) { + if (anno.name === lazy.PlacesUtils.LMANNO_FEEDURI) { + isLivemark = true; + treeNode.feedUri = anno.value; + } else if (anno.name === lazy.PlacesUtils.LMANNO_SITEURI) { + isLivemark = true; + treeNode.siteUri = anno.value; + } + } + } + itemType = isLivemark ? "livemark" : "folder"; + break; + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: + itemType = "separator"; + break; + } + + if (treeNode.tags) { + treeNode.tags = treeNode.tags.split(","); + } else { + treeNode.tags = []; + } + treeNode.type = itemType; + treeNode.pos = treeNode.index; + treeNode.bmkUri = treeNode.uri; + records.push(treeNode); + if (treeNode.guid in lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME) { + let queryId = lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME[treeNode.guid]; + recordsByQueryId.set(queryId, treeNode); + } + if (localId) { + // Always add the ID, since it's still possible for a query to + // reference a root without using the well-known name. For example, + // `place:folder=${PlacesUtils.mobileFolderId}` and + // `place:folder=MOBILE_BOOKMARKS` are equivalent. + recordsByQueryId.set(localId.toString(10), treeNode); + } + if (treeNode.type === "folder") { + treeNode.childGUIDs = []; + if (!treeNode.children) { + treeNode.children = []; + } + + await lazy.Async.yieldingForEach( + treeNode.children, + async child => { + await traverse(child, synced); + child.parent = treeNode; + child.parentid = guid; + treeNode.childGUIDs.push(child.guid); + }, + this.yieldState + ); + } + }; + + await traverse(clientTree, false); + + clientTree.id = "places"; + await this._followQueries(recordsByQueryId); + return records; + } + + /** + * Process the server-side list. Mainly this builds the records into a tree, + * but it also records information about problems, and produces arrays of the + * deleted and non-deleted nodes. + * + * Returns an object containing: + * - records:Array of non-deleted records. Each record contains the following + * properties + * - childGUIDs (array of strings, only present if type is 'folder'): the + * list of child GUIDs stored on the server. + * - children (array of records, only present if type is 'folder'): + * each record has these same properties. This may differ in content + * from what you may expect from the childGUIDs list, as it won't + * contain any records that could not be found. + * - parent (record): The parent to this record. + * - Unchanged properties send down from the server: id, title, type, + * parentName, parentid, bmkURI, keyword, tags, pos, queryId + * - root: Root of the server-side bookmark tree. Has the same properties as + * above. + * - deletedRecords: As above, but only contains items that the server sent + * where it also sent indication that the item should be deleted. + * - problemData: a BookmarkProblemData object, with the caveat that + * the fields describing client/server relationship will not have been filled + * out yet. + */ + async inspectServerRecords(serverRecords) { + const data = await ServerRecordInspection.create(serverRecords); + return { + deletedRecords: data.deletedRecords, + records: data.liveRecords, + problemData: data.problemData, + root: data.root, + }; + } + + // Perform client-side sanity checking that doesn't involve server data + async _validateClient(problemData, clientRecords) { + problemData.clientCycles = await detectCycles(clientRecords); + for (let rootGUID of lazy.SYNCED_ROOTS) { + let record = clientRecords.find(record => record.guid === rootGUID); + if (!record || record.parentid !== "places") { + problemData.badClientRoots.push(rootGUID); + } + } + } + + async _computeUnifiedRecordMap(serverRecords, clientRecords) { + let allRecords = new Map(); + await lazy.Async.yieldingForEach( + serverRecords, + sr => { + if (sr.fake) { + return; + } + allRecords.set(sr.id, { client: null, server: sr }); + }, + this.yieldState + ); + + await lazy.Async.yieldingForEach( + clientRecords, + cr => { + let unified = allRecords.get(cr.id); + if (!unified) { + allRecords.set(cr.id, { client: cr, server: null }); + } else { + unified.client = cr; + } + }, + this.yieldState + ); + + return allRecords; + } + + _recordMissing(problems, id, clientRecord, serverRecord, serverTombstones) { + if (!clientRecord && serverRecord) { + problems.clientMissing.push(id); + } + if (!serverRecord && clientRecord) { + if (serverTombstones.has(id)) { + problems.serverDeleted.push(id); + } else if (!clientRecord.ignored && clientRecord.id != "places") { + problems.serverMissing.push(id); + } + } + } + + _compareRecords(client, server) { + let structuralDifferences = []; + let differences = []; + + // Don't bother comparing titles of roots. It's okay if locally it's + // "Mobile Bookmarks", but the server thinks it's "mobile". + // TODO: We probably should be handing other localized bookmarks (e.g. + // default bookmarks) here as well, see bug 1316041. + if (!lazy.SYNCED_ROOTS.includes(client.guid)) { + // We want to treat undefined, null and an empty string as identical + if ((client.title || "") !== (server.title || "")) { + differences.push("title"); + } + } + + if (client.parentid || server.parentid) { + if (client.parentid !== server.parentid) { + structuralDifferences.push("parentid"); + } + } + + if (client.tags || server.tags) { + let cl = client.tags ? [...client.tags].sort() : []; + let sl = server.tags ? [...server.tags].sort() : []; + if (!CommonUtils.arrayEqual(cl, sl)) { + differences.push("tags"); + } + } + + let sameType = client.type === server.type; + if (!sameType) { + if ( + server.type === "query" && + client.type === "bookmark" && + client.bmkUri.startsWith(QUERY_PROTOCOL) + ) { + sameType = true; + } + } + + if (!sameType) { + differences.push("type"); + } else { + switch (server.type) { + case "bookmark": + case "query": + if (!areURLsEqual(server.bmkUri, client.bmkUri)) { + differences.push("bmkUri"); + } + break; + case "separator": + if (server.pos != client.pos) { + differences.push("pos"); + } + break; + case "livemark": + if (server.feedUri != client.feedUri) { + differences.push("feedUri"); + } + if (server.siteUri != client.siteUri) { + differences.push("siteUri"); + } + break; + case "folder": + if (server.id === "places" && server.fake) { + // It's the fabricated places root. It won't have the GUIDs, but + // it doesn't matter. + break; + } + if (client.childGUIDs || server.childGUIDs) { + let cl = client.childGUIDs || []; + let sl = server.childGUIDs || []; + if (!CommonUtils.arrayEqual(cl, sl)) { + structuralDifferences.push("childGUIDs"); + } + } + break; + } + } + return { differences, structuralDifferences }; + } + + /** + * Compare the list of server records with the client tree. + * + * Returns the same data as described in the inspectServerRecords comment, + * with the following additional fields. + * - clientRecords: an array of client records in a similar format to + * the .records (ie, server records) entry. + * - problemData is the same as for inspectServerRecords, except all properties + * will be filled out. + */ + async compareServerWithClient(serverRecords, clientTree) { + let clientRecords = await this.createClientRecordsFromTree(clientTree); + let inspectionInfo = await this.inspectServerRecords(serverRecords); + inspectionInfo.clientRecords = clientRecords; + + // Mainly do this to remove deleted items and normalize child guids. + serverRecords = inspectionInfo.records; + let problemData = inspectionInfo.problemData; + + await this._validateClient(problemData, clientRecords); + + let allRecords = await this._computeUnifiedRecordMap( + serverRecords, + clientRecords + ); + + let serverDeleted = new Set(inspectionInfo.deletedRecords.map(r => r.id)); + + await lazy.Async.yieldingForEach( + allRecords, + ([id, { client, server }]) => { + if (!client || !server) { + this._recordMissing(problemData, id, client, server, serverDeleted); + return; + } + if (server && client && client.ignored) { + problemData.serverUnexpected.push(id); + } + let { differences, structuralDifferences } = this._compareRecords( + client, + server + ); + + if (differences.length) { + problemData.differences.push({ id, differences }); + } + if (structuralDifferences.length) { + problemData.structuralDifferences.push({ + id, + differences: structuralDifferences, + }); + } + }, + this.yieldState + ); + + return inspectionInfo; + } + + async _getServerState(engine) { + let collection = engine.itemSource(); + let collectionKey = engine.service.collectionKeys.keyForCollection( + engine.name + ); + collection.full = true; + let result = await collection.getBatched(); + if (!result.response.success) { + throw result.response; + } + let cleartexts = []; + await lazy.Async.yieldingForEach( + result.records, + async record => { + await record.decrypt(collectionKey); + cleartexts.push(record.cleartext); + }, + this.yieldState + ); + return cleartexts; + } + + async validate(engine) { + let start = Date.now(); + let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", { + includeItemIds: true, + }); + let serverState = await this._getServerState(engine); + let serverRecordCount = serverState.length; + let result = await this.compareServerWithClient(serverState, clientTree); + let end = Date.now(); + let duration = end - start; + + engine._log.debug(`Validated bookmarks in ${duration}ms`); + engine._log.debug(`Problem summary`); + for (let { name, count } of result.problemData.getSummary()) { + engine._log.debug(` ${name}: ${count}`); + } + + return { + duration, + version: this.version, + problems: result.problemData, + recordCount: serverRecordCount, + }; + } +} + +BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION; diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs new file mode 100644 index 0000000000..e4aac948b5 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs @@ -0,0 +1,833 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { PlacesBackups } from "resource://gre/modules/PlacesBackups.sys.mjs"; + +import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs"; +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +export async function DumpBookmarks() { + let [bookmarks] = await PlacesBackups.getBookmarksTree(); + Logger.logInfo( + "Dumping Bookmarks...\n" + JSON.stringify(bookmarks, undefined, 2) + "\n\n" + ); +} + +/** + * extend, causes a child object to inherit from a parent + */ +function extend(child, supertype) { + Object.setPrototypeOf(child.prototype, supertype.prototype); +} +/** + * PlacesItemProps object, holds properties for places items + */ +function PlacesItemProps(props) { + this.location = null; + this.uri = null; + this.keyword = null; + this.title = null; + this.after = null; + this.before = null; + this.folder = null; + this.position = null; + this.delete = false; + this.tags = null; + this.last_item_pos = null; + this.type = null; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * PlacesItem object. Base class for places items. + */ +export function PlacesItem(props) { + this.props = new PlacesItemProps(props); + if (this.props.location == null) { + this.props.location = "menu"; + } + if ("changes" in props) { + this.updateProps = new PlacesItemProps(props.changes); + } else { + this.updateProps = null; + } +} + +/** + * Instance methods for generic places items. + */ +PlacesItem.prototype = { + // an array of possible root folders for places items + _bookmarkFolders: { + places: PlacesUtils.bookmarks.rootGuid, + menu: PlacesUtils.bookmarks.menuGuid, + tags: PlacesUtils.bookmarks.tagsGuid, + unfiled: PlacesUtils.bookmarks.unfiledGuid, + toolbar: PlacesUtils.bookmarks.toolbarGuid, + mobile: PlacesUtils.bookmarks.mobileGuid, + }, + + _typeMap: new Map([ + [PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER, PlacesUtils.bookmarks.TYPE_FOLDER], + [ + PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR, + PlacesUtils.bookmarks.TYPE_SEPARATOR, + ], + [PlacesUtils.TYPE_X_MOZ_PLACE, PlacesUtils.bookmarks.TYPE_BOOKMARK], + ]), + + toString() { + var that = this; + var props = ["uri", "title", "location", "folder"]; + var string = + (this.props.type ? this.props.type + " " : "") + + "(" + + (function () { + var ret = []; + for (var i in props) { + if (that.props[props[i]]) { + ret.push(props[i] + ": " + that.props[props[i]]); + } + } + return ret; + })().join(", ") + + ")"; + return string; + }, + + /** + * GetPlacesChildGuid + * + * Finds the guid of the an item with the specified properties in the places + * database under the specified parent. + * + * @param folder The guid of the folder to search + * @param type The type of the item to find, or null to match any item; + * this is one of the PlacesUtils.bookmarks.TYPE_* values + * @param title The title of the item to find, or null to match any title + * @param uri The uri of the item to find, or null to match any uri + * + * @return the node id if the item was found, otherwise null + */ + async GetPlacesChildGuid(folder, type, title, uri) { + let children = (await PlacesUtils.promiseBookmarksTree(folder)).children; + if (!children) { + return null; + } + let guid = null; + for (let node of children) { + if (node.title == title) { + let nodeType = this._typeMap.get(node.type); + if (type == null || type == undefined || nodeType == type) { + if (uri == undefined || uri == null || node.uri.spec == uri.spec) { + // Note that this is suspect as we return the *last* matching + // child, which some tests rely on (ie, an early-return here causes + // at least 1 test to fail). But that's a yak for another day. + guid = node.guid; + } + } + } + } + return guid; + }, + + /** + * IsAdjacentTo + * + * Determines if this object is immediately adjacent to another. + * + * @param itemName The name of the other object; this may be any kind of + * places item + * @param relativePos The relative position of the other object. If -1, + * it means the other object should precede this one, if +1, + * the other object should come after this one + * @return true if this object is immediately adjacent to the other object, + * otherwise false + */ + async IsAdjacentTo(itemName, relativePos) { + Logger.AssertTrue( + this.props.folder_id != -1 && this.props.guid != null, + "Either folder_id or guid was invalid" + ); + let otherGuid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + itemName + ); + Logger.AssertTrue(otherGuid, "item " + itemName + " not found"); + let other_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index; + let this_pos = (await PlacesUtils.bookmarks.fetch(this.props.guid)).index; + if (other_pos + relativePos != this_pos) { + Logger.logPotentialError( + "Invalid position - " + + (this.props.title ? this.props.title : this.props.folder) + + " not " + + (relativePos == 1 ? "after " : "before ") + + itemName + + " for " + + this.toString() + ); + return false; + } + return true; + }, + + /** + * GetItemIndex + * + * Gets the item index for this places item. + * + * @return the item index, or -1 if there's an error + */ + async GetItemIndex() { + if (this.props.guid == null) { + return -1; + } + return (await PlacesUtils.bookmarks.fetch(this.props.guid)).index; + }, + + /** + * GetFolder + * + * Gets the folder guid for the specified bookmark folder + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder guid if the folder is found, otherwise null + */ + async GetFolder(location) { + let folder_parts = location.split("/"); + if (!(folder_parts[0] in this._bookmarkFolders)) { + return null; + } + let folderGuid = this._bookmarkFolders[folder_parts[0]]; + for (let i = 1; i < folder_parts.length; i++) { + let guid = await this.GetPlacesChildGuid( + folderGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + folder_parts[i] + ); + if (guid == null) { + return null; + } + folderGuid = guid; + } + return folderGuid; + }, + + /** + * CreateFolder + * + * Creates a bookmark folder. + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder id if the folder was created, otherwise -1 + */ + async CreateFolder(location) { + let folder_parts = location.split("/"); + if (!(folder_parts[0] in this._bookmarkFolders)) { + return -1; + } + let folderGuid = this._bookmarkFolders[folder_parts[0]]; + for (let i = 1; i < folder_parts.length; i++) { + let subfolderGuid = await this.GetPlacesChildGuid( + folderGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + folder_parts[i] + ); + if (subfolderGuid == null) { + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: folderGuid, + name: folder_parts[i], + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + folderGuid = guid; + } else { + folderGuid = subfolderGuid; + } + } + return folderGuid; + }, + + /** + * GetOrCreateFolder + * + * Locates the specified folder; if not found it is created. + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder id if the folder was found or created, otherwise -1 + */ + async GetOrCreateFolder(location) { + let parentGuid = await this.GetFolder(location); + if (parentGuid == null) { + parentGuid = await this.CreateFolder(location); + } + return parentGuid; + }, + + /** + * CheckPosition + * + * Verifies the position of this places item. + * + * @param before The name of the places item that this item should be + before, or null if this check should be skipped + * @param after The name of the places item that this item should be + after, or null if this check should be skipped + * @param last_item_pos The index of the places item above this one, + * or null if this check should be skipped + * @return true if this item is in the correct position, otherwise false + */ + async CheckPosition(before, after, last_item_pos) { + if (after) { + if (!(await this.IsAdjacentTo(after, 1))) { + return false; + } + } + if (before) { + if (!(await this.IsAdjacentTo(before, -1))) { + return false; + } + } + if (last_item_pos != null && last_item_pos > -1) { + let index = await this.GetItemIndex(); + if (index != last_item_pos + 1) { + Logger.logPotentialError( + "Item not found at the expected index, got " + + index + + ", expected " + + (last_item_pos + 1) + + " for " + + this.toString() + ); + return false; + } + } + return true; + }, + + /** + * SetLocation + * + * Moves this places item to a different folder. + * + * @param location The full path of the folder to which to move this + * places item, which must begin with one of the bookmark root + * folders; if null, no changes are made + * @return nothing if successful, otherwise an exception is thrown + */ + async SetLocation(location) { + if (location != null) { + let newfolderGuid = await this.GetOrCreateFolder(location); + Logger.AssertTrue( + newfolderGuid, + "Location " + location + " doesn't exist; can't change item's location" + ); + await PlacesUtils.bookmarks.update({ + guid: this.props.guid, + parentGuid: newfolderGuid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + this.props.parentGuid = newfolderGuid; + } + }, + + /** + * SetPosition + * + * Updates the position of this places item within this item's current + * folder. Use SetLocation to change folders. + * + * @param position The new index this item should be moved to; if null, + * no changes are made; if -1, this item is moved to the bottom of + * the current folder. Otherwise, must be a string which is the + * title of an existing item in the folder, who's current position + * is used as the index. + * @return nothing if successful, otherwise an exception is thrown + */ + async SetPosition(position) { + if (position == null) { + return; + } + let index = -1; + if (position != -1) { + let existingGuid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + position + ); + if (existingGuid) { + index = (await PlacesUtils.bookmarks.fetch(existingGuid)).index; + } + Logger.AssertTrue( + index != -1, + "position " + position + " is invalid; unable to change position" + ); + } + await PlacesUtils.bookmarks.update({ guid: this.props.guid, index }); + }, + + /** + * Update the title of this places item + * + * @param title The new title to set for this item; if null, no changes + * are made + * @return nothing + */ + async SetTitle(title) { + if (title != null) { + await PlacesUtils.bookmarks.update({ guid: this.props.guid, title }); + } + }, +}; + +/** + * Bookmark class constructor. Initializes instance properties. + */ +export function Bookmark(props) { + PlacesItem.call(this, props); + if (this.props.title == null) { + this.props.title = this.props.uri; + } + this.props.type = "bookmark"; +} + +/** + * Bookmark instance methods. + */ +Bookmark.prototype = { + /** + * SetKeyword + * + * Update this bookmark's keyword. + * + * @param keyword The keyword to set for this bookmark; if null, no + * changes are made + * @return nothing + */ + async SetKeyword(keyword) { + if (keyword != null) { + // Mirror logic from PlacesSyncUtils's updateBookmarkMetadata + let entry = await PlacesUtils.keywords.fetch({ url: this.props.uri }); + if (entry) { + await PlacesUtils.keywords.remove(entry); + } + await PlacesUtils.keywords.insert({ keyword, url: this.props.uri }); + } + }, + + /** + * SetUri + * + * Updates this bookmark's URI. + * + * @param uri The new URI to set for this boomark; if null, no changes + * are made + * @return nothing + */ + async SetUri(uri) { + if (uri) { + let url = Services.io.newURI(uri); + await PlacesUtils.bookmarks.update({ guid: this.props.guid, url }); + } + }, + + /** + * SetTags + * + * Updates this bookmark's tags. + * + * @param tags An array of tags which should be associated with this + * bookmark; any previous tags are removed; if this param is null, + * no changes are made. If this param is an empty array, all + * tags are removed from this bookmark. + * @return nothing + */ + SetTags(tags) { + if (tags != null) { + let URI = Services.io.newURI(this.props.uri); + PlacesUtils.tagging.untagURI(URI, null); + if (tags.length) { + PlacesUtils.tagging.tagURI(URI, tags); + } + } + }, + + /** + * Create + * + * Creates the bookmark described by this object's properties. + * + * @return the id of the created bookmark + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "bookmark, error creating folder " + + this.props.location + ); + let bookmarkURI = Services.io.newURI(this.props.uri); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + url: bookmarkURI, + title: this.props.title, + }); + this.props.guid = guid; + await this.SetKeyword(this.props.keyword); + await this.SetTags(this.props.tags); + return this.props.guid; + }, + + /** + * Update + * + * Updates this bookmark's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetTitle(this.updateProps.title); + await this.SetUri(this.updateProps.uri); + await this.SetKeyword(this.updateProps.keyword); + await this.SetTags(this.updateProps.tags); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + }, + + /** + * Find + * + * Locates the bookmark which corresponds to this object's properties. + * + * @return the bookmark guid if the bookmark was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + let bookmarkTitle = this.props.title; + this.props.guid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + bookmarkTitle, + this.props.uri + ); + + if (!this.props.guid) { + Logger.logPotentialError(this.toString() + " not found"); + return null; + } + if (this.props.keyword != null) { + let { keyword } = await PlacesSyncUtils.bookmarks.fetch(this.props.guid); + if (keyword != this.props.keyword) { + Logger.logPotentialError( + "Incorrect keyword - expected: " + + this.props.keyword + + ", actual: " + + keyword + + " for " + + this.toString() + ); + return null; + } + } + if (this.props.tags != null) { + try { + let URI = Services.io.newURI(this.props.uri); + let tags = PlacesUtils.tagging.getTagsForURI(URI); + tags.sort(); + this.props.tags.sort(); + if (JSON.stringify(tags) != JSON.stringify(this.props.tags)) { + Logger.logPotentialError( + "Wrong tags - expected: " + + JSON.stringify(this.props.tags) + + ", actual: " + + JSON.stringify(tags) + + " for " + + this.toString() + ); + return null; + } + } catch (e) { + Logger.logPotentialError("error processing tags " + e); + return null; + } + } + if ( + !(await this.CheckPosition( + this.props.before, + this.props.after, + this.props.last_item_pos + )) + ) { + return null; + } + return this.props.guid; + }, + + /** + * Remove + * + * Removes this bookmark. The bookmark should have been located previously + * by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Remove"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, +}; + +extend(Bookmark, PlacesItem); + +/** + * BookmarkFolder class constructor. Initializes instance properties. + */ +export function BookmarkFolder(props) { + PlacesItem.call(this, props); + this.props.type = "folder"; +} + +/** + * BookmarkFolder instance methods + */ +BookmarkFolder.prototype = { + /** + * Create + * + * Creates the bookmark folder described by this object's properties. + * + * @return the id of the created bookmark folder + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "folder, error creating parent folder " + + this.props.location + ); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + title: this.props.folder, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + this.props.guid = guid; + return this.props.parentGuid; + }, + + /** + * Find + * + * Locates the bookmark folder which corresponds to this object's + * properties. + * + * @return the folder guid if the folder was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + this.props.guid = await this.GetPlacesChildGuid( + this.props.parentGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + this.props.folder + ); + if (this.props.guid == null) { + return null; + } + if ( + !(await this.CheckPosition( + this.props.before, + this.props.after, + this.props.last_item_pos + )) + ) { + return null; + } + return this.props.guid; + }, + + /** + * Remove + * + * Removes this folder. The folder should have been located previously + * by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Remove"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, + + /** + * Update + * + * Updates this bookmark's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + await this.SetTitle(this.updateProps.folder); + }, +}; + +extend(BookmarkFolder, PlacesItem); + +/** + * Separator class constructor. Initializes instance properties. + */ +export function Separator(props) { + PlacesItem.call(this, props); + this.props.type = "separator"; +} + +/** + * Separator instance methods. + */ +Separator.prototype = { + /** + * Create + * + * Creates the bookmark separator described by this object's properties. + * + * @return the id of the created separator + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "folder, error creating parent folder " + + this.props.location + ); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + this.props.guid = guid; + return guid; + }, + + /** + * Find + * + * Locates the bookmark separator which corresponds to this object's + * properties. + * + * @return the item guid if the separator was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + if (this.props.before == null && this.props.last_item_pos == null) { + Logger.logPotentialError( + "Separator requires 'before' attribute if it's the" + + "first item in the list" + ); + return null; + } + let expected_pos = -1; + if (this.props.before) { + let otherGuid = this.GetPlacesChildGuid( + this.props.parentGuid, + null, + this.props.before + ); + if (otherGuid == null) { + Logger.logPotentialError( + "Can't find places item " + + this.props.before + + " for locating separator" + ); + return null; + } + expected_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index - 1; + } else { + expected_pos = this.props.last_item_pos + 1; + } + // Note these are IDs instead of GUIDs. + let children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + this.props.parentGuid + ); + this.props.guid = children[expected_pos]; + if (this.props.guid == null) { + Logger.logPotentialError( + "No separator found at position " + expected_pos + ); + return null; + } + let info = await PlacesUtils.bookmarks.fetch(this.props.guid); + if (info.type != PlacesUtils.bookmarks.TYPE_SEPARATOR) { + Logger.logPotentialError( + "Places item at position " + expected_pos + " is not a separator" + ); + return null; + } + return this.props.guid; + }, + + /** + * Update + * + * Updates this separator's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + return true; + }, + + /** + * Remove + * + * Removes this separator. The separator should have been located + * previously by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, +}; + +extend(Separator, PlacesItem); diff --git a/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs new file mode 100644 index 0000000000..587d7668f4 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs @@ -0,0 +1,128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + OSKeyStore: "resource://gre/modules/OSKeyStore.sys.mjs", + formAutofillStorage: "resource://autofill/FormAutofillStorage.sys.mjs", +}); + +class FormAutofillBase { + constructor(props, subStorageName, fields) { + this._subStorageName = subStorageName; + this._fields = fields; + + this.props = {}; + this.updateProps = null; + if ("changes" in props) { + this.updateProps = props.changes; + } + for (const field of this._fields) { + this.props[field] = field in props ? props[field] : null; + } + } + + async getStorage() { + await lazy.formAutofillStorage.initialize(); + return lazy.formAutofillStorage[this._subStorageName]; + } + + async Create() { + const storage = await this.getStorage(); + await storage.add(this.props); + } + + async Find() { + const storage = await this.getStorage(); + return storage._data.find(entry => + this._fields.every(field => entry[field] === this.props[field]) + ); + } + + async Update() { + const storage = await this.getStorage(); + const { guid } = await this.Find(); + await storage.update(guid, this.updateProps, true); + } + + async Remove() { + const storage = await this.getStorage(); + const { guid } = await this.Find(); + storage.remove(guid); + } +} + +async function DumpStorage(subStorageName) { + await lazy.formAutofillStorage.initialize(); + Logger.logInfo(`\ndumping ${subStorageName} list\n`, true); + const entries = lazy.formAutofillStorage[subStorageName]._data; + for (const entry of entries) { + Logger.logInfo(JSON.stringify(entry), true); + } + Logger.logInfo(`\n\nend ${subStorageName} list\n`, true); +} + +const ADDRESS_FIELDS = [ + "given-name", + "additional-name", + "family-name", + "organization", + "street-address", + "address-level2", + "address-level1", + "postal-code", + "country", + "tel", + "email", +]; + +export class Address extends FormAutofillBase { + constructor(props) { + super(props, "addresses", ADDRESS_FIELDS); + } +} + +export async function DumpAddresses() { + await DumpStorage("addresses"); +} + +const CREDIT_CARD_FIELDS = [ + "cc-name", + "cc-number", + "cc-exp-month", + "cc-exp-year", +]; + +export class CreditCard extends FormAutofillBase { + constructor(props) { + super(props, "creditCards", CREDIT_CARD_FIELDS); + } + + async Find() { + const storage = await this.getStorage(); + await Promise.all( + storage._data.map( + async entry => + (entry["cc-number"] = await lazy.OSKeyStore.decrypt( + entry["cc-number-encrypted"] + )) + ) + ); + return storage._data.find(entry => { + return this._fields.every(field => entry[field] === this.props[field]); + }); + } +} + +export async function DumpCreditCards() { + await DumpStorage("creditCards"); +} diff --git a/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs new file mode 100644 index 0000000000..35b5f5c03b --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs @@ -0,0 +1,205 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. Only the following + listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +import { FormHistory } from "resource://gre/modules/FormHistory.sys.mjs"; + +/** + * FormDB + * + * Helper object containing methods to interact with the FormHistory module. + */ +var FormDB = { + async _update(data) { + await FormHistory.update(data); + }, + + /** + * insertValue + * + * Adds the specified value for the specified fieldname into form history. + * + * @param fieldname The form fieldname to insert + * @param value The form value to insert + * @param us The time, in microseconds, to use for the lastUsed + * and firstUsed columns + * @return Promise<undefined> + */ + insertValue(fieldname, value, us) { + let data = { + op: "add", + fieldname, + value, + timesUsed: 1, + firstUsed: us, + lastUsed: us, + }; + return this._update(data); + }, + + /** + * updateValue + * + * Updates a row in the moz_formhistory table with a new value. + * + * @param id The id of the row to update + * @param newvalue The new value to set + * @return Promise<undefined> + */ + updateValue(id, newvalue) { + return this._update({ op: "update", guid: id, value: newvalue }); + }, + + /** + * getDataForValue + * + * Retrieves a set of values for a row in the database that + * corresponds to the given fieldname and value. + * + * @param fieldname The fieldname of the row to query + * @param value The value of the row to query + * @return Promise<null if no row is found with the specified fieldname and value, + * or an object containing the row's guid, lastUsed, and firstUsed + * values> + */ + async getDataForValue(fieldname, value) { + let results = await FormHistory.search(["guid", "lastUsed", "firstUsed"], { + fieldname, + value, + }); + if (results.length > 1) { + throw new Error("more than 1 result for this query"); + } + return results; + }, + + /** + * remove + * + * Removes the specified GUID from the database. + * + * @param guid The guid of the item to delete + * @return Promise<> + */ + remove(guid) { + return this._update({ op: "remove", guid }); + }, +}; + +/** + * FormData class constructor + * + * Initializes instance properties. + */ +export function FormData(props, msSinceEpoch) { + this.fieldname = null; + this.value = null; + this.date = 0; + this.newvalue = null; + this.usSinceEpoch = msSinceEpoch * 1000; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * FormData instance methods + */ +FormData.prototype = { + /** + * hours_to_us + * + * Converts hours since present to microseconds since epoch. + * + * @param hours The number of hours since the present time (e.g., 0 is + * 'now', and -1 is 1 hour ago) + * @return the corresponding number of microseconds since the epoch + */ + hours_to_us(hours) { + return this.usSinceEpoch + hours * 60 * 60 * 1000 * 1000; + }, + + /** + * Create + * + * If this FormData object doesn't exist in the moz_formhistory database, + * add it. Throws on error. + * + * @return nothing + */ + Create() { + Logger.AssertTrue( + this.fieldname != null && this.value != null, + "Must specify both fieldname and value" + ); + + return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => { + if (!formdata) { + // this item doesn't exist yet in the db, so we need to insert it + return FormDB.insertValue( + this.fieldname, + this.value, + this.hours_to_us(this.date) + ); + } + /* Right now, we ignore this case. If bug 552531 is ever fixed, + we might need to add code here to update the firstUsed or + lastUsed fields, as appropriate. + */ + return null; + }); + }, + + /** + * Find + * + * Attempts to locate an entry in the moz_formhistory database that + * matches the fieldname and value for this FormData object. + * + * @return true if this entry exists in the database, otherwise false + */ + Find() { + return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => { + let status = formdata != null; + if (status) { + /* + //form history dates currently not synced! bug 552531 + let us = this.hours_to_us(this.date); + status = Logger.AssertTrue( + us >= formdata.firstUsed && us <= formdata.lastUsed, + "No match for with that date value"); + + if (status) + */ + this.id = formdata.guid; + } + return status; + }); + }, + + /** + * Remove + * + * Removes the row represented by this FormData instance from the + * moz_formhistory database. + * + * @return nothing + */ + async Remove() { + const formdata = await FormDB.getDataForValue(this.fieldname, this.value); + if (!formdata) { + return; + } + await FormDB.remove(formdata.guid); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs new file mode 100644 index 0000000000..845bab3aa9 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs @@ -0,0 +1,158 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +export var DumpHistory = async function TPS_History__DumpHistory() { + let query = PlacesUtils.history.getNewQuery(); + let options = PlacesUtils.history.getNewQueryOptions(); + let root = PlacesUtils.history.executeQuery(query, options).root; + root.containerOpen = true; + Logger.logInfo("\n\ndumping history\n", true); + for (var i = 0; i < root.childCount; i++) { + let node = root.getChild(i); + let uri = node.uri; + let guid = await PlacesSyncUtils.history + .fetchGuidForURL(uri) + .catch(() => "?".repeat(12)); + let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(uri); + for (var visit of curvisits) { + Logger.logInfo( + `GUID: ${guid}, URI: ${uri}, type=${visit.type}, date=${visit.date}`, + true + ); + } + } + root.containerOpen = false; + Logger.logInfo("\nend history dump\n", true); +}; + +/** + * HistoryEntry object + * + * Contains methods for manipulating browser history entries. + */ +export var HistoryEntry = { + /** + * Add + * + * Adds visits for a uri to the history database. Throws on error. + * + * @param item An object representing one or more visits to a specific uri + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return nothing + */ + async Add(item, msSinceEpoch) { + Logger.AssertTrue( + "visits" in item && "uri" in item, + "History entry in test file must have both 'visits' " + + "and 'uri' properties" + ); + let place = { + url: item.uri, + visits: [], + }; + for (let visit of item.visits) { + let date = new Date( + Math.round(msSinceEpoch + visit.date * 60 * 60 * 1000) + ); + place.visits.push({ date, transition: visit.type }); + } + if ("title" in item) { + place.title = item.title; + } + return PlacesUtils.history.insert(place); + }, + + /** + * Find + * + * Finds visits for a uri to the history database. Throws on error. + * + * @param item An object representing one or more visits to a specific uri + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return true if all the visits for the uri are found, otherwise false + */ + async Find(item, msSinceEpoch) { + Logger.AssertTrue( + "visits" in item && "uri" in item, + "History entry in test file must have both 'visits' " + + "and 'uri' properties" + ); + let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(item.uri); + for (let visit of curvisits) { + for (let itemvisit of item.visits) { + // Note: in microseconds. + let expectedDate = + itemvisit.date * 60 * 60 * 1000 * 1000 + msSinceEpoch * 1000; + if (visit.type == itemvisit.type) { + if (itemvisit.date === undefined || visit.date == expectedDate) { + itemvisit.found = true; + } + } + } + } + + let all_items_found = true; + for (let itemvisit of item.visits) { + all_items_found = all_items_found && "found" in itemvisit; + Logger.logInfo( + `History entry for ${item.uri}, type: ${itemvisit.type}, date: ${itemvisit.date}` + + `(${ + itemvisit.date * 60 * 60 * 1000 * 1000 + }), found = ${!!itemvisit.found}` + ); + } + return all_items_found; + }, + + /** + * Delete + * + * Removes visits from the history database. Throws on error. + * + * @param item An object representing items to delete + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return nothing + */ + async Delete(item, msSinceEpoch) { + if ("uri" in item) { + let removedAny = await PlacesUtils.history.remove(item.uri); + if (!removedAny) { + Logger.log("Warning: Removed 0 history visits for uri " + item.uri); + } + } else if ("host" in item) { + await PlacesUtils.history.removeByFilter({ host: item.host }); + } else if ("begin" in item && "end" in item) { + let filter = { + beginDate: new Date(msSinceEpoch + item.begin * 60 * 60 * 1000), + endDate: new Date(msSinceEpoch + item.end * 60 * 60 * 1000), + }; + let removedAny = await PlacesUtils.history.removeVisitsByFilter(filter); + if (!removedAny) { + Logger.log( + "Warning: Removed 0 history visits with " + + JSON.stringify({ item, filter }) + ); + } + } else { + Logger.AssertTrue( + false, + "invalid entry in delete history " + JSON.stringify(item) + ); + } + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs new file mode 100644 index 0000000000..976755e989 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs @@ -0,0 +1,187 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +var nsLoginInfo = new Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +export var DumpPasswords = async function TPS__Passwords__DumpPasswords() { + let logins = await Services.logins.getAllLogins(); + Logger.logInfo("\ndumping password list\n", true); + for (var i = 0; i < logins.length; i++) { + Logger.logInfo( + "* origin=" + + logins[i].origin + + ", formActionOrigin=" + + logins[i].formActionOrigin + + ", realm=" + + logins[i].httpRealm + + ", password=" + + logins[i].password + + ", passwordField=" + + logins[i].passwordField + + ", username=" + + logins[i].username + + ", usernameField=" + + logins[i].usernameField, + true + ); + } + Logger.logInfo("\n\nend password list\n", true); +}; + +/** + * PasswordProps object; holds password properties. + */ +function PasswordProps(props) { + this.hostname = null; + this.submitURL = null; + this.realm = null; + this.username = ""; + this.password = ""; + this.usernameField = ""; + this.passwordField = ""; + this.delete = false; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * Password class constructor. Initializes instance properties. + */ +export function Password(props) { + this.props = new PasswordProps(props); + if ("changes" in props) { + this.updateProps = new PasswordProps(props); + for (var prop in props.changes) { + if (prop in this.updateProps) { + this.updateProps[prop] = props.changes[prop]; + } + } + } else { + this.updateProps = null; + } +} + +/** + * Password instance methods. + */ +Password.prototype = { + /** + * Create + * + * Adds a password entry to the login manager for the password + * represented by this object's properties. Throws on error. + * + * @return the new login guid + */ + async Create() { + let login = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + await Services.logins.addLoginAsync(login); + login.QueryInterface(Ci.nsILoginMetaInfo); + return login.guid; + }, + + /** + * Find + * + * Finds a password entry in the login manager, for the password + * represented by this object's properties. + * + * @return the guid of the password if found, otherwise -1 + */ + async Find() { + let logins = await Services.logins.searchLoginsAsync({ + origin: this.props.hostname, + formActionOrigin: this.props.submitURL, + httpRealm: this.props.realm, + }); + for (var i = 0; i < logins.length; i++) { + if ( + logins[i].username == this.props.username && + logins[i].password == this.props.password && + logins[i].usernameField == this.props.usernameField && + logins[i].passwordField == this.props.passwordField + ) { + logins[i].QueryInterface(Ci.nsILoginMetaInfo); + return logins[i].guid; + } + } + return -1; + }, + + /** + * Update + * + * Updates an existing password entry in the login manager with + * new properties. Throws on error. The 'old' properties are this + * object's properties, the 'new' properties are the properties in + * this object's 'updateProps' object. + * + * @return nothing + */ + Update() { + let oldlogin = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + let newlogin = new nsLoginInfo( + this.updateProps.hostname, + this.updateProps.submitURL, + this.updateProps.realm, + this.updateProps.username, + this.updateProps.password, + this.updateProps.usernameField, + this.updateProps.passwordField + ); + Services.logins.modifyLogin(oldlogin, newlogin); + }, + + /** + * Remove + * + * Removes an entry from the login manager for a password which + * matches this object's properties. Throws on error. + * + * @return nothing + */ + Remove() { + let login = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + Services.logins.removeLogin(login); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs new file mode 100644 index 0000000000..c629ef4a73 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +const WEAVE_PREF_PREFIX = "services.sync.prefs.sync."; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +/** + * Preference class constructor + * + * Initializes instance properties. + */ +export function Preference(props) { + Logger.AssertTrue( + "name" in props && "value" in props, + "Preference must have both name and value" + ); + + this.name = props.name; + this.value = props.value; +} + +/** + * Preference instance methods + */ +Preference.prototype = { + /** + * Modify + * + * Sets the value of the preference this.name to this.value. + * Throws on error. + * + * @return nothing + */ + Modify() { + // Determine if this pref is actually something Weave even looks at. + let weavepref = WEAVE_PREF_PREFIX + this.name; + try { + let syncPref = Services.prefs.getBoolPref(weavepref); + if (!syncPref) { + Services.prefs.setBoolPref(weavepref, true); + } + } catch (e) { + Logger.AssertTrue(false, "Weave doesn't sync pref " + this.name); + } + + // Modify the pref; throw an exception if the pref type is different + // than the value type specified in the test. + let prefType = Services.prefs.getPrefType(this.name); + switch (prefType) { + case Ci.nsIPrefBranch.PREF_INT: + Logger.AssertEqual( + typeof this.value, + "number", + "Wrong type used for preference value" + ); + Services.prefs.setIntPref(this.name, this.value); + break; + case Ci.nsIPrefBranch.PREF_STRING: + Logger.AssertEqual( + typeof this.value, + "string", + "Wrong type used for preference value" + ); + Services.prefs.setStringPref(this.name, this.value); + break; + case Ci.nsIPrefBranch.PREF_BOOL: + Logger.AssertEqual( + typeof this.value, + "boolean", + "Wrong type used for preference value" + ); + Services.prefs.setBoolPref(this.name, this.value); + break; + } + }, + + /** + * Find + * + * Verifies that the preference this.name has the value + * this.value. Throws on error, or if the pref's type or value + * doesn't match. + * + * @return nothing + */ + Find() { + // Read the pref value. + let value; + try { + let prefType = Services.prefs.getPrefType(this.name); + switch (prefType) { + case Ci.nsIPrefBranch.PREF_INT: + value = Services.prefs.getIntPref(this.name); + break; + case Ci.nsIPrefBranch.PREF_STRING: + value = Services.prefs.getStringPref(this.name); + break; + case Ci.nsIPrefBranch.PREF_BOOL: + value = Services.prefs.getBoolPref(this.name); + break; + } + } catch (e) { + Logger.AssertTrue(false, "Error accessing pref " + this.name); + } + + // Throw an exception if the current and expected values aren't of + // the same type, or don't have the same values. + Logger.AssertEqual( + typeof value, + typeof this.value, + "Value types don't match" + ); + Logger.AssertEqual(value, this.value, "Preference values don't match"); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs new file mode 100644 index 0000000000..8ea8f3b780 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +import { Weave } from "resource://services-sync/main.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +// Unfortunately, due to where TPS is run, we can't directly reuse the logic from +// BrowserTestUtils.sys.mjs. Moreover, we can't resolve the URI it loads the content +// frame script from ("chrome://mochikit/content/tests/BrowserTestUtils/content-utils.js"), +// hence the hackiness here and in BrowserTabs.Add. +Services.mm.loadFrameScript( + "data:application/javascript;charset=utf-8," + + encodeURIComponent(` + addEventListener("load", function(event) { + let subframe = event.target != content.document; + sendAsyncMessage("tps:loadEvent", {subframe: subframe, url: event.target.documentURI}); + }, true)`), + true, + true +); + +export var BrowserTabs = { + /** + * Add + * + * Opens a new tab in the current browser window for the + * given uri. Rejects on error. + * + * @param uri The uri to load in the new tab + * @return Promise + */ + async Add(uri) { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let browser = mainWindow.gBrowser; + let newtab = browser.addTrustedTab(uri); + + // Wait for the tab to load. + await new Promise(resolve => { + let mm = browser.ownerGlobal.messageManager; + mm.addMessageListener("tps:loadEvent", function onLoad(msg) { + mm.removeMessageListener("tps:loadEvent", onLoad); + resolve(); + }); + }); + + browser.selectedTab = newtab; + }, + + /** + * Find + * + * Finds the specified uri and title in Weave's list of remote tabs + * for the specified profile. + * + * @param uri The uri of the tab to find + * @param title The page title of the tab to find + * @param profile The profile to search for tabs + * @return true if the specified tab could be found, otherwise false + */ + async Find(uri, title, profile) { + // Find the uri in Weave's list of tabs for the given profile. + let tabEngine = Weave.Service.engineManager.get("tabs"); + for (let client of Weave.Service.clientsEngine.remoteClients) { + let tabClients = await tabEngine.getAllClients(); + let tabClient = tabClients.find(x => x.id === client.id); + if (!tabClient || !tabClient.tabs) { + continue; + } + for (let key in tabClient.tabs) { + let tab = tabClient.tabs[key]; + let weaveTabUrl = tab.urlHistory[0]; + if (uri == weaveTabUrl && profile == client.name) { + if (title == undefined || title == tab.title) { + return true; + } + } + } + Logger.logInfo( + `Dumping tabs for ${client.name}...\n` + + JSON.stringify(tabClient.tabs, null, 2) + ); + } + return false; + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs new file mode 100644 index 0000000000..b0798b9031 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +export var BrowserWindows = { + /** + * Add + * + * Opens a new window. Throws on error. + * + * @param aPrivate The private option. + * @return nothing + */ + Add(aPrivate, fn) { + return new Promise(resolve => { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let win = mainWindow.OpenBrowserWindow({ private: aPrivate }); + win.addEventListener( + "load", + function () { + resolve(win); + }, + { once: true } + ); + }); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/quit.sys.mjs b/services/sync/tps/extensions/tps/resource/quit.sys.mjs new file mode 100644 index 0000000000..e2cb8d8c22 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/quit.sys.mjs @@ -0,0 +1,38 @@ +/* -*- indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + From mozilla/toolkit/content + These files did not have a license +*/ +function canQuitApplication() { + try { + var cancelQuit = Cc["@mozilla.org/supports-PRBool;1"].createInstance( + Ci.nsISupportsPRBool + ); + Services.obs.notifyObservers(cancelQuit, "quit-application-requested"); + + // Something aborted the quit process. + if (cancelQuit.data) { + return false; + } + } catch (ex) {} + + return true; +} + +export function goQuitApplication() { + if (!canQuitApplication()) { + return false; + } + + try { + Services.startup.quit(Ci.nsIAppStartup.eForceQuit); + } catch (ex) { + throw new Error(`goQuitApplication: ${ex.message}`); + } + + return true; +} diff --git a/services/sync/tps/extensions/tps/resource/tps.sys.mjs b/services/sync/tps/extensions/tps/resource/tps.sys.mjs new file mode 100644 index 0000000000..2c4a5994a6 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/tps.sys.mjs @@ -0,0 +1,1583 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Addon: "resource://tps/modules/addons.sys.mjs", + AddonValidator: "resource://services-sync/engines/addons.sys.mjs", + Address: "resource://tps/modules/formautofill.sys.mjs", + Async: "resource://services-common/async.sys.mjs", + Authentication: "resource://tps/auth/fxaccounts.sys.mjs", + Bookmark: "resource://tps/modules/bookmarks.sys.mjs", + BookmarkFolder: "resource://tps/modules/bookmarks.sys.mjs", + BookmarkValidator: "resource://tps/modules/bookmarkValidator.sys.mjs", + BrowserTabs: "resource://tps/modules/tabs.sys.mjs", + BrowserWindows: "resource://tps/modules/windows.sys.mjs", + CommonUtils: "resource://services-common/utils.sys.mjs", + CreditCard: "resource://tps/modules/formautofill.sys.mjs", + DumpAddresses: "resource://tps/modules/formautofill.sys.mjs", + DumpBookmarks: "resource://tps/modules/bookmarks.sys.mjs", + DumpCreditCards: "resource://tps/modules/formautofill.sys.mjs", + DumpHistory: "resource://tps/modules/history.sys.mjs", + DumpPasswords: "resource://tps/modules/passwords.sys.mjs", + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", + FormData: "resource://tps/modules/forms.sys.mjs", + FormValidator: "resource://services-sync/engines/forms.sys.mjs", + HistoryEntry: "resource://tps/modules/history.sys.mjs", + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + Livemark: "resource://tps/modules/bookmarks.sys.mjs", + Log: "resource://gre/modules/Log.sys.mjs", + Logger: "resource://tps/logger.sys.mjs", + NetUtil: "resource://gre/modules/NetUtil.sys.mjs", + Password: "resource://tps/modules/passwords.sys.mjs", + PasswordValidator: "resource://services-sync/engines/passwords.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Preference: "resource://tps/modules/prefs.sys.mjs", + STATUS_OK: "resource://services-sync/constants.sys.mjs", + Separator: "resource://tps/modules/bookmarks.sys.mjs", + SessionStore: "resource:///modules/sessionstore/SessionStore.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + SyncTelemetry: "resource://services-sync/telemetry.sys.mjs", + WEAVE_VERSION: "resource://services-sync/constants.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "fileProtocolHandler", () => { + let fileHandler = Services.io.getProtocolHandler("file"); + return fileHandler.QueryInterface(Ci.nsIFileProtocolHandler); +}); + +ChromeUtils.defineLazyGetter(lazy, "gTextDecoder", () => { + return new TextDecoder(); +}); + +// Options for wiping data during a sync +const SYNC_RESET_CLIENT = "resetClient"; +const SYNC_WIPE_CLIENT = "wipeClient"; +const SYNC_WIPE_REMOTE = "wipeRemote"; + +// Actions a test can perform +const ACTION_ADD = "add"; +const ACTION_DELETE = "delete"; +const ACTION_MODIFY = "modify"; +const ACTION_SET_ENABLED = "set-enabled"; +const ACTION_SYNC = "sync"; +const ACTION_SYNC_RESET_CLIENT = SYNC_RESET_CLIENT; +const ACTION_SYNC_WIPE_CLIENT = SYNC_WIPE_CLIENT; +const ACTION_SYNC_WIPE_REMOTE = SYNC_WIPE_REMOTE; +const ACTION_VERIFY = "verify"; +const ACTION_VERIFY_NOT = "verify-not"; + +const OBSERVER_TOPICS = [ + "fxaccounts:onlogin", + "fxaccounts:onlogout", + "profile-before-change", + "weave:service:tracking-started", + "weave:service:tracking-stopped", + "weave:service:login:error", + "weave:service:setup-complete", + "weave:service:sync:finish", + "weave:service:sync:delayed", + "weave:service:sync:error", + "weave:service:sync:start", + "weave:service:resyncs-finished", + "places-browser-init-complete", +]; + +export var TPS = { + _currentAction: -1, + _currentPhase: -1, + _enabledEngines: null, + _errors: 0, + _isTracking: false, + _phaseFinished: false, + _phaselist: {}, + _setupComplete: false, + _syncActive: false, + _syncCount: 0, + _syncsReportedViaTelemetry: 0, + _syncErrors: 0, + _syncWipeAction: null, + _tabsAdded: 0, + _tabsFinished: 0, + _test: null, + _triggeredSync: false, + _msSinceEpoch: 0, + _requestedQuit: false, + shouldValidateAddons: false, + shouldValidateBookmarks: false, + shouldValidatePasswords: false, + shouldValidateForms: false, + _placesInitDeferred: Promise.withResolvers(), + ACTIONS: [ + ACTION_ADD, + ACTION_DELETE, + ACTION_MODIFY, + ACTION_SET_ENABLED, + ACTION_SYNC, + ACTION_SYNC_RESET_CLIENT, + ACTION_SYNC_WIPE_CLIENT, + ACTION_SYNC_WIPE_REMOTE, + ACTION_VERIFY, + ACTION_VERIFY_NOT, + ], + + _init: function TPS__init() { + this.delayAutoSync(); + + OBSERVER_TOPICS.forEach(function (aTopic) { + Services.obs.addObserver(this, aTopic, true); + }, this); + + // Some engines bump their score during their sync, which then causes + // another sync immediately (notably, prefs and addons). We don't want + // this to happen, and there's no obvious preference to kill it - so + // we do this nasty hack to ensure the global score is always zero. + Services.prefs.addObserver("services.sync.globalScore", () => { + if (lazy.Weave.Service.scheduler.globalScore != 0) { + lazy.Weave.Service.scheduler.globalScore = 0; + } + }); + }, + + DumpError(msg, exc = null) { + this._errors++; + let errInfo; + if (exc) { + errInfo = lazy.Log.exceptionStr(exc); // includes details and stack-trace. + } else { + // always write a stack even if no error passed. + errInfo = lazy.Log.stackTrace(new Error()); + } + lazy.Logger.logError(`[phase ${this._currentPhase}] ${msg} - ${errInfo}`); + this.quit(); + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + observe: function TPS__observe(subject, topic, data) { + try { + lazy.Logger.logInfo("----------event observed: " + topic); + + switch (topic) { + case "profile-before-change": + OBSERVER_TOPICS.forEach(function (topic) { + Services.obs.removeObserver(this, topic); + }, this); + + lazy.Logger.close(); + + break; + + case "places-browser-init-complete": + this._placesInitDeferred.resolve(); + break; + + case "weave:service:setup-complete": + this._setupComplete = true; + + if (this._syncWipeAction) { + lazy.Weave.Svc.PrefBranch.setStringPref( + "firstSync", + this._syncWipeAction + ); + this._syncWipeAction = null; + } + + break; + + case "weave:service:sync:error": + this._syncActive = false; + + this.delayAutoSync(); + + // If this is the first sync error, retry... + if (this._syncErrors === 0) { + lazy.Logger.logInfo("Sync error; retrying..."); + this._syncErrors++; + lazy.CommonUtils.nextTick(() => { + this.RunNextTestAction().catch(err => { + this.DumpError("RunNextTestActionFailed", err); + }); + }); + } else { + this._triggeredSync = false; + this.DumpError("Sync error; aborting test"); + } + + break; + + case "weave:service:resyncs-finished": + this._syncActive = false; + this._syncErrors = 0; + this._triggeredSync = false; + + this.delayAutoSync(); + break; + + case "weave:service:sync:start": + // Ensure that the sync operation has been started by TPS + if (!this._triggeredSync) { + this.DumpError( + "Automatic sync got triggered, which is not allowed." + ); + } + + this._syncActive = true; + break; + + case "weave:service:tracking-started": + this._isTracking = true; + break; + + case "weave:service:tracking-stopped": + this._isTracking = false; + break; + + case "fxaccounts:onlogin": + // A user signed in - for TPS that always means sync - so configure + // that. + lazy.Weave.Service.configure().catch(e => { + this.DumpError("Configuring sync failed.", e); + }); + break; + + default: + lazy.Logger.logInfo(`unhandled event: ${topic}`); + } + } catch (e) { + this.DumpError("Observer failed", e); + } + }, + + /** + * Given that we cannot completely disable the automatic sync operations, we + * massively delay the next sync. Sync operations have to only happen when + * directly called via TPS.Sync()! + */ + delayAutoSync: function TPS_delayAutoSync() { + lazy.Weave.Svc.PrefBranch.setIntPref("scheduler.immediateInterval", 7200); + lazy.Weave.Svc.PrefBranch.setIntPref("scheduler.idleInterval", 7200); + lazy.Weave.Svc.PrefBranch.setIntPref("scheduler.activeInterval", 7200); + lazy.Weave.Svc.PrefBranch.setIntPref("syncThreshold", 10000000); + }, + + quit: function TPS__quit() { + lazy.Logger.logInfo("quitting"); + this._requestedQuit = true; + this.goQuitApplication(); + }, + + async HandleWindows(aWindow, action) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on window " + + JSON.stringify(aWindow) + ); + switch (action) { + case ACTION_ADD: + await lazy.BrowserWindows.Add(aWindow.private); + break; + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on windows" + ); + }, + + async HandleTabs(tabs, action) { + for (let tab of tabs) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on tab " + + JSON.stringify(tab) + ); + switch (action) { + case ACTION_ADD: + await lazy.BrowserTabs.Add(tab.uri); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + typeof tab.profile != "undefined", + "profile must be defined when verifying tabs" + ); + lazy.Logger.AssertTrue( + await lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile), + "error locating tab" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + typeof tab.profile != "undefined", + "profile must be defined when verifying tabs" + ); + lazy.Logger.AssertTrue( + await !lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile), + "tab found which was expected to be absent" + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on tabs" + ); + }, + + async HandlePrefs(prefs, action) { + for (let pref of prefs) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on pref " + + JSON.stringify(pref) + ); + let preference = new lazy.Preference(pref); + switch (action) { + case ACTION_MODIFY: + preference.Modify(); + break; + case ACTION_VERIFY: + preference.Find(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on pref" + ); + }, + + async HandleForms(data, action) { + this.shouldValidateForms = true; + for (let datum of data) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on form entry " + + JSON.stringify(datum) + ); + let formdata = new lazy.FormData(datum, this._msSinceEpoch); + switch (action) { + case ACTION_ADD: + await formdata.Create(); + break; + case ACTION_DELETE: + await formdata.Remove(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue(await formdata.Find(), "form data not found"); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await formdata.Find()), + "form data found, but it shouldn't be present" + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on formdata" + ); + }, + + async HandleHistory(entries, action) { + try { + for (let entry of entries) { + const entryString = JSON.stringify(entry); + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on history entry " + + entryString + ); + switch (action) { + case ACTION_ADD: + await lazy.HistoryEntry.Add(entry, this._msSinceEpoch); + break; + case ACTION_DELETE: + await lazy.HistoryEntry.Delete(entry, this._msSinceEpoch); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await lazy.HistoryEntry.Find(entry, this._msSinceEpoch), + "Uri visits not found in history database: " + entryString + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await lazy.HistoryEntry.Find(entry, this._msSinceEpoch)), + "Uri visits found in history database, but they shouldn't be: " + + entryString + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on history" + ); + } catch (e) { + await lazy.DumpHistory(); + throw e; + } + }, + + async HandlePasswords(passwords, action) { + this.shouldValidatePasswords = true; + try { + for (let password of passwords) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on password " + + JSON.stringify(password) + ); + let passwordOb = new lazy.Password(password); + switch (action) { + case ACTION_ADD: + lazy.Logger.AssertTrue( + (await passwordOb.Create()) > -1, + "error adding password" + ); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + (await passwordOb.Find()) != -1, + "password not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + (await passwordOb.Find()) == -1, + "password found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue( + (await passwordOb.Find()) != -1, + "password not found" + ); + passwordOb.Remove(); + break; + case ACTION_MODIFY: + if (passwordOb.updateProps != null) { + lazy.Logger.AssertTrue( + (await passwordOb.Find()) != -1, + "password not found" + ); + passwordOb.Update(); + } + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on passwords" + ); + } catch (e) { + await lazy.DumpPasswords(); + throw e; + } + }, + + async HandleAddons(addons, action, state) { + this.shouldValidateAddons = true; + for (let entry of addons) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on addon " + + JSON.stringify(entry) + ); + let addon = new lazy.Addon(this, entry); + switch (action) { + case ACTION_ADD: + await addon.install(); + break; + case ACTION_DELETE: + await addon.uninstall(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await addon.find(state), + "addon " + addon.id + " not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertFalse( + await addon.find(state), + "addon " + addon.id + " is present, but it shouldn't be" + ); + break; + case ACTION_SET_ENABLED: + lazy.Logger.AssertTrue( + await addon.setEnabled(state), + "addon " + addon.id + " not found" + ); + break; + default: + throw new Error("Unknown action for add-on: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on addons" + ); + }, + + async HandleBookmarks(bookmarks, action) { + // wait for default bookmarks to be created. + await this._placesInitDeferred.promise; + this.shouldValidateBookmarks = true; + try { + let items = []; + for (let folder in bookmarks) { + let last_item_pos = -1; + for (let bookmark of bookmarks[folder]) { + lazy.Logger.clearPotentialError(); + let placesItem; + bookmark.location = folder; + + if (last_item_pos != -1) { + bookmark.last_item_pos = last_item_pos; + } + let itemGuid = null; + + if (action != ACTION_MODIFY && action != ACTION_DELETE) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on bookmark " + + JSON.stringify(bookmark) + ); + } + + if ("uri" in bookmark) { + placesItem = new lazy.Bookmark(bookmark); + } else if ("folder" in bookmark) { + placesItem = new lazy.BookmarkFolder(bookmark); + } else if ("livemark" in bookmark) { + placesItem = new lazy.Livemark(bookmark); + } else if ("separator" in bookmark) { + placesItem = new lazy.Separator(bookmark); + } + + if (action == ACTION_ADD) { + itemGuid = await placesItem.Create(); + } else { + itemGuid = await placesItem.Find(); + if (action == ACTION_VERIFY_NOT) { + lazy.Logger.AssertTrue( + itemGuid == null, + "places item exists but it shouldn't: " + + JSON.stringify(bookmark) + ); + } else { + lazy.Logger.AssertTrue(itemGuid, "places item not found", true); + } + } + + last_item_pos = await placesItem.GetItemIndex(); + items.push(placesItem); + } + } + + if (action == ACTION_DELETE || action == ACTION_MODIFY) { + for (let item of items) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on bookmark " + + JSON.stringify(item) + ); + switch (action) { + case ACTION_DELETE: + await item.Remove(); + break; + case ACTION_MODIFY: + if (item.updateProps != null) { + await item.Update(); + } + break; + } + } + } + + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on bookmarks" + ); + } catch (e) { + await lazy.DumpBookmarks(); + throw e; + } + }, + + async HandleAddresses(addresses, action) { + try { + for (let address of addresses) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on address " + + JSON.stringify(address) + ); + let addressOb = new lazy.Address(address); + switch (action) { + case ACTION_ADD: + await addressOb.Create(); + break; + case ACTION_MODIFY: + await addressOb.Update(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue(await addressOb.Find(), "address not found"); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await addressOb.Find()), + "address found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue(await addressOb.Find(), "address not found"); + await addressOb.Remove(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on addresses" + ); + } catch (e) { + await lazy.DumpAddresses(); + throw e; + } + }, + + async HandleCreditCards(creditCards, action) { + try { + for (let creditCard of creditCards) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on creditCard " + + JSON.stringify(creditCard) + ); + let creditCardOb = new lazy.CreditCard(creditCard); + switch (action) { + case ACTION_ADD: + await creditCardOb.Create(); + break; + case ACTION_MODIFY: + await creditCardOb.Update(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await creditCardOb.Find(), + "creditCard not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await creditCardOb.Find()), + "creditCard found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue( + await creditCardOb.Find(), + "creditCard not found" + ); + await creditCardOb.Remove(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on creditCards" + ); + } catch (e) { + await lazy.DumpCreditCards(); + throw e; + } + }, + + async Cleanup() { + try { + await this.WipeServer(); + } catch (ex) { + lazy.Logger.logError( + "Failed to wipe server: " + lazy.Log.exceptionStr(ex) + ); + } + try { + if (await lazy.Authentication.isLoggedIn()) { + // signout and wait for Sync to completely reset itself. + lazy.Logger.logInfo("signing out"); + let waiter = this.promiseObserver("weave:service:start-over:finish"); + await lazy.Authentication.signOut(); + await waiter; + lazy.Logger.logInfo("signout complete"); + } + await lazy.Authentication.deleteEmail(this.config.fx_account.username); + } catch (e) { + lazy.Logger.logError("Failed to sign out: " + lazy.Log.exceptionStr(e)); + } + }, + + /** + * Use Sync's bookmark validation code to see if we've corrupted the tree. + */ + async ValidateBookmarks() { + let getServerBookmarkState = async () => { + let bookmarkEngine = lazy.Weave.Service.engineManager.get("bookmarks"); + let collection = bookmarkEngine.itemSource(); + let collectionKey = + bookmarkEngine.service.collectionKeys.keyForCollection( + bookmarkEngine.name + ); + collection.full = true; + let items = []; + let resp = await collection.get(); + for (let json of resp.obj) { + let record = new collection._recordObj(); + record.deserialize(json); + await record.decrypt(collectionKey); + items.push(record.cleartext); + } + return items; + }; + let serverRecordDumpStr; + try { + lazy.Logger.logInfo("About to perform bookmark validation"); + let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", { + includeItemIds: true, + }); + let serverRecords = await getServerBookmarkState(); + // We can't wait until catch to stringify this, since at that point it will have cycles. + serverRecordDumpStr = JSON.stringify(serverRecords); + + let validator = new lazy.BookmarkValidator(); + let { problemData } = await validator.compareServerWithClient( + serverRecords, + clientTree + ); + + for (let { name, count } of problemData.getSummary()) { + // Exclude mobile showing up on the server hackily so that we don't + // report it every time, see bug 1273234 and 1274394 for more information. + if ( + name === "serverUnexpected" && + problemData.serverUnexpected.includes("mobile") + ) { + --count; + } + if (count) { + // Log this out before we assert. This is useful in the context of TPS logs, since we + // can see the IDs in the test files. + lazy.Logger.logInfo( + `Validation problem: "${name}": ${JSON.stringify( + problemData[name] + )}` + ); + } + lazy.Logger.AssertEqual( + count, + 0, + `Bookmark validation error of type ${name}` + ); + } + } catch (e) { + // Dump the client records (should always be doable) + lazy.DumpBookmarks(); + // Dump the server records if gotten them already. + if (serverRecordDumpStr) { + lazy.Logger.logInfo( + "Server bookmark records:\n" + serverRecordDumpStr + "\n" + ); + } + this.DumpError("Bookmark validation failed", e); + } + lazy.Logger.logInfo("Bookmark validation finished"); + }, + + async ValidateCollection(engineName, ValidatorType) { + let serverRecordDumpStr; + let clientRecordDumpStr; + try { + lazy.Logger.logInfo(`About to perform validation for "${engineName}"`); + let engine = lazy.Weave.Service.engineManager.get(engineName); + let validator = new ValidatorType(engine); + let serverRecords = await validator.getServerItems(engine); + let clientRecords = await validator.getClientItems(); + try { + // This substantially improves the logs for addons while not making a + // substantial difference for the other two + clientRecordDumpStr = JSON.stringify( + clientRecords.map(r => { + let res = validator.normalizeClientItem(r); + delete res.original; // Try and prevent cyclic references + return res; + }) + ); + } catch (e) { + // ignore the error, the dump string is just here to make debugging easier. + clientRecordDumpStr = "<Cyclic value>"; + } + try { + serverRecordDumpStr = JSON.stringify(serverRecords); + } catch (e) { + // as above + serverRecordDumpStr = "<Cyclic value>"; + } + let { problemData } = await validator.compareClientWithServer( + clientRecords, + serverRecords + ); + for (let { name, count } of problemData.getSummary()) { + if (count) { + lazy.Logger.logInfo( + `Validation problem: "${name}": ${JSON.stringify( + problemData[name] + )}` + ); + } + lazy.Logger.AssertEqual( + count, + 0, + `Validation error for "${engineName}" of type "${name}"` + ); + } + } catch (e) { + // Dump the client records if possible + if (clientRecordDumpStr) { + lazy.Logger.logInfo( + `Client state for ${engineName}:\n${clientRecordDumpStr}\n` + ); + } + // Dump the server records if gotten them already. + if (serverRecordDumpStr) { + lazy.Logger.logInfo( + `Server state for ${engineName}:\n${serverRecordDumpStr}\n` + ); + } + this.DumpError(`Validation failed for ${engineName}`, e); + } + lazy.Logger.logInfo(`Validation finished for ${engineName}`); + }, + + ValidatePasswords() { + return this.ValidateCollection("passwords", lazy.PasswordValidator); + }, + + ValidateForms() { + return this.ValidateCollection("forms", lazy.FormValidator); + }, + + ValidateAddons() { + return this.ValidateCollection("addons", lazy.AddonValidator); + }, + + async RunNextTestAction() { + lazy.Logger.logInfo("Running next test action"); + try { + if (this._currentAction >= this._phaselist[this._currentPhase].length) { + // Run necessary validations and then finish up + lazy.Logger.logInfo("No more actions - running validations..."); + if (this.shouldValidateBookmarks) { + await this.ValidateBookmarks(); + } + if (this.shouldValidatePasswords) { + await this.ValidatePasswords(); + } + if (this.shouldValidateForms) { + await this.ValidateForms(); + } + if (this.shouldValidateAddons) { + await this.ValidateAddons(); + } + // Force this early so that we run the validation and detect missing pings + // *before* we start shutting down, since if we do it after, the python + // code won't notice the failure. + lazy.SyncTelemetry.shutdown(); + // we're all done + lazy.Logger.logInfo( + "test phase " + + this._currentPhase + + ": " + + (this._errors ? "FAIL" : "PASS") + ); + this._phaseFinished = true; + this.quit(); + return; + } + this.seconds_since_epoch = Services.prefs.getIntPref( + "tps.seconds_since_epoch" + ); + if (this.seconds_since_epoch) { + // Places dislikes it if we add visits in the future. We pretend the + // real time is 1 minute ago to avoid issues caused by places using a + // different clock than the one that set the seconds_since_epoch pref. + this._msSinceEpoch = (this.seconds_since_epoch - 60) * 1000; + } else { + this.DumpError("seconds-since-epoch not set"); + return; + } + + let phase = this._phaselist[this._currentPhase]; + let action = phase[this._currentAction]; + lazy.Logger.logInfo("starting action: " + action[0].name); + await action[0].apply(this, action.slice(1)); + + this._currentAction++; + } catch (e) { + if (lazy.Async.isShutdownException(e)) { + if (this._requestedQuit) { + lazy.Logger.logInfo("Sync aborted due to requested shutdown"); + } else { + this.DumpError( + "Sync aborted due to shutdown, but we didn't request it" + ); + } + } else { + this.DumpError("RunNextTestAction failed", e); + } + return; + } + await this.RunNextTestAction(); + }, + + _getFileRelativeToSourceRoot(testFileURL, relativePath) { + let file = lazy.fileProtocolHandler.getFileFromURLSpec(testFileURL); + let root = file.parent.parent.parent.parent.parent; // <root>/services/sync/tests/tps/test_foo.js // <root>/services/sync/tests/tps // <root>/services/sync/tests // <root>/services/sync // <root>/services // <root> + root.appendRelativePath(relativePath); + root.normalize(); + return root; + }, + + _pingValidator: null, + + // Default ping validator that always says the ping passes. This should be + // overridden unless the `testing.tps.skipPingValidation` pref is true. + get pingValidator() { + return this._pingValidator + ? this._pingValidator + : { + validate() { + lazy.Logger.logInfo( + "Not validating ping -- disabled by pref or failure to load schema" + ); + return { valid: true, errors: [] }; + }, + }; + }, + + // Attempt to load the sync_ping_schema.json and initialize `this.pingValidator` + // based on the source of the tps file. Assumes that it's at "../unit/sync_ping_schema.json" + // relative to the directory the tps test file (testFile) is contained in. + _tryLoadPingSchema(testFile) { + if (Services.prefs.getBoolPref("testing.tps.skipPingValidation", false)) { + return; + } + try { + let schemaFile = this._getFileRelativeToSourceRoot( + testFile, + "services/sync/tests/unit/sync_ping_schema.json" + ); + + let stream = Cc[ + "@mozilla.org/network/file-input-stream;1" + ].createInstance(Ci.nsIFileInputStream); + + stream.init( + schemaFile, + lazy.FileUtils.MODE_RDONLY, + lazy.FileUtils.PERMS_FILE, + 0 + ); + + let bytes = lazy.NetUtil.readInputStream(stream, stream.available()); + let schema = JSON.parse(lazy.gTextDecoder.decode(bytes)); + lazy.Logger.logInfo("Successfully loaded schema"); + + this._pingValidator = new lazy.JsonSchema.Validator(schema); + } catch (e) { + this.DumpError( + `Failed to load ping schema relative to "${testFile}".`, + e + ); + } + }, + + /** + * Runs a single test phase. + * + * This is the main entry point for each phase of a test. The TPS command + * line driver loads this module and calls into the function with the + * arguments from the command line. + * + * When a phase is executed, the file is loaded as JavaScript into the + * current object. + * + * The following keys in the options argument have meaning: + * + * - ignoreUnusedEngines If true, unused engines will be unloaded from + * Sync. This makes output easier to parse and is + * useful for debugging test failures. + * + * @param file + * String URI of the file to open. + * @param phase + * String name of the phase to run. + * @param logpath + * String path of the log file to write to. + * @param options + * Object defining addition run-time options. + */ + async RunTestPhase(file, phase, logpath, options) { + try { + let settings = options || {}; + + lazy.Logger.init(logpath); + lazy.Logger.logInfo("Sync version: " + lazy.WEAVE_VERSION); + lazy.Logger.logInfo("Firefox buildid: " + Services.appinfo.appBuildID); + lazy.Logger.logInfo("Firefox version: " + Services.appinfo.version); + lazy.Logger.logInfo( + "Firefox source revision: " + + (AppConstants.SOURCE_REVISION_URL || "unknown") + ); + lazy.Logger.logInfo("Firefox platform: " + AppConstants.platform); + + // do some sync housekeeping + if (lazy.Weave.Service.isLoggedIn) { + this.DumpError("Sync logged in on startup...profile may be dirty"); + return; + } + + // Wait for Sync service to become ready. + if (!lazy.Weave.Status.ready) { + this.waitForEvent("weave:service:ready"); + } + + await lazy.Weave.Service.promiseInitialized; + + // We only want to do this if we modified the bookmarks this phase. + this.shouldValidateBookmarks = false; + + // Always give Sync an extra tick to initialize. If we waited for the + // service:ready event, this is required to ensure all handlers have + // executed. + await lazy.Async.promiseYield(); + await this._executeTestPhase(file, phase, settings); + } catch (e) { + this.DumpError("RunTestPhase failed", e); + } + }, + + /** + * Executes a single test phase. + * + * This is called by RunTestPhase() after the environment is validated. + */ + async _executeTestPhase(file, phase, settings) { + try { + this.config = JSON.parse(Services.prefs.getStringPref("tps.config")); + // parse the test file + Services.scriptloader.loadSubScript(file, this); + this._currentPhase = phase; + // cleanup phases are in the format `cleanup-${profileName}`. + if (this._currentPhase.startsWith("cleanup-")) { + let profileToClean = this._currentPhase.slice("cleanup-".length); + this.phases[this._currentPhase] = profileToClean; + this.Phase(this._currentPhase, [[this.Cleanup]]); + } else { + // Don't bother doing this for cleanup phases. + this._tryLoadPingSchema(file); + } + let this_phase = this._phaselist[this._currentPhase]; + + if (this_phase == undefined) { + this.DumpError("invalid phase " + this._currentPhase); + return; + } + + if (this.phases[this._currentPhase] == undefined) { + this.DumpError("no profile defined for phase " + this._currentPhase); + return; + } + + // If we have restricted the active engines, unregister engines we don't + // care about. + if (settings.ignoreUnusedEngines && Array.isArray(this._enabledEngines)) { + let names = {}; + for (let name of this._enabledEngines) { + names[name] = true; + } + for (let engine of lazy.Weave.Service.engineManager.getEnabled()) { + if (!(engine.name in names)) { + lazy.Logger.logInfo("Unregistering unused engine: " + engine.name); + await lazy.Weave.Service.engineManager.unregister(engine); + } + } + } + lazy.Logger.logInfo("Starting phase " + this._currentPhase); + + lazy.Logger.logInfo( + "setting client.name to " + this.phases[this._currentPhase] + ); + lazy.Weave.Svc.PrefBranch.setStringPref( + "client.name", + this.phases[this._currentPhase] + ); + + this._interceptSyncTelemetry(); + + // start processing the test actions + this._currentAction = 0; + await lazy.SessionStore.promiseAllWindowsRestored; + await this.RunNextTestAction(); + } catch (e) { + this.DumpError("_executeTestPhase failed", e); + } + }, + + /** + * Override sync telemetry functions so that we can detect errors generating + * the sync ping, and count how many pings we report. + */ + _interceptSyncTelemetry() { + let originalObserve = lazy.SyncTelemetry.observe; + let self = this; + lazy.SyncTelemetry.observe = function () { + try { + originalObserve.apply(this, arguments); + } catch (e) { + self.DumpError("Error when generating sync telemetry", e); + } + }; + lazy.SyncTelemetry.submit = record => { + lazy.Logger.logInfo( + "Intercepted sync telemetry submission: " + JSON.stringify(record) + ); + this._syncsReportedViaTelemetry += + record.syncs.length + (record.discarded || 0); + if (record.discarded) { + if (record.syncs.length != lazy.SyncTelemetry.maxPayloadCount) { + this.DumpError( + "Syncs discarded from ping before maximum payload count reached" + ); + } + } + // If this is the shutdown ping, check and see that the telemetry saw all the syncs. + if (record.why === "shutdown") { + // If we happen to sync outside of tps manually causing it, its not an + // error in the telemetry, so we only complain if we didn't see all of them. + if (this._syncsReportedViaTelemetry < this._syncCount) { + this.DumpError( + `Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.` + ); + } + } + if (!record.syncs.length) { + // Note: we're overwriting submit, so this is called even for pings that + // may have no data (which wouldn't be submitted to telemetry and would + // fail validation). + return; + } + // Our ping may have some undefined values, which we rely on JSON stripping + // out as part of the ping submission - but our validator fails with them, + // so round-trip via JSON here to avoid that. + record = JSON.parse(JSON.stringify(record)); + const result = this.pingValidator.validate(record); + if (!result.valid) { + // Note that we already logged the record. + this.DumpError( + "Sync ping validation failed with errors: " + + JSON.stringify(result.errors) + ); + } + }; + }, + + /** + * Register a single phase with the test harness. + * + * This is called when loading individual test files. + * + * @param phasename + * String name of the phase being loaded. + * @param fnlist + * Array of functions/actions to perform. + */ + Phase: function Test__Phase(phasename, fnlist) { + if (Object.keys(this._phaselist).length === 0) { + // This is the first phase we should force a log in + fnlist.unshift([this.Login]); + } + this._phaselist[phasename] = fnlist; + }, + + /** + * Restrict enabled Sync engines to a specified set. + * + * This can be called by a test to limit what engines are enabled. It is + * recommended to call it to reduce the overhead and log clutter for the + * test. + * + * The "clients" engine is special and is always enabled, so there is no + * need to specify it. + * + * @param names + * Array of Strings for engines to make active during the test. + */ + EnableEngines: function EnableEngines(names) { + if (!Array.isArray(names)) { + throw new Error( + "Argument to RestrictEngines() is not an array: " + typeof names + ); + } + + this._enabledEngines = names; + }, + + /** + * Returns a promise that resolves when a specific observer notification is + * resolved. This is similar to the various waitFor* functions, although is + * typically safer if you need to do some other work that may make the event + * fire. + * + * eg: + * doSomething(); // causes the event to be fired. + * await promiseObserver("something"); + * is risky as the call to doSomething may trigger the event before the + * promiseObserver call is made. Contrast with: + * + * let waiter = promiseObserver("something"); + * doSomething(); // causes the event to be fired. + * await waiter; // will return as soon as the event fires, even if it fires + * // before this function is called. + * + * @param aEventName + * String event to wait for. + */ + promiseObserver(aEventName) { + return new Promise(resolve => { + lazy.Logger.logInfo("Setting up wait for " + aEventName + "..."); + let handler = () => { + lazy.Logger.logInfo("Observed " + aEventName); + lazy.Svc.Obs.remove(aEventName, handler); + resolve(); + }; + lazy.Svc.Obs.add(aEventName, handler); + }); + }, + + /** + * Wait for the named event to be observed. + * + * Note that in general, you should probably use promiseObserver unless you + * are 100% sure that the event being waited on can only be sent after this + * call adds the listener. + * + * @param aEventName + * String event to wait for. + */ + async waitForEvent(aEventName) { + await this.promiseObserver(aEventName); + }, + + /** + * Waits for Sync to logged in before returning + */ + async waitForSetupComplete() { + if (!this._setupComplete) { + await this.waitForEvent("weave:service:setup-complete"); + } + }, + + /** + * Waits for Sync to be finished before returning + */ + async waitForSyncFinished() { + if (lazy.Weave.Service.locked) { + await this.waitForEvent("weave:service:resyncs-finished"); + } + }, + + /** + * Waits for Sync to start tracking before returning. + */ + async waitForTracking() { + if (!this._isTracking) { + await this.waitForEvent("weave:service:tracking-started"); + } + }, + + /** + * Login on the server + */ + async Login() { + if (await lazy.Authentication.isReady()) { + return; + } + + lazy.Logger.logInfo("Setting client credentials and login."); + await lazy.Authentication.signIn(this.config.fx_account); + await this.waitForSetupComplete(); + lazy.Logger.AssertEqual( + lazy.Weave.Status.service, + lazy.STATUS_OK, + "Weave status OK" + ); + await this.waitForTracking(); + }, + + /** + * Triggers a sync operation + * + * @param {String} [wipeAction] + * Type of wipe to perform (resetClient, wipeClient, wipeRemote) + * + */ + async Sync(wipeAction) { + if (this._syncActive) { + this.DumpError("Sync currently active which should be impossible"); + return; + } + lazy.Logger.logInfo( + "Executing Sync" + (wipeAction ? ": " + wipeAction : "") + ); + + // Force a wipe action if requested. In case of an initial sync the pref + // will be overwritten by Sync itself (see bug 992198), so ensure that we + // also handle it via the "weave:service:setup-complete" notification. + if (wipeAction) { + this._syncWipeAction = wipeAction; + lazy.Weave.Svc.PrefBranch.setStringPref("firstSync", wipeAction); + } else { + lazy.Weave.Svc.PrefBranch.clearUserPref("firstSync"); + } + if (!(await lazy.Weave.Service.login())) { + // We need to complete verification. + lazy.Logger.logInfo("Logging in before performing sync"); + await this.Login(); + } + ++this._syncCount; + + lazy.Logger.logInfo( + "Executing Sync" + (wipeAction ? ": " + wipeAction : "") + ); + this._triggeredSync = true; + await lazy.Weave.Service.sync(); + lazy.Logger.logInfo("Sync is complete"); + // wait a second for things to settle... + await new Promise(resolve => { + lazy.CommonUtils.namedTimer(resolve, 1000, this, "postsync"); + }); + }, + + async WipeServer() { + lazy.Logger.logInfo("Wiping data from server."); + + await this.Login(); + await lazy.Weave.Service.login(); + await lazy.Weave.Service.wipeServer(); + }, + + /** + * Action which ensures changes are being tracked before returning. + */ + async EnsureTracking() { + await this.Login(); + await this.waitForTracking(); + }, + + Addons: { + async install(addons) { + await TPS.HandleAddons(addons, ACTION_ADD); + }, + async setEnabled(addons, state) { + await TPS.HandleAddons(addons, ACTION_SET_ENABLED, state); + }, + async uninstall(addons) { + await TPS.HandleAddons(addons, ACTION_DELETE); + }, + async verify(addons, state) { + await TPS.HandleAddons(addons, ACTION_VERIFY, state); + }, + async verifyNot(addons) { + await TPS.HandleAddons(addons, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidateAddons = false; + }, + }, + + Addresses: { + async add(addresses) { + await this.HandleAddresses(addresses, ACTION_ADD); + }, + async modify(addresses) { + await this.HandleAddresses(addresses, ACTION_MODIFY); + }, + async delete(addresses) { + await this.HandleAddresses(addresses, ACTION_DELETE); + }, + async verify(addresses) { + await this.HandleAddresses(addresses, ACTION_VERIFY); + }, + async verifyNot(addresses) { + await this.HandleAddresses(addresses, ACTION_VERIFY_NOT); + }, + }, + + Bookmarks: { + async add(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_ADD); + }, + async modify(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_MODIFY); + }, + async delete(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_DELETE); + }, + async verify(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY); + }, + async verifyNot(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidateBookmarks = false; + }, + }, + CreditCards: { + async add(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_ADD); + }, + async modify(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_MODIFY); + }, + async delete(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_DELETE); + }, + async verify(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_VERIFY); + }, + async verifyNot(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_VERIFY_NOT); + }, + }, + + Formdata: { + async add(formdata) { + await this.HandleForms(formdata, ACTION_ADD); + }, + async delete(formdata) { + await this.HandleForms(formdata, ACTION_DELETE); + }, + async verify(formdata) { + await this.HandleForms(formdata, ACTION_VERIFY); + }, + async verifyNot(formdata) { + await this.HandleForms(formdata, ACTION_VERIFY_NOT); + }, + }, + History: { + async add(history) { + await this.HandleHistory(history, ACTION_ADD); + }, + async delete(history) { + await this.HandleHistory(history, ACTION_DELETE); + }, + async verify(history) { + await this.HandleHistory(history, ACTION_VERIFY); + }, + async verifyNot(history) { + await this.HandleHistory(history, ACTION_VERIFY_NOT); + }, + }, + Passwords: { + async add(passwords) { + await this.HandlePasswords(passwords, ACTION_ADD); + }, + async modify(passwords) { + await this.HandlePasswords(passwords, ACTION_MODIFY); + }, + async delete(passwords) { + await this.HandlePasswords(passwords, ACTION_DELETE); + }, + async verify(passwords) { + await this.HandlePasswords(passwords, ACTION_VERIFY); + }, + async verifyNot(passwords) { + await this.HandlePasswords(passwords, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidatePasswords = false; + }, + }, + Prefs: { + async modify(prefs) { + await TPS.HandlePrefs(prefs, ACTION_MODIFY); + }, + async verify(prefs) { + await TPS.HandlePrefs(prefs, ACTION_VERIFY); + }, + }, + Tabs: { + async add(tabs) { + await TPS.HandleTabs(tabs, ACTION_ADD); + }, + async verify(tabs) { + await TPS.HandleTabs(tabs, ACTION_VERIFY); + }, + async verifyNot(tabs) { + await TPS.HandleTabs(tabs, ACTION_VERIFY_NOT); + }, + }, + Windows: { + async add(aWindow) { + await TPS.HandleWindows(aWindow, ACTION_ADD); + }, + }, + + // Jumping through loads of hoops via calling back into a "HandleXXX" method + // and adding an ACTION_XXX indirection adds no value - let's KISS! + // eslint-disable-next-line no-unused-vars + ExtStorage: { + async set(id, data) { + lazy.Logger.logInfo(`setting data for '${id}': ${data}`); + await lazy.extensionStorageSync.set({ id }, data); + }, + async verify(id, keys, data) { + let got = await lazy.extensionStorageSync.get({ id }, keys); + lazy.Logger.AssertEqual(got, data, `data for '${id}'/${keys}`); + }, + }, +}; + +// Initialize TPS +TPS._init(); diff --git a/services/sync/tps/extensions/tps/schema.json b/services/sync/tps/extensions/tps/schema.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/services/sync/tps/extensions/tps/schema.json @@ -0,0 +1 @@ +[] |