summaryrefslogtreecommitdiffstats
path: root/services/sync/tps
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /services/sync/tps
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'services/sync/tps')
-rw-r--r--services/sync/tps/extensions/tps/api.js77
-rw-r--r--services/sync/tps/extensions/tps/manifest.json23
-rw-r--r--services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs209
-rw-r--r--services/sync/tps/extensions/tps/resource/logger.sys.mjs170
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs93
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs1065
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs833
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs128
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs205
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/history.sys.mjs158
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs187
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs122
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs92
-rw-r--r--services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs32
-rw-r--r--services/sync/tps/extensions/tps/resource/quit.sys.mjs38
-rw-r--r--services/sync/tps/extensions/tps/resource/tps.jsm1614
-rw-r--r--services/sync/tps/extensions/tps/schema.json1
17 files changed, 5047 insertions, 0 deletions
diff --git a/services/sync/tps/extensions/tps/api.js b/services/sync/tps/extensions/tps/api.js
new file mode 100644
index 0000000000..686abdc6fd
--- /dev/null
+++ b/services/sync/tps/extensions/tps/api.js
@@ -0,0 +1,77 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { FileUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/FileUtils.sys.mjs"
+);
+
+/* globals ExtensionAPI, Services, XPCOMUtils */
+
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "resProto",
+ "@mozilla.org/network/protocol;1?name=resource",
+ "nsISubstitutingProtocolHandler"
+);
+
+async function tpsStartup() {
+ try {
+ var { TPS } = ChromeUtils.import("resource://tps/tps.jsm");
+ let { goQuitApplication } = ChromeUtils.importESModule(
+ "resource://tps/quit.sys.mjs"
+ );
+ TPS.goQuitApplication = goQuitApplication;
+
+ let testFile = Services.prefs.getStringPref("testing.tps.testFile", "");
+ let testPhase = Services.prefs.getStringPref("testing.tps.testPhase", "");
+ if (!testFile || !testPhase) {
+ // Note: this quits.
+ TPS.DumpError(
+ "TPS no longer takes arguments from the command line. " +
+ "instead you need to pass preferences `testing.tps.{testFile,testPhase}` " +
+ "and optionally `testing.tps.{logFile,ignoreUnusedEngines}`.\n"
+ );
+ }
+
+ let logFile = Services.prefs.getStringPref("testing.tps.logFile", "");
+ let ignoreUnusedEngines = Services.prefs.getBoolPref(
+ "testing.tps.ignoreUnusedEngines",
+ false
+ );
+ let options = { ignoreUnusedEngines };
+ let testFileUri = Services.io.newFileURI(new FileUtils.File(testFile)).spec;
+
+ try {
+ await TPS.RunTestPhase(testFileUri, testPhase, logFile, options);
+ } catch (err) {
+ TPS.DumpError("TestPhase failed", err);
+ }
+ } catch (e) {
+ if (typeof TPS != "undefined") {
+ // Note: This calls quit() under the hood
+ TPS.DumpError("Test initialization failed", e);
+ }
+ dump(`TPS test initialization failed: ${e} - ${e.stack}\n`);
+ // Try and quit right away, no reason to wait around for python
+ // to kill us if initialization failed.
+ Services.startup.quit(Ci.nsIAppStartup.eForceQuit);
+ }
+}
+
+this.tps = class extends ExtensionAPI {
+ onStartup() {
+ resProto.setSubstitution(
+ "tps",
+ Services.io.newURI("resource/", null, this.extension.rootURI)
+ );
+ /* Ignore the platform's online/offline status while running tests. */
+ Services.io.manageOfflineStatus = false;
+ Services.io.offline = false;
+ tpsStartup();
+ }
+
+ onShutdown() {
+ resProto.setSubstitution("tps", null);
+ }
+};
diff --git a/services/sync/tps/extensions/tps/manifest.json b/services/sync/tps/extensions/tps/manifest.json
new file mode 100644
index 0000000000..c961e76506
--- /dev/null
+++ b/services/sync/tps/extensions/tps/manifest.json
@@ -0,0 +1,23 @@
+{
+ "manifest_version": 2,
+ "name": "TPS",
+ "version": "1.0",
+
+ "browser_specific_settings": {
+ "gecko": {
+ "id": "tps@mozilla.org"
+ }
+ },
+
+ "experiment_apis": {
+ "tps": {
+ "schema": "schema.json",
+ "parent": {
+ "scopes": ["addon_parent"],
+ "script": "api.js",
+ "paths": [["tps"]],
+ "events": ["startup"]
+ }
+ }
+ }
+}
diff --git a/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs
new file mode 100644
index 0000000000..81c0fd578a
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs
@@ -0,0 +1,209 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+import { clearTimeout, setTimeout } from "resource://gre/modules/Timer.sys.mjs";
+
+import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs";
+
+const fxAccounts = getFxAccountsSingleton();
+import { FxAccountsClient } from "resource://gre/modules/FxAccountsClient.sys.mjs";
+import { FxAccountsConfig } from "resource://gre/modules/FxAccountsConfig.sys.mjs";
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+/**
+ * Helper object for Firefox Accounts authentication
+ */
+export var Authentication = {
+ /**
+ * Check if an user has been logged in
+ */
+ async isLoggedIn() {
+ return !!(await this.getSignedInUser());
+ },
+
+ async isReady() {
+ let user = await this.getSignedInUser();
+ return user && user.verified;
+ },
+
+ _getRestmailUsername(user) {
+ const restmailSuffix = "@restmail.net";
+ if (user.toLowerCase().endsWith(restmailSuffix)) {
+ return user.slice(0, -restmailSuffix.length);
+ }
+ return null;
+ },
+
+ async shortWaitForVerification(ms) {
+ let userData = await this.getSignedInUser();
+ let timeoutID;
+ let timeoutPromise = new Promise(resolve => {
+ timeoutID = setTimeout(() => {
+ Logger.logInfo(`Warning: no verification after ${ms}ms.`);
+ resolve();
+ }, ms);
+ });
+ await Promise.race([
+ fxAccounts.whenVerified(userData).finally(() => clearTimeout(timeoutID)),
+ timeoutPromise,
+ ]);
+ userData = await this.getSignedInUser();
+ return userData && userData.verified;
+ },
+
+ async _openVerificationPage(uri) {
+ let mainWindow = Services.wm.getMostRecentWindow("navigator:browser");
+ let newtab = mainWindow.gBrowser.addWebTab(uri);
+ let win = mainWindow.gBrowser.getBrowserForTab(newtab);
+ await new Promise(resolve => {
+ win.addEventListener("loadend", resolve, { once: true });
+ });
+ let didVerify = await this.shortWaitForVerification(10000);
+ mainWindow.gBrowser.removeTab(newtab);
+ return didVerify;
+ },
+
+ async _completeVerification(user) {
+ let username = this._getRestmailUsername(user);
+ if (!username) {
+ Logger.logInfo(
+ `Username "${user}" isn't a restmail username so can't complete verification`
+ );
+ return false;
+ }
+ Logger.logInfo("Fetching mail (from restmail) for user " + username);
+ let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent(
+ username
+ )}`;
+ let triedAlready = new Set();
+ const tries = 10;
+ const normalWait = 2000;
+ for (let i = 0; i < tries; ++i) {
+ let resp = await fetch(restmailURI);
+ let messages = await resp.json();
+ // Sort so that the most recent emails are first.
+ messages.sort((a, b) => new Date(b.receivedAt) - new Date(a.receivedAt));
+ for (let m of messages) {
+ // We look for a link that has a x-link that we haven't yet tried.
+ if (!m.headers["x-link"] || triedAlready.has(m.headers["x-link"])) {
+ continue;
+ }
+ let confirmLink = m.headers["x-link"];
+ triedAlready.add(confirmLink);
+ Logger.logInfo("Trying confirmation link " + confirmLink);
+ try {
+ if (await this._openVerificationPage(confirmLink)) {
+ return true;
+ }
+ } catch (e) {
+ Logger.logInfo(
+ "Warning: Failed to follow confirmation link: " +
+ Log.exceptionStr(e)
+ );
+ }
+ }
+ if (i === 0) {
+ // first time through after failing we'll do this.
+ await fxAccounts.resendVerificationEmail();
+ }
+ if (await this.shortWaitForVerification(normalWait)) {
+ return true;
+ }
+ }
+ // One last try.
+ return this.shortWaitForVerification(normalWait);
+ },
+
+ async deleteEmail(user) {
+ let username = this._getRestmailUsername(user);
+ if (!username) {
+ Logger.logInfo("Not a restmail username, can't delete");
+ return false;
+ }
+ Logger.logInfo("Deleting mail (from restmail) for user " + username);
+ let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent(
+ username
+ )}`;
+ try {
+ // Clean up after ourselves.
+ let deleteResult = await fetch(restmailURI, { method: "DELETE" });
+ if (!deleteResult.ok) {
+ Logger.logInfo(
+ `Warning: Got non-success status ${deleteResult.status} when deleting emails`
+ );
+ return false;
+ }
+ } catch (e) {
+ Logger.logInfo(
+ "Warning: Failed to delete old emails: " + Log.exceptionStr(e)
+ );
+ return false;
+ }
+ return true;
+ },
+
+ /**
+ * Wrapper to retrieve the currently signed in user
+ *
+ * @returns Information about the currently signed in user
+ */
+ async getSignedInUser() {
+ try {
+ return await fxAccounts.getSignedInUser();
+ } catch (error) {
+ Logger.logError(
+ "getSignedInUser() failed with: " + JSON.stringify(error)
+ );
+ throw error;
+ }
+ },
+
+ /**
+ * Wrapper to synchronize the login of a user
+ *
+ * @param account
+ * Account information of the user to login
+ * @param account.username
+ * The username for the account (utf8)
+ * @param account.password
+ * The user's password
+ */
+ async signIn(account) {
+ Logger.AssertTrue(account.username, "Username has been found");
+ Logger.AssertTrue(account.password, "Password has been found");
+
+ Logger.logInfo("Login user: " + account.username);
+
+ try {
+ // Required here since we don't go through the real login page
+ await FxAccountsConfig.ensureConfigured();
+
+ let client = new FxAccountsClient();
+ let credentials = await client.signIn(
+ account.username,
+ account.password,
+ true
+ );
+ await fxAccounts._internal.setSignedInUser(credentials);
+ if (!credentials.verified) {
+ await this._completeVerification(account.username);
+ }
+
+ return true;
+ } catch (error) {
+ throw new Error("signIn() failed with: " + error.message);
+ }
+ },
+
+ /**
+ * Sign out of Firefox Accounts.
+ */
+ async signOut() {
+ if (await Authentication.isLoggedIn()) {
+ // Note: This will clean up the device ID.
+ await fxAccounts.signOut();
+ }
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/logger.sys.mjs b/services/sync/tps/extensions/tps/resource/logger.sys.mjs
new file mode 100644
index 0000000000..5d1a0d9ce5
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/logger.sys.mjs
@@ -0,0 +1,170 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ ChromeUtils.import() and acts as a singleton.
+ Only the following listed symbols will exposed on import, and only when
+ and where imported. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
+});
+
+export var Logger = {
+ _foStream: null,
+ _converter: null,
+ _potentialError: null,
+
+ init(path) {
+ if (this._converter != null) {
+ // we're already open!
+ return;
+ }
+
+ if (path) {
+ Services.prefs.setCharPref("tps.logfile", path);
+ } else {
+ path = Services.prefs.getCharPref("tps.logfile");
+ }
+
+ this._file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+ this._file.initWithPath(path);
+ var exists = this._file.exists();
+
+ // Make a file output stream and converter to handle it.
+ this._foStream = Cc[
+ "@mozilla.org/network/file-output-stream;1"
+ ].createInstance(Ci.nsIFileOutputStream);
+ // If the file already exists, append it, otherwise create it.
+ var fileflags = exists ? 0x02 | 0x08 | 0x10 : 0x02 | 0x08 | 0x20;
+
+ this._foStream.init(this._file, fileflags, 0o666, 0);
+ this._converter = Cc[
+ "@mozilla.org/intl/converter-output-stream;1"
+ ].createInstance(Ci.nsIConverterOutputStream);
+ this._converter.init(this._foStream, "UTF-8");
+ },
+
+ write(data) {
+ if (this._converter == null) {
+ console.error("TPS Logger.write called with _converter == null!");
+ return;
+ }
+ this._converter.writeString(data);
+ },
+
+ close() {
+ if (this._converter != null) {
+ this._converter.close();
+ this._converter = null;
+ this._foStream = null;
+ }
+ },
+
+ AssertTrue(bool, msg, showPotentialError) {
+ if (bool) {
+ return;
+ }
+
+ if (showPotentialError && this._potentialError) {
+ msg += "; " + this._potentialError;
+ this._potentialError = null;
+ }
+ throw new Error("ASSERTION FAILED! " + msg);
+ },
+
+ AssertFalse(bool, msg, showPotentialError) {
+ return this.AssertTrue(!bool, msg, showPotentialError);
+ },
+
+ AssertEqual(got, expected, msg) {
+ if (!lazy.ObjectUtils.deepEqual(got, expected)) {
+ throw new Error(
+ "ASSERTION FAILED! " +
+ msg +
+ "; expected " +
+ JSON.stringify(expected) +
+ ", got " +
+ JSON.stringify(got)
+ );
+ }
+ },
+
+ log(msg, withoutPrefix) {
+ dump(msg + "\n");
+ if (withoutPrefix) {
+ this.write(msg + "\n");
+ } else {
+ function pad(n, len) {
+ let s = "0000" + n;
+ return s.slice(-len);
+ }
+
+ let now = new Date();
+ let year = pad(now.getFullYear(), 4);
+ let month = pad(now.getMonth() + 1, 2);
+ let day = pad(now.getDate(), 2);
+ let hour = pad(now.getHours(), 2);
+ let minutes = pad(now.getMinutes(), 2);
+ let seconds = pad(now.getSeconds(), 2);
+ let ms = pad(now.getMilliseconds(), 3);
+
+ this.write(
+ year +
+ "-" +
+ month +
+ "-" +
+ day +
+ " " +
+ hour +
+ ":" +
+ minutes +
+ ":" +
+ seconds +
+ "." +
+ ms +
+ " " +
+ msg +
+ "\n"
+ );
+ }
+ },
+
+ clearPotentialError() {
+ this._potentialError = null;
+ },
+
+ logPotentialError(msg) {
+ this._potentialError = msg;
+ },
+
+ logLastPotentialError(msg) {
+ var message = msg;
+ if (this._potentialError) {
+ message = this._poentialError;
+ this._potentialError = null;
+ }
+ this.log("CROSSWEAVE ERROR: " + message);
+ },
+
+ logError(msg) {
+ this.log("CROSSWEAVE ERROR: " + msg);
+ },
+
+ logInfo(msg, withoutPrefix) {
+ if (withoutPrefix) {
+ this.log(msg, true);
+ } else {
+ this.log("CROSSWEAVE INFO: " + msg);
+ }
+ },
+
+ logPass(msg) {
+ this.log("CROSSWEAVE TEST PASS: " + msg);
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs
new file mode 100644
index 0000000000..596f942a06
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs
@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { AddonManager } from "resource://gre/modules/AddonManager.sys.mjs";
+import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs";
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+export const STATE_ENABLED = 1;
+export const STATE_DISABLED = 2;
+
+export function Addon(TPS, id) {
+ this.TPS = TPS;
+ this.id = id;
+}
+
+Addon.prototype = {
+ addon: null,
+
+ async uninstall() {
+ // find our addon locally
+ let addon = await AddonManager.getAddonByID(this.id);
+ Logger.AssertTrue(
+ !!addon,
+ "could not find addon " + this.id + " to uninstall"
+ );
+ await AddonUtils.uninstallAddon(addon);
+ },
+
+ async find(state) {
+ let addon = await AddonManager.getAddonByID(this.id);
+
+ if (!addon) {
+ Logger.logInfo("Could not find add-on with ID: " + this.id);
+ return false;
+ }
+
+ this.addon = addon;
+
+ Logger.logInfo(
+ "add-on found: " + addon.id + ", enabled: " + !addon.userDisabled
+ );
+ if (state == STATE_ENABLED) {
+ Logger.AssertFalse(addon.userDisabled, "add-on is disabled: " + addon.id);
+ return true;
+ } else if (state == STATE_DISABLED) {
+ Logger.AssertTrue(addon.userDisabled, "add-on is enabled: " + addon.id);
+ return true;
+ } else if (state) {
+ throw new Error("Don't know how to handle state: " + state);
+ } else {
+ // No state, so just checking that it exists.
+ return true;
+ }
+ },
+
+ async install() {
+ // For Install, the id parameter initially passed is really the filename
+ // for the addon's install .xml; we'll read the actual id from the .xml.
+
+ const result = await AddonUtils.installAddons([
+ { id: this.id, requireSecureURI: false },
+ ]);
+
+ Logger.AssertEqual(
+ 1,
+ result.installedIDs.length,
+ "Exactly 1 add-on was installed."
+ );
+ Logger.AssertEqual(
+ this.id,
+ result.installedIDs[0],
+ "Add-on was installed successfully: " + this.id
+ );
+ },
+
+ async setEnabled(flag) {
+ Logger.AssertTrue(await this.find(), "Add-on is available.");
+
+ let userDisabled;
+ if (flag == STATE_ENABLED) {
+ userDisabled = false;
+ } else if (flag == STATE_DISABLED) {
+ userDisabled = true;
+ } else {
+ throw new Error("Unknown flag to setEnabled: " + flag);
+ }
+
+ AddonUtils.updateUserDisabled(this.addon, userDisabled);
+
+ return true;
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs
new file mode 100644
index 0000000000..e075b55149
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs
@@ -0,0 +1,1065 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file was moved to tps from the main production code as it was unused
+// after removal of the non-mirror bookmarks engine.
+// It used to have a test before it was moved:
+// https://searchfox.org/mozilla-central/rev/b1a5802e0f73bfd6d2096e5fefc2b47831a50b2d/services/sync/tests/unit/test_bookmark_validator.js
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { Utils } from "resource://services-sync/util.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Async: "resource://services-common/async.sys.mjs",
+ PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+const QUERY_PROTOCOL = "place:";
+
+function areURLsEqual(a, b) {
+ if (a === b) {
+ return true;
+ }
+ if (a.startsWith(QUERY_PROTOCOL) != b.startsWith(QUERY_PROTOCOL)) {
+ return false;
+ }
+ // Tag queries are special because we rewrite them to point to the
+ // local tag folder ID. It's expected that the folders won't match,
+ // but all other params should.
+ let aParams = new URLSearchParams(a.slice(QUERY_PROTOCOL.length));
+ let aType = +aParams.get("type");
+ if (aType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
+ return false;
+ }
+ let bParams = new URLSearchParams(b.slice(QUERY_PROTOCOL.length));
+ let bType = +bParams.get("type");
+ if (bType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
+ return false;
+ }
+ let aKeys = new Set(aParams.keys());
+ let bKeys = new Set(bParams.keys());
+ if (aKeys.size != bKeys.size) {
+ return false;
+ }
+ // Tag queries shouldn't reference multiple folders, or named folders like
+ // "TOOLBAR" or "BOOKMARKS_MENU". Just in case, we make sure all folder IDs
+ // are numeric. If they are, we ignore them when comparing the query params.
+ if (aKeys.has("folder") && aParams.getAll("folder").every(isFinite)) {
+ aKeys.delete("folder");
+ }
+ if (bKeys.has("folder") && bParams.getAll("folder").every(isFinite)) {
+ bKeys.delete("folder");
+ }
+ for (let key of aKeys) {
+ if (!bKeys.has(key)) {
+ return false;
+ }
+ if (
+ !CommonUtils.arrayEqual(
+ aParams.getAll(key).sort(),
+ bParams.getAll(key).sort()
+ )
+ ) {
+ return false;
+ }
+ }
+ for (let key of bKeys) {
+ if (!aKeys.has(key)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+const BOOKMARK_VALIDATOR_VERSION = 1;
+
+/**
+ * Result of bookmark validation. Contains the following fields which describe
+ * server-side problems unless otherwise specified.
+ *
+ * - missingIDs (number): # of objects with missing ids
+ * - duplicates (array of ids): ids seen more than once
+ * - parentChildMismatches (array of {parent: parentid, child: childid}):
+ * instances where the child's parentid and the parent's children array
+ * do not match
+ * - cycles (array of array of ids). List of cycles found in the server-side tree.
+ * - clientCycles (array of array of ids). List of cycles found in the client-side tree.
+ * - orphans (array of {id: string, parent: string}): List of nodes with
+ * either no parentid, or where the parent could not be found.
+ * - missingChildren (array of {parent: id, child: id}):
+ * List of parent/children where the child id couldn't be found
+ * - deletedChildren (array of { parent: id, child: id }):
+ * List of parent/children where child id was a deleted item (but still showed up
+ * in the children array)
+ * - multipleParents (array of {child: id, parents: array of ids}):
+ * List of children that were part of multiple parent arrays
+ * - deletedParents (array of ids) : List of records that aren't deleted but
+ * had deleted parents
+ * - childrenOnNonFolder (array of ids): list of non-folders that still have
+ * children arrays
+ * - duplicateChildren (array of ids): list of records who have the same
+ * child listed multiple times in their children array
+ * - parentNotFolder (array of ids): list of records that have parents that
+ * aren't folders
+ * - rootOnServer (boolean): true if the root came from the server
+ * - badClientRoots (array of ids): Contains any client-side root ids where
+ * the root is missing or isn't a (direct) child of the places root.
+ *
+ * - clientMissing: Array of ids on the server missing from the client
+ * - serverMissing: Array of ids on the client missing from the server
+ * - serverDeleted: Array of ids on the client that the server had marked as deleted.
+ * - serverUnexpected: Array of ids that appear on the server but shouldn't
+ * because the client attempts to never upload them.
+ * - differences: Array of {id: string, differences: string array} recording
+ * the non-structural properties that are differente between the client and server
+ * - structuralDifferences: As above, but contains the items where the differences were
+ * structural, that is, they contained childGUIDs or parentid
+ */
+export class BookmarkProblemData {
+ constructor() {
+ this.rootOnServer = false;
+ this.missingIDs = 0;
+
+ this.duplicates = [];
+ this.parentChildMismatches = [];
+ this.cycles = [];
+ this.clientCycles = [];
+ this.orphans = [];
+ this.missingChildren = [];
+ this.deletedChildren = [];
+ this.multipleParents = [];
+ this.deletedParents = [];
+ this.childrenOnNonFolder = [];
+ this.duplicateChildren = [];
+ this.parentNotFolder = [];
+
+ this.badClientRoots = [];
+ this.clientMissing = [];
+ this.serverMissing = [];
+ this.serverDeleted = [];
+ this.serverUnexpected = [];
+ this.differences = [];
+ this.structuralDifferences = [];
+ }
+
+ /**
+ * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into
+ * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc.
+ */
+ _summarizeDifferences(prefix, diffs) {
+ let diffCounts = new Map();
+ for (let { differences } of diffs) {
+ for (let type of differences) {
+ let name = prefix + ":" + type;
+ let count = diffCounts.get(name) || 0;
+ diffCounts.set(name, count + 1);
+ }
+ }
+ return [...diffCounts].map(([name, count]) => ({ name, count }));
+ }
+
+ /**
+ * Produce a list summarizing problems found. Each entry contains {name, count},
+ * where name is the field name for the problem, and count is the number of times
+ * the problem was encountered.
+ *
+ * Validation has failed if all counts are not 0.
+ *
+ * If the `full` argument is truthy, we also include information about which
+ * properties we saw structural differences in. Currently, this means either
+ * "sdiff:parentid" and "sdiff:childGUIDS" may be present.
+ */
+ getSummary(full) {
+ let result = [
+ { name: "clientMissing", count: this.clientMissing.length },
+ { name: "serverMissing", count: this.serverMissing.length },
+ { name: "serverDeleted", count: this.serverDeleted.length },
+ { name: "serverUnexpected", count: this.serverUnexpected.length },
+
+ {
+ name: "structuralDifferences",
+ count: this.structuralDifferences.length,
+ },
+ { name: "differences", count: this.differences.length },
+
+ { name: "missingIDs", count: this.missingIDs },
+ { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 },
+
+ { name: "duplicates", count: this.duplicates.length },
+ {
+ name: "parentChildMismatches",
+ count: this.parentChildMismatches.length,
+ },
+ { name: "cycles", count: this.cycles.length },
+ { name: "clientCycles", count: this.clientCycles.length },
+ { name: "badClientRoots", count: this.badClientRoots.length },
+ { name: "orphans", count: this.orphans.length },
+ { name: "missingChildren", count: this.missingChildren.length },
+ { name: "deletedChildren", count: this.deletedChildren.length },
+ { name: "multipleParents", count: this.multipleParents.length },
+ { name: "deletedParents", count: this.deletedParents.length },
+ { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length },
+ { name: "duplicateChildren", count: this.duplicateChildren.length },
+ { name: "parentNotFolder", count: this.parentNotFolder.length },
+ ];
+ if (full) {
+ let structural = this._summarizeDifferences(
+ "sdiff",
+ this.structuralDifferences
+ );
+ result.push.apply(result, structural);
+ }
+ return result;
+ }
+}
+
+// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon.
+XPCOMUtils.defineLazyGetter(lazy, "SYNCED_ROOTS", () => [
+ lazy.PlacesUtils.bookmarks.menuGuid,
+ lazy.PlacesUtils.bookmarks.toolbarGuid,
+ lazy.PlacesUtils.bookmarks.unfiledGuid,
+ lazy.PlacesUtils.bookmarks.mobileGuid,
+]);
+
+// Maps root GUIDs to their query folder names from
+// toolkit/components/places/nsNavHistoryQuery.cpp. We follow queries that
+// reference existing folders in the client tree, and detect cycles where a
+// query references its containing folder.
+XPCOMUtils.defineLazyGetter(lazy, "ROOT_GUID_TO_QUERY_FOLDER_NAME", () => ({
+ [lazy.PlacesUtils.bookmarks.rootGuid]: "PLACES_ROOT",
+ [lazy.PlacesUtils.bookmarks.menuGuid]: "BOOKMARKS_MENU",
+
+ // Tags should never show up in our client tree, and never form cycles, but we
+ // report them just in case.
+ [lazy.PlacesUtils.bookmarks.tagsGuid]: "TAGS",
+
+ [lazy.PlacesUtils.bookmarks.unfiledGuid]: "UNFILED_BOOKMARKS",
+ [lazy.PlacesUtils.bookmarks.toolbarGuid]: "TOOLBAR",
+ [lazy.PlacesUtils.bookmarks.mobileGuid]: "MOBILE_BOOKMARKS",
+}));
+
+async function detectCycles(records) {
+ // currentPath and pathLookup contain the same data. pathLookup is faster to
+ // query, but currentPath gives is the order of traversal that we need in
+ // order to report the members of the cycles.
+ let pathLookup = new Set();
+ let currentPath = [];
+ let cycles = [];
+ let seenEver = new Set();
+ const yieldState = lazy.Async.yieldState();
+
+ const traverse = async node => {
+ if (pathLookup.has(node)) {
+ let cycleStart = currentPath.lastIndexOf(node);
+ let cyclePath = currentPath.slice(cycleStart).map(n => n.id);
+ cycles.push(cyclePath);
+ return;
+ } else if (seenEver.has(node)) {
+ // If we're checking the server, this is a problem, but it should already be reported.
+ // On the client, this could happen due to including `node.concrete` in the child list.
+ return;
+ }
+ seenEver.add(node);
+ let children = node.children || [];
+ if (node.concreteItems) {
+ children.push(...node.concreteItems);
+ }
+ if (children.length) {
+ pathLookup.add(node);
+ currentPath.push(node);
+ await lazy.Async.yieldingForEach(children, traverse, yieldState);
+ currentPath.pop();
+ pathLookup.delete(node);
+ }
+ };
+
+ await lazy.Async.yieldingForEach(
+ records,
+ async record => {
+ if (!seenEver.has(record)) {
+ await traverse(record);
+ }
+ },
+ yieldState
+ );
+
+ return cycles;
+}
+
+class ServerRecordInspection {
+ constructor() {
+ this.serverRecords = null;
+ this.liveRecords = [];
+
+ this.folders = [];
+
+ this.root = null;
+
+ this.idToRecord = new Map();
+
+ this.deletedIds = new Set();
+ this.deletedRecords = [];
+
+ this.problemData = new BookmarkProblemData();
+
+ // These are handled outside of problemData
+ this._orphans = new Map();
+ this._multipleParents = new Map();
+
+ this.yieldState = lazy.Async.yieldState();
+ }
+
+ static async create(records) {
+ return new ServerRecordInspection().performInspection(records);
+ }
+
+ async performInspection(records) {
+ await this._setRecords(records);
+ await this._linkParentIDs();
+ await this._linkChildren();
+ await this._findOrphans();
+ await this._finish();
+ return this;
+ }
+
+ // We don't set orphans in this.problemData. Instead, we walk the tree at the
+ // end to find unreachable items.
+ _noteOrphan(id, parentId = undefined) {
+ // This probably shouldn't be called with a parentId twice, but if it
+ // happens we take the most recent one.
+ if (parentId || !this._orphans.has(id)) {
+ this._orphans.set(id, parentId);
+ }
+ }
+
+ noteParent(child, parent) {
+ let parents = this._multipleParents.get(child);
+ if (!parents) {
+ this._multipleParents.set(child, [parent]);
+ } else {
+ parents.push(parent);
+ }
+ }
+
+ noteMismatch(child, parent) {
+ let exists = this.problemData.parentChildMismatches.some(
+ match => match.child == child && match.parent == parent
+ );
+ if (!exists) {
+ this.problemData.parentChildMismatches.push({ child, parent });
+ }
+ }
+
+ // - Populates `this.deletedIds`, `this.folders`, and `this.idToRecord`
+ // - calls `_initRoot` (thus initializing `this.root`).
+ async _setRecords(records) {
+ if (this.serverRecords) {
+ // In general this class is expected to be created, have
+ // `performInspection` called, and then only read from from that point on.
+ throw new Error("Bug: ServerRecordInspection can't `setRecords` twice");
+ }
+ this.serverRecords = records;
+ let rootChildren = [];
+
+ await lazy.Async.yieldingForEach(
+ this.serverRecords,
+ async record => {
+ if (!record.id) {
+ ++this.problemData.missingIDs;
+ return;
+ }
+
+ if (record.deleted) {
+ this.deletedIds.add(record.id);
+ }
+ if (this.idToRecord.has(record.id)) {
+ this.problemData.duplicates.push(record.id);
+ return;
+ }
+
+ this.idToRecord.set(record.id, record);
+
+ if (!record.deleted) {
+ this.liveRecords.push(record);
+
+ if (record.parentid == "places") {
+ rootChildren.push(record);
+ }
+ }
+
+ if (!record.children) {
+ return;
+ }
+
+ if (record.type != "folder") {
+ // Due to implementation details in engines/bookmarks.js, (Livemark
+ // subclassing BookmarkFolder) Livemarks will have a children array,
+ // but it should still be empty.
+ if (!record.children.length) {
+ return;
+ }
+ // Otherwise we mark it as an error and still try to resolve the children
+ this.problemData.childrenOnNonFolder.push(record.id);
+ }
+
+ this.folders.push(record);
+
+ if (new Set(record.children).size !== record.children.length) {
+ this.problemData.duplicateChildren.push(record.id);
+ }
+
+ // After we're through with them, folder records store 3 (ugh) arrays that
+ // represent their folder information. The final fields looks like:
+ //
+ // - childGUIDs: The original `children` array, which is an array of
+ // record IDs.
+ //
+ // - unfilteredChildren: Contains more or less `childGUIDs.map(id =>
+ // idToRecord.get(id))`, without the nulls for missing children. It will
+ // still have deleted, duplicate, mismatching, etc. children.
+ //
+ // - children: This is the 'cleaned' version of the child records that are
+ // safe to iterate over, etc.. If there are no reported problems, it should
+ // be identical to unfilteredChildren.
+ //
+ // The last two are left alone until later `this._linkChildren`, however.
+ record.childGUIDs = record.children;
+
+ await lazy.Async.yieldingForEach(
+ record.childGUIDs,
+ id => {
+ this.noteParent(id, record.id);
+ },
+ this.yieldState
+ );
+
+ record.children = [];
+ },
+ this.yieldState
+ );
+
+ // Finish up some parts we can easily do now that we have idToRecord.
+ this.deletedRecords = Array.from(this.deletedIds, id =>
+ this.idToRecord.get(id)
+ );
+
+ this._initRoot(rootChildren);
+ }
+
+ _initRoot(rootChildren) {
+ let serverRoot = this.idToRecord.get("places");
+ if (serverRoot) {
+ this.root = serverRoot;
+ this.problemData.rootOnServer = true;
+ return;
+ }
+
+ // Fabricate a root. We want to be able to remember that it's fake, but
+ // would like to avoid it needing too much special casing, so we come up
+ // with children for it too (we just get these while we're iterating over
+ // the records to avoid needing two passes over a potentially large number
+ // of records).
+
+ this.root = {
+ id: "places",
+ fake: true,
+ children: rootChildren,
+ childGUIDs: rootChildren.map(record => record.id),
+ type: "folder",
+ title: "",
+ };
+ this.liveRecords.push(this.root);
+ this.idToRecord.set("places", this.root);
+ }
+
+ // Adds `parent` to all records it can that have `parentid`
+ async _linkParentIDs() {
+ await lazy.Async.yieldingForEach(
+ this.idToRecord,
+ ([id, record]) => {
+ if (record == this.root || record.deleted) {
+ return false;
+ }
+
+ // Check and update our orphan map.
+ let parentID = record.parentid;
+ let parent = this.idToRecord.get(parentID);
+ if (!parentID || !parent) {
+ this._noteOrphan(id, parentID);
+ return false;
+ }
+
+ record.parent = parent;
+
+ if (parent.deleted) {
+ this.problemData.deletedParents.push(id);
+ return true;
+ } else if (parent.type != "folder") {
+ this.problemData.parentNotFolder.push(record.id);
+ return true;
+ }
+
+ if (parent.id !== "place" || this.problemData.rootOnServer) {
+ if (!parent.childGUIDs.includes(record.id)) {
+ this.noteMismatch(record.id, parent.id);
+ }
+ }
+
+ if (parent.deleted && !record.deleted) {
+ this.problemData.deletedParents.push(record.id);
+ }
+
+ // Note: We used to check if the parentName on the server matches the
+ // actual local parent name, but given this is used only for de-duping a
+ // record the first time it is seen and expensive to keep up-to-date, we
+ // decided to just stop recording it. See bug 1276969 for more.
+ return false;
+ },
+ this.yieldState
+ );
+ }
+
+ // Build the children and unfilteredChildren arrays, (which are of record
+ // objects, not ids)
+ async _linkChildren() {
+ // Check that we aren't missing any children.
+ await lazy.Async.yieldingForEach(
+ this.folders,
+ async folder => {
+ folder.children = [];
+ folder.unfilteredChildren = [];
+
+ let idsThisFolder = new Set();
+
+ await lazy.Async.yieldingForEach(
+ folder.childGUIDs,
+ childID => {
+ let child = this.idToRecord.get(childID);
+
+ if (!child) {
+ this.problemData.missingChildren.push({
+ parent: folder.id,
+ child: childID,
+ });
+ return;
+ }
+
+ if (child.deleted) {
+ this.problemData.deletedChildren.push({
+ parent: folder.id,
+ child: childID,
+ });
+ return;
+ }
+
+ if (child.parentid != folder.id) {
+ this.noteMismatch(childID, folder.id);
+ return;
+ }
+
+ if (idsThisFolder.has(childID)) {
+ // Already recorded earlier, we just don't want to mess up `children`
+ return;
+ }
+ folder.children.push(child);
+ },
+ this.yieldState
+ );
+ },
+ this.yieldState
+ );
+ }
+
+ // Finds the orphans in the tree using something similar to a `mark and sweep`
+ // strategy. That is, we iterate over the children from the root, remembering
+ // which items we've seen. Then, we iterate all items, and know the ones we
+ // haven't seen are orphans.
+ async _findOrphans() {
+ let seen = new Set([this.root.id]);
+
+ const inCycle = await lazy.Async.yieldingForEach(
+ Utils.walkTree(this.root),
+ ([node]) => {
+ if (seen.has(node.id)) {
+ // We're in an infloop due to a cycle.
+ // Return early to avoid reporting false positives for orphans.
+ return true;
+ }
+ seen.add(node.id);
+
+ return false;
+ },
+ this.yieldState
+ );
+
+ if (inCycle) {
+ return;
+ }
+
+ await lazy.Async.yieldingForEach(
+ this.liveRecords,
+ (record, i) => {
+ if (!seen.has(record.id)) {
+ // We intentionally don't record the parentid here, since we only record
+ // that if the record refers to a parent that doesn't exist, which we
+ // have already handled (when linking parentid's).
+ this._noteOrphan(record.id);
+ }
+ },
+ this.yieldState
+ );
+
+ await lazy.Async.yieldingForEach(
+ this._orphans,
+ ([id, parent]) => {
+ this.problemData.orphans.push({ id, parent });
+ },
+ this.yieldState
+ );
+ }
+
+ async _finish() {
+ this.problemData.cycles = await detectCycles(this.liveRecords);
+
+ for (const [child, recordedParents] of this._multipleParents) {
+ let parents = new Set(recordedParents);
+ if (parents.size > 1) {
+ this.problemData.multipleParents.push({ child, parents: [...parents] });
+ }
+ }
+ // Dedupe simple arrays in the problem data, so that we don't have to worry
+ // about it in the code
+ const idArrayProps = [
+ "duplicates",
+ "deletedParents",
+ "childrenOnNonFolder",
+ "duplicateChildren",
+ "parentNotFolder",
+ ];
+ for (let prop of idArrayProps) {
+ this.problemData[prop] = [...new Set(this.problemData[prop])];
+ }
+ }
+}
+
+export class BookmarkValidator {
+ constructor() {
+ this.yieldState = lazy.Async.yieldState();
+ }
+
+ async canValidate() {
+ return !(await lazy.PlacesSyncUtils.bookmarks.havePendingChanges());
+ }
+
+ async _followQueries(recordsByQueryId) {
+ await lazy.Async.yieldingForEach(
+ recordsByQueryId.values(),
+ entry => {
+ if (
+ entry.type !== "query" &&
+ (!entry.bmkUri || !entry.bmkUri.startsWith(QUERY_PROTOCOL))
+ ) {
+ return;
+ }
+ let params = new URLSearchParams(
+ entry.bmkUri.slice(QUERY_PROTOCOL.length)
+ );
+ // Queries with `excludeQueries` won't form cycles because they'll
+ // exclude all queries, including themselves, from the result set.
+ let excludeQueries = params.get("excludeQueries");
+ if (excludeQueries === "1" || excludeQueries === "true") {
+ // `nsNavHistoryQuery::ParseQueryBooleanString` allows `1` and `true`.
+ return;
+ }
+ entry.concreteItems = [];
+ let queryIds = params.getAll("folder");
+ for (let queryId of queryIds) {
+ let concreteItem = recordsByQueryId.get(queryId);
+ if (concreteItem) {
+ entry.concreteItems.push(concreteItem);
+ }
+ }
+ },
+ this.yieldState
+ );
+ }
+
+ async createClientRecordsFromTree(clientTree) {
+ // Iterate over the treeNode, converting it to something more similar to what
+ // the server stores.
+ let records = [];
+ // A map of local IDs and well-known query folder names to records. Unlike
+ // GUIDs, local IDs aren't synced, since they're not stable across devices.
+ // New Places APIs use GUIDs to refer to bookmarks, but the legacy APIs
+ // still use local IDs. We use this mapping to parse `place:` queries that
+ // refer to folders via their local IDs.
+ let recordsByQueryId = new Map();
+ let syncedRoots = lazy.SYNCED_ROOTS;
+
+ const traverse = async (treeNode, synced) => {
+ if (!synced) {
+ synced = syncedRoots.includes(treeNode.guid);
+ }
+ let localId = treeNode.id;
+ let guid = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(treeNode.guid);
+ let itemType = "item";
+ treeNode.ignored = !synced;
+ treeNode.id = guid;
+ switch (treeNode.type) {
+ case lazy.PlacesUtils.TYPE_X_MOZ_PLACE:
+ if (treeNode.uri.startsWith(QUERY_PROTOCOL)) {
+ itemType = "query";
+ } else {
+ itemType = "bookmark";
+ }
+ break;
+ case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
+ let isLivemark = false;
+ if (treeNode.annos) {
+ for (let anno of treeNode.annos) {
+ if (anno.name === lazy.PlacesUtils.LMANNO_FEEDURI) {
+ isLivemark = true;
+ treeNode.feedUri = anno.value;
+ } else if (anno.name === lazy.PlacesUtils.LMANNO_SITEURI) {
+ isLivemark = true;
+ treeNode.siteUri = anno.value;
+ }
+ }
+ }
+ itemType = isLivemark ? "livemark" : "folder";
+ break;
+ case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
+ itemType = "separator";
+ break;
+ }
+
+ if (treeNode.tags) {
+ treeNode.tags = treeNode.tags.split(",");
+ } else {
+ treeNode.tags = [];
+ }
+ treeNode.type = itemType;
+ treeNode.pos = treeNode.index;
+ treeNode.bmkUri = treeNode.uri;
+ records.push(treeNode);
+ if (treeNode.guid in lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME) {
+ let queryId = lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME[treeNode.guid];
+ recordsByQueryId.set(queryId, treeNode);
+ }
+ if (localId) {
+ // Always add the ID, since it's still possible for a query to
+ // reference a root without using the well-known name. For example,
+ // `place:folder=${PlacesUtils.mobileFolderId}` and
+ // `place:folder=MOBILE_BOOKMARKS` are equivalent.
+ recordsByQueryId.set(localId.toString(10), treeNode);
+ }
+ if (treeNode.type === "folder") {
+ treeNode.childGUIDs = [];
+ if (!treeNode.children) {
+ treeNode.children = [];
+ }
+
+ await lazy.Async.yieldingForEach(
+ treeNode.children,
+ async child => {
+ await traverse(child, synced);
+ child.parent = treeNode;
+ child.parentid = guid;
+ treeNode.childGUIDs.push(child.guid);
+ },
+ this.yieldState
+ );
+ }
+ };
+
+ await traverse(clientTree, false);
+
+ clientTree.id = "places";
+ await this._followQueries(recordsByQueryId);
+ return records;
+ }
+
+ /**
+ * Process the server-side list. Mainly this builds the records into a tree,
+ * but it also records information about problems, and produces arrays of the
+ * deleted and non-deleted nodes.
+ *
+ * Returns an object containing:
+ * - records:Array of non-deleted records. Each record contains the following
+ * properties
+ * - childGUIDs (array of strings, only present if type is 'folder'): the
+ * list of child GUIDs stored on the server.
+ * - children (array of records, only present if type is 'folder'):
+ * each record has these same properties. This may differ in content
+ * from what you may expect from the childGUIDs list, as it won't
+ * contain any records that could not be found.
+ * - parent (record): The parent to this record.
+ * - Unchanged properties send down from the server: id, title, type,
+ * parentName, parentid, bmkURI, keyword, tags, pos, queryId
+ * - root: Root of the server-side bookmark tree. Has the same properties as
+ * above.
+ * - deletedRecords: As above, but only contains items that the server sent
+ * where it also sent indication that the item should be deleted.
+ * - problemData: a BookmarkProblemData object, with the caveat that
+ * the fields describing client/server relationship will not have been filled
+ * out yet.
+ */
+ async inspectServerRecords(serverRecords) {
+ const data = await ServerRecordInspection.create(serverRecords);
+ return {
+ deletedRecords: data.deletedRecords,
+ records: data.liveRecords,
+ problemData: data.problemData,
+ root: data.root,
+ };
+ }
+
+ // Perform client-side sanity checking that doesn't involve server data
+ async _validateClient(problemData, clientRecords) {
+ problemData.clientCycles = await detectCycles(clientRecords);
+ for (let rootGUID of lazy.SYNCED_ROOTS) {
+ let record = clientRecords.find(record => record.guid === rootGUID);
+ if (!record || record.parentid !== "places") {
+ problemData.badClientRoots.push(rootGUID);
+ }
+ }
+ }
+
+ async _computeUnifiedRecordMap(serverRecords, clientRecords) {
+ let allRecords = new Map();
+ await lazy.Async.yieldingForEach(
+ serverRecords,
+ sr => {
+ if (sr.fake) {
+ return;
+ }
+ allRecords.set(sr.id, { client: null, server: sr });
+ },
+ this.yieldState
+ );
+
+ await lazy.Async.yieldingForEach(
+ clientRecords,
+ cr => {
+ let unified = allRecords.get(cr.id);
+ if (!unified) {
+ allRecords.set(cr.id, { client: cr, server: null });
+ } else {
+ unified.client = cr;
+ }
+ },
+ this.yieldState
+ );
+
+ return allRecords;
+ }
+
+ _recordMissing(problems, id, clientRecord, serverRecord, serverTombstones) {
+ if (!clientRecord && serverRecord) {
+ problems.clientMissing.push(id);
+ }
+ if (!serverRecord && clientRecord) {
+ if (serverTombstones.has(id)) {
+ problems.serverDeleted.push(id);
+ } else if (!clientRecord.ignored && clientRecord.id != "places") {
+ problems.serverMissing.push(id);
+ }
+ }
+ }
+
+ _compareRecords(client, server) {
+ let structuralDifferences = [];
+ let differences = [];
+
+ // Don't bother comparing titles of roots. It's okay if locally it's
+ // "Mobile Bookmarks", but the server thinks it's "mobile".
+ // TODO: We probably should be handing other localized bookmarks (e.g.
+ // default bookmarks) here as well, see bug 1316041.
+ if (!lazy.SYNCED_ROOTS.includes(client.guid)) {
+ // We want to treat undefined, null and an empty string as identical
+ if ((client.title || "") !== (server.title || "")) {
+ differences.push("title");
+ }
+ }
+
+ if (client.parentid || server.parentid) {
+ if (client.parentid !== server.parentid) {
+ structuralDifferences.push("parentid");
+ }
+ }
+
+ if (client.tags || server.tags) {
+ let cl = client.tags ? [...client.tags].sort() : [];
+ let sl = server.tags ? [...server.tags].sort() : [];
+ if (!CommonUtils.arrayEqual(cl, sl)) {
+ differences.push("tags");
+ }
+ }
+
+ let sameType = client.type === server.type;
+ if (!sameType) {
+ if (
+ server.type === "query" &&
+ client.type === "bookmark" &&
+ client.bmkUri.startsWith(QUERY_PROTOCOL)
+ ) {
+ sameType = true;
+ }
+ }
+
+ if (!sameType) {
+ differences.push("type");
+ } else {
+ switch (server.type) {
+ case "bookmark":
+ case "query":
+ if (!areURLsEqual(server.bmkUri, client.bmkUri)) {
+ differences.push("bmkUri");
+ }
+ break;
+ case "separator":
+ if (server.pos != client.pos) {
+ differences.push("pos");
+ }
+ break;
+ case "livemark":
+ if (server.feedUri != client.feedUri) {
+ differences.push("feedUri");
+ }
+ if (server.siteUri != client.siteUri) {
+ differences.push("siteUri");
+ }
+ break;
+ case "folder":
+ if (server.id === "places" && server.fake) {
+ // It's the fabricated places root. It won't have the GUIDs, but
+ // it doesn't matter.
+ break;
+ }
+ if (client.childGUIDs || server.childGUIDs) {
+ let cl = client.childGUIDs || [];
+ let sl = server.childGUIDs || [];
+ if (!CommonUtils.arrayEqual(cl, sl)) {
+ structuralDifferences.push("childGUIDs");
+ }
+ }
+ break;
+ }
+ }
+ return { differences, structuralDifferences };
+ }
+
+ /**
+ * Compare the list of server records with the client tree.
+ *
+ * Returns the same data as described in the inspectServerRecords comment,
+ * with the following additional fields.
+ * - clientRecords: an array of client records in a similar format to
+ * the .records (ie, server records) entry.
+ * - problemData is the same as for inspectServerRecords, except all properties
+ * will be filled out.
+ */
+ async compareServerWithClient(serverRecords, clientTree) {
+ let clientRecords = await this.createClientRecordsFromTree(clientTree);
+ let inspectionInfo = await this.inspectServerRecords(serverRecords);
+ inspectionInfo.clientRecords = clientRecords;
+
+ // Mainly do this to remove deleted items and normalize child guids.
+ serverRecords = inspectionInfo.records;
+ let problemData = inspectionInfo.problemData;
+
+ await this._validateClient(problemData, clientRecords);
+
+ let allRecords = await this._computeUnifiedRecordMap(
+ serverRecords,
+ clientRecords
+ );
+
+ let serverDeleted = new Set(inspectionInfo.deletedRecords.map(r => r.id));
+
+ await lazy.Async.yieldingForEach(
+ allRecords,
+ ([id, { client, server }]) => {
+ if (!client || !server) {
+ this._recordMissing(problemData, id, client, server, serverDeleted);
+ return;
+ }
+ if (server && client && client.ignored) {
+ problemData.serverUnexpected.push(id);
+ }
+ let { differences, structuralDifferences } = this._compareRecords(
+ client,
+ server
+ );
+
+ if (differences.length) {
+ problemData.differences.push({ id, differences });
+ }
+ if (structuralDifferences.length) {
+ problemData.structuralDifferences.push({
+ id,
+ differences: structuralDifferences,
+ });
+ }
+ },
+ this.yieldState
+ );
+
+ return inspectionInfo;
+ }
+
+ async _getServerState(engine) {
+ let collection = engine.itemSource();
+ let collectionKey = engine.service.collectionKeys.keyForCollection(
+ engine.name
+ );
+ collection.full = true;
+ let result = await collection.getBatched();
+ if (!result.response.success) {
+ throw result.response;
+ }
+ let cleartexts = [];
+ await lazy.Async.yieldingForEach(
+ result.records,
+ async record => {
+ await record.decrypt(collectionKey);
+ cleartexts.push(record.cleartext);
+ },
+ this.yieldState
+ );
+ return cleartexts;
+ }
+
+ async validate(engine) {
+ let start = Date.now();
+ let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", {
+ includeItemIds: true,
+ });
+ let serverState = await this._getServerState(engine);
+ let serverRecordCount = serverState.length;
+ let result = await this.compareServerWithClient(serverState, clientTree);
+ let end = Date.now();
+ let duration = end - start;
+
+ engine._log.debug(`Validated bookmarks in ${duration}ms`);
+ engine._log.debug(`Problem summary`);
+ for (let { name, count } of result.problemData.getSummary()) {
+ engine._log.debug(` ${name}: ${count}`);
+ }
+
+ return {
+ duration,
+ version: this.version,
+ problems: result.problemData,
+ recordCount: serverRecordCount,
+ };
+ }
+}
+
+BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION;
diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs
new file mode 100644
index 0000000000..e4aac948b5
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs
@@ -0,0 +1,833 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ * ChromeUtils.import() and acts as a singleton. Only the following
+ * listed symbols will exposed on import, and only when and where imported.
+ */
+
+import { PlacesBackups } from "resource://gre/modules/PlacesBackups.sys.mjs";
+
+import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs";
+import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs";
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+export async function DumpBookmarks() {
+ let [bookmarks] = await PlacesBackups.getBookmarksTree();
+ Logger.logInfo(
+ "Dumping Bookmarks...\n" + JSON.stringify(bookmarks, undefined, 2) + "\n\n"
+ );
+}
+
+/**
+ * extend, causes a child object to inherit from a parent
+ */
+function extend(child, supertype) {
+ Object.setPrototypeOf(child.prototype, supertype.prototype);
+}
+/**
+ * PlacesItemProps object, holds properties for places items
+ */
+function PlacesItemProps(props) {
+ this.location = null;
+ this.uri = null;
+ this.keyword = null;
+ this.title = null;
+ this.after = null;
+ this.before = null;
+ this.folder = null;
+ this.position = null;
+ this.delete = false;
+ this.tags = null;
+ this.last_item_pos = null;
+ this.type = null;
+
+ for (var prop in props) {
+ if (prop in this) {
+ this[prop] = props[prop];
+ }
+ }
+}
+
+/**
+ * PlacesItem object. Base class for places items.
+ */
+export function PlacesItem(props) {
+ this.props = new PlacesItemProps(props);
+ if (this.props.location == null) {
+ this.props.location = "menu";
+ }
+ if ("changes" in props) {
+ this.updateProps = new PlacesItemProps(props.changes);
+ } else {
+ this.updateProps = null;
+ }
+}
+
+/**
+ * Instance methods for generic places items.
+ */
+PlacesItem.prototype = {
+ // an array of possible root folders for places items
+ _bookmarkFolders: {
+ places: PlacesUtils.bookmarks.rootGuid,
+ menu: PlacesUtils.bookmarks.menuGuid,
+ tags: PlacesUtils.bookmarks.tagsGuid,
+ unfiled: PlacesUtils.bookmarks.unfiledGuid,
+ toolbar: PlacesUtils.bookmarks.toolbarGuid,
+ mobile: PlacesUtils.bookmarks.mobileGuid,
+ },
+
+ _typeMap: new Map([
+ [PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER, PlacesUtils.bookmarks.TYPE_FOLDER],
+ [
+ PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR,
+ PlacesUtils.bookmarks.TYPE_SEPARATOR,
+ ],
+ [PlacesUtils.TYPE_X_MOZ_PLACE, PlacesUtils.bookmarks.TYPE_BOOKMARK],
+ ]),
+
+ toString() {
+ var that = this;
+ var props = ["uri", "title", "location", "folder"];
+ var string =
+ (this.props.type ? this.props.type + " " : "") +
+ "(" +
+ (function () {
+ var ret = [];
+ for (var i in props) {
+ if (that.props[props[i]]) {
+ ret.push(props[i] + ": " + that.props[props[i]]);
+ }
+ }
+ return ret;
+ })().join(", ") +
+ ")";
+ return string;
+ },
+
+ /**
+ * GetPlacesChildGuid
+ *
+ * Finds the guid of the an item with the specified properties in the places
+ * database under the specified parent.
+ *
+ * @param folder The guid of the folder to search
+ * @param type The type of the item to find, or null to match any item;
+ * this is one of the PlacesUtils.bookmarks.TYPE_* values
+ * @param title The title of the item to find, or null to match any title
+ * @param uri The uri of the item to find, or null to match any uri
+ *
+ * @return the node id if the item was found, otherwise null
+ */
+ async GetPlacesChildGuid(folder, type, title, uri) {
+ let children = (await PlacesUtils.promiseBookmarksTree(folder)).children;
+ if (!children) {
+ return null;
+ }
+ let guid = null;
+ for (let node of children) {
+ if (node.title == title) {
+ let nodeType = this._typeMap.get(node.type);
+ if (type == null || type == undefined || nodeType == type) {
+ if (uri == undefined || uri == null || node.uri.spec == uri.spec) {
+ // Note that this is suspect as we return the *last* matching
+ // child, which some tests rely on (ie, an early-return here causes
+ // at least 1 test to fail). But that's a yak for another day.
+ guid = node.guid;
+ }
+ }
+ }
+ }
+ return guid;
+ },
+
+ /**
+ * IsAdjacentTo
+ *
+ * Determines if this object is immediately adjacent to another.
+ *
+ * @param itemName The name of the other object; this may be any kind of
+ * places item
+ * @param relativePos The relative position of the other object. If -1,
+ * it means the other object should precede this one, if +1,
+ * the other object should come after this one
+ * @return true if this object is immediately adjacent to the other object,
+ * otherwise false
+ */
+ async IsAdjacentTo(itemName, relativePos) {
+ Logger.AssertTrue(
+ this.props.folder_id != -1 && this.props.guid != null,
+ "Either folder_id or guid was invalid"
+ );
+ let otherGuid = await this.GetPlacesChildGuid(
+ this.props.parentGuid,
+ null,
+ itemName
+ );
+ Logger.AssertTrue(otherGuid, "item " + itemName + " not found");
+ let other_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index;
+ let this_pos = (await PlacesUtils.bookmarks.fetch(this.props.guid)).index;
+ if (other_pos + relativePos != this_pos) {
+ Logger.logPotentialError(
+ "Invalid position - " +
+ (this.props.title ? this.props.title : this.props.folder) +
+ " not " +
+ (relativePos == 1 ? "after " : "before ") +
+ itemName +
+ " for " +
+ this.toString()
+ );
+ return false;
+ }
+ return true;
+ },
+
+ /**
+ * GetItemIndex
+ *
+ * Gets the item index for this places item.
+ *
+ * @return the item index, or -1 if there's an error
+ */
+ async GetItemIndex() {
+ if (this.props.guid == null) {
+ return -1;
+ }
+ return (await PlacesUtils.bookmarks.fetch(this.props.guid)).index;
+ },
+
+ /**
+ * GetFolder
+ *
+ * Gets the folder guid for the specified bookmark folder
+ *
+ * @param location The full path of the folder, which must begin
+ * with one of the bookmark root folders
+ * @return the folder guid if the folder is found, otherwise null
+ */
+ async GetFolder(location) {
+ let folder_parts = location.split("/");
+ if (!(folder_parts[0] in this._bookmarkFolders)) {
+ return null;
+ }
+ let folderGuid = this._bookmarkFolders[folder_parts[0]];
+ for (let i = 1; i < folder_parts.length; i++) {
+ let guid = await this.GetPlacesChildGuid(
+ folderGuid,
+ PlacesUtils.bookmarks.TYPE_FOLDER,
+ folder_parts[i]
+ );
+ if (guid == null) {
+ return null;
+ }
+ folderGuid = guid;
+ }
+ return folderGuid;
+ },
+
+ /**
+ * CreateFolder
+ *
+ * Creates a bookmark folder.
+ *
+ * @param location The full path of the folder, which must begin
+ * with one of the bookmark root folders
+ * @return the folder id if the folder was created, otherwise -1
+ */
+ async CreateFolder(location) {
+ let folder_parts = location.split("/");
+ if (!(folder_parts[0] in this._bookmarkFolders)) {
+ return -1;
+ }
+ let folderGuid = this._bookmarkFolders[folder_parts[0]];
+ for (let i = 1; i < folder_parts.length; i++) {
+ let subfolderGuid = await this.GetPlacesChildGuid(
+ folderGuid,
+ PlacesUtils.bookmarks.TYPE_FOLDER,
+ folder_parts[i]
+ );
+ if (subfolderGuid == null) {
+ let { guid } = await PlacesUtils.bookmarks.insert({
+ parentGuid: folderGuid,
+ name: folder_parts[i],
+ type: PlacesUtils.bookmarks.TYPE_FOLDER,
+ });
+ folderGuid = guid;
+ } else {
+ folderGuid = subfolderGuid;
+ }
+ }
+ return folderGuid;
+ },
+
+ /**
+ * GetOrCreateFolder
+ *
+ * Locates the specified folder; if not found it is created.
+ *
+ * @param location The full path of the folder, which must begin
+ * with one of the bookmark root folders
+ * @return the folder id if the folder was found or created, otherwise -1
+ */
+ async GetOrCreateFolder(location) {
+ let parentGuid = await this.GetFolder(location);
+ if (parentGuid == null) {
+ parentGuid = await this.CreateFolder(location);
+ }
+ return parentGuid;
+ },
+
+ /**
+ * CheckPosition
+ *
+ * Verifies the position of this places item.
+ *
+ * @param before The name of the places item that this item should be
+ before, or null if this check should be skipped
+ * @param after The name of the places item that this item should be
+ after, or null if this check should be skipped
+ * @param last_item_pos The index of the places item above this one,
+ * or null if this check should be skipped
+ * @return true if this item is in the correct position, otherwise false
+ */
+ async CheckPosition(before, after, last_item_pos) {
+ if (after) {
+ if (!(await this.IsAdjacentTo(after, 1))) {
+ return false;
+ }
+ }
+ if (before) {
+ if (!(await this.IsAdjacentTo(before, -1))) {
+ return false;
+ }
+ }
+ if (last_item_pos != null && last_item_pos > -1) {
+ let index = await this.GetItemIndex();
+ if (index != last_item_pos + 1) {
+ Logger.logPotentialError(
+ "Item not found at the expected index, got " +
+ index +
+ ", expected " +
+ (last_item_pos + 1) +
+ " for " +
+ this.toString()
+ );
+ return false;
+ }
+ }
+ return true;
+ },
+
+ /**
+ * SetLocation
+ *
+ * Moves this places item to a different folder.
+ *
+ * @param location The full path of the folder to which to move this
+ * places item, which must begin with one of the bookmark root
+ * folders; if null, no changes are made
+ * @return nothing if successful, otherwise an exception is thrown
+ */
+ async SetLocation(location) {
+ if (location != null) {
+ let newfolderGuid = await this.GetOrCreateFolder(location);
+ Logger.AssertTrue(
+ newfolderGuid,
+ "Location " + location + " doesn't exist; can't change item's location"
+ );
+ await PlacesUtils.bookmarks.update({
+ guid: this.props.guid,
+ parentGuid: newfolderGuid,
+ index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+ });
+ this.props.parentGuid = newfolderGuid;
+ }
+ },
+
+ /**
+ * SetPosition
+ *
+ * Updates the position of this places item within this item's current
+ * folder. Use SetLocation to change folders.
+ *
+ * @param position The new index this item should be moved to; if null,
+ * no changes are made; if -1, this item is moved to the bottom of
+ * the current folder. Otherwise, must be a string which is the
+ * title of an existing item in the folder, who's current position
+ * is used as the index.
+ * @return nothing if successful, otherwise an exception is thrown
+ */
+ async SetPosition(position) {
+ if (position == null) {
+ return;
+ }
+ let index = -1;
+ if (position != -1) {
+ let existingGuid = await this.GetPlacesChildGuid(
+ this.props.parentGuid,
+ null,
+ position
+ );
+ if (existingGuid) {
+ index = (await PlacesUtils.bookmarks.fetch(existingGuid)).index;
+ }
+ Logger.AssertTrue(
+ index != -1,
+ "position " + position + " is invalid; unable to change position"
+ );
+ }
+ await PlacesUtils.bookmarks.update({ guid: this.props.guid, index });
+ },
+
+ /**
+ * Update the title of this places item
+ *
+ * @param title The new title to set for this item; if null, no changes
+ * are made
+ * @return nothing
+ */
+ async SetTitle(title) {
+ if (title != null) {
+ await PlacesUtils.bookmarks.update({ guid: this.props.guid, title });
+ }
+ },
+};
+
+/**
+ * Bookmark class constructor. Initializes instance properties.
+ */
+export function Bookmark(props) {
+ PlacesItem.call(this, props);
+ if (this.props.title == null) {
+ this.props.title = this.props.uri;
+ }
+ this.props.type = "bookmark";
+}
+
+/**
+ * Bookmark instance methods.
+ */
+Bookmark.prototype = {
+ /**
+ * SetKeyword
+ *
+ * Update this bookmark's keyword.
+ *
+ * @param keyword The keyword to set for this bookmark; if null, no
+ * changes are made
+ * @return nothing
+ */
+ async SetKeyword(keyword) {
+ if (keyword != null) {
+ // Mirror logic from PlacesSyncUtils's updateBookmarkMetadata
+ let entry = await PlacesUtils.keywords.fetch({ url: this.props.uri });
+ if (entry) {
+ await PlacesUtils.keywords.remove(entry);
+ }
+ await PlacesUtils.keywords.insert({ keyword, url: this.props.uri });
+ }
+ },
+
+ /**
+ * SetUri
+ *
+ * Updates this bookmark's URI.
+ *
+ * @param uri The new URI to set for this boomark; if null, no changes
+ * are made
+ * @return nothing
+ */
+ async SetUri(uri) {
+ if (uri) {
+ let url = Services.io.newURI(uri);
+ await PlacesUtils.bookmarks.update({ guid: this.props.guid, url });
+ }
+ },
+
+ /**
+ * SetTags
+ *
+ * Updates this bookmark's tags.
+ *
+ * @param tags An array of tags which should be associated with this
+ * bookmark; any previous tags are removed; if this param is null,
+ * no changes are made. If this param is an empty array, all
+ * tags are removed from this bookmark.
+ * @return nothing
+ */
+ SetTags(tags) {
+ if (tags != null) {
+ let URI = Services.io.newURI(this.props.uri);
+ PlacesUtils.tagging.untagURI(URI, null);
+ if (tags.length) {
+ PlacesUtils.tagging.tagURI(URI, tags);
+ }
+ }
+ },
+
+ /**
+ * Create
+ *
+ * Creates the bookmark described by this object's properties.
+ *
+ * @return the id of the created bookmark
+ */
+ async Create() {
+ this.props.parentGuid = await this.GetOrCreateFolder(this.props.location);
+ Logger.AssertTrue(
+ this.props.parentGuid,
+ "Unable to create " +
+ "bookmark, error creating folder " +
+ this.props.location
+ );
+ let bookmarkURI = Services.io.newURI(this.props.uri);
+ let { guid } = await PlacesUtils.bookmarks.insert({
+ parentGuid: this.props.parentGuid,
+ url: bookmarkURI,
+ title: this.props.title,
+ });
+ this.props.guid = guid;
+ await this.SetKeyword(this.props.keyword);
+ await this.SetTags(this.props.tags);
+ return this.props.guid;
+ },
+
+ /**
+ * Update
+ *
+ * Updates this bookmark's properties according the properties on this
+ * object's 'updateProps' property.
+ *
+ * @return nothing
+ */
+ async Update() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Update");
+ await this.SetTitle(this.updateProps.title);
+ await this.SetUri(this.updateProps.uri);
+ await this.SetKeyword(this.updateProps.keyword);
+ await this.SetTags(this.updateProps.tags);
+ await this.SetLocation(this.updateProps.location);
+ await this.SetPosition(this.updateProps.position);
+ },
+
+ /**
+ * Find
+ *
+ * Locates the bookmark which corresponds to this object's properties.
+ *
+ * @return the bookmark guid if the bookmark was found, otherwise null
+ */
+ async Find() {
+ this.props.parentGuid = await this.GetFolder(this.props.location);
+
+ if (this.props.parentGuid == null) {
+ Logger.logError("Unable to find folder " + this.props.location);
+ return null;
+ }
+ let bookmarkTitle = this.props.title;
+ this.props.guid = await this.GetPlacesChildGuid(
+ this.props.parentGuid,
+ null,
+ bookmarkTitle,
+ this.props.uri
+ );
+
+ if (!this.props.guid) {
+ Logger.logPotentialError(this.toString() + " not found");
+ return null;
+ }
+ if (this.props.keyword != null) {
+ let { keyword } = await PlacesSyncUtils.bookmarks.fetch(this.props.guid);
+ if (keyword != this.props.keyword) {
+ Logger.logPotentialError(
+ "Incorrect keyword - expected: " +
+ this.props.keyword +
+ ", actual: " +
+ keyword +
+ " for " +
+ this.toString()
+ );
+ return null;
+ }
+ }
+ if (this.props.tags != null) {
+ try {
+ let URI = Services.io.newURI(this.props.uri);
+ let tags = PlacesUtils.tagging.getTagsForURI(URI);
+ tags.sort();
+ this.props.tags.sort();
+ if (JSON.stringify(tags) != JSON.stringify(this.props.tags)) {
+ Logger.logPotentialError(
+ "Wrong tags - expected: " +
+ JSON.stringify(this.props.tags) +
+ ", actual: " +
+ JSON.stringify(tags) +
+ " for " +
+ this.toString()
+ );
+ return null;
+ }
+ } catch (e) {
+ Logger.logPotentialError("error processing tags " + e);
+ return null;
+ }
+ }
+ if (
+ !(await this.CheckPosition(
+ this.props.before,
+ this.props.after,
+ this.props.last_item_pos
+ ))
+ ) {
+ return null;
+ }
+ return this.props.guid;
+ },
+
+ /**
+ * Remove
+ *
+ * Removes this bookmark. The bookmark should have been located previously
+ * by a call to Find.
+ *
+ * @return nothing
+ */
+ async Remove() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Remove");
+ await PlacesUtils.bookmarks.remove(this.props.guid);
+ },
+};
+
+extend(Bookmark, PlacesItem);
+
+/**
+ * BookmarkFolder class constructor. Initializes instance properties.
+ */
+export function BookmarkFolder(props) {
+ PlacesItem.call(this, props);
+ this.props.type = "folder";
+}
+
+/**
+ * BookmarkFolder instance methods
+ */
+BookmarkFolder.prototype = {
+ /**
+ * Create
+ *
+ * Creates the bookmark folder described by this object's properties.
+ *
+ * @return the id of the created bookmark folder
+ */
+ async Create() {
+ this.props.parentGuid = await this.GetOrCreateFolder(this.props.location);
+ Logger.AssertTrue(
+ this.props.parentGuid,
+ "Unable to create " +
+ "folder, error creating parent folder " +
+ this.props.location
+ );
+ let { guid } = await PlacesUtils.bookmarks.insert({
+ parentGuid: this.props.parentGuid,
+ title: this.props.folder,
+ index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+ type: PlacesUtils.bookmarks.TYPE_FOLDER,
+ });
+ this.props.guid = guid;
+ return this.props.parentGuid;
+ },
+
+ /**
+ * Find
+ *
+ * Locates the bookmark folder which corresponds to this object's
+ * properties.
+ *
+ * @return the folder guid if the folder was found, otherwise null
+ */
+ async Find() {
+ this.props.parentGuid = await this.GetFolder(this.props.location);
+ if (this.props.parentGuid == null) {
+ Logger.logError("Unable to find folder " + this.props.location);
+ return null;
+ }
+ this.props.guid = await this.GetPlacesChildGuid(
+ this.props.parentGuid,
+ PlacesUtils.bookmarks.TYPE_FOLDER,
+ this.props.folder
+ );
+ if (this.props.guid == null) {
+ return null;
+ }
+ if (
+ !(await this.CheckPosition(
+ this.props.before,
+ this.props.after,
+ this.props.last_item_pos
+ ))
+ ) {
+ return null;
+ }
+ return this.props.guid;
+ },
+
+ /**
+ * Remove
+ *
+ * Removes this folder. The folder should have been located previously
+ * by a call to Find.
+ *
+ * @return nothing
+ */
+ async Remove() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Remove");
+ await PlacesUtils.bookmarks.remove(this.props.guid);
+ },
+
+ /**
+ * Update
+ *
+ * Updates this bookmark's properties according the properties on this
+ * object's 'updateProps' property.
+ *
+ * @return nothing
+ */
+ async Update() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Update");
+ await this.SetLocation(this.updateProps.location);
+ await this.SetPosition(this.updateProps.position);
+ await this.SetTitle(this.updateProps.folder);
+ },
+};
+
+extend(BookmarkFolder, PlacesItem);
+
+/**
+ * Separator class constructor. Initializes instance properties.
+ */
+export function Separator(props) {
+ PlacesItem.call(this, props);
+ this.props.type = "separator";
+}
+
+/**
+ * Separator instance methods.
+ */
+Separator.prototype = {
+ /**
+ * Create
+ *
+ * Creates the bookmark separator described by this object's properties.
+ *
+ * @return the id of the created separator
+ */
+ async Create() {
+ this.props.parentGuid = await this.GetOrCreateFolder(this.props.location);
+ Logger.AssertTrue(
+ this.props.parentGuid,
+ "Unable to create " +
+ "folder, error creating parent folder " +
+ this.props.location
+ );
+ let { guid } = await PlacesUtils.bookmarks.insert({
+ parentGuid: this.props.parentGuid,
+ type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+ });
+ this.props.guid = guid;
+ return guid;
+ },
+
+ /**
+ * Find
+ *
+ * Locates the bookmark separator which corresponds to this object's
+ * properties.
+ *
+ * @return the item guid if the separator was found, otherwise null
+ */
+ async Find() {
+ this.props.parentGuid = await this.GetFolder(this.props.location);
+ if (this.props.parentGuid == null) {
+ Logger.logError("Unable to find folder " + this.props.location);
+ return null;
+ }
+ if (this.props.before == null && this.props.last_item_pos == null) {
+ Logger.logPotentialError(
+ "Separator requires 'before' attribute if it's the" +
+ "first item in the list"
+ );
+ return null;
+ }
+ let expected_pos = -1;
+ if (this.props.before) {
+ let otherGuid = this.GetPlacesChildGuid(
+ this.props.parentGuid,
+ null,
+ this.props.before
+ );
+ if (otherGuid == null) {
+ Logger.logPotentialError(
+ "Can't find places item " +
+ this.props.before +
+ " for locating separator"
+ );
+ return null;
+ }
+ expected_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index - 1;
+ } else {
+ expected_pos = this.props.last_item_pos + 1;
+ }
+ // Note these are IDs instead of GUIDs.
+ let children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+ this.props.parentGuid
+ );
+ this.props.guid = children[expected_pos];
+ if (this.props.guid == null) {
+ Logger.logPotentialError(
+ "No separator found at position " + expected_pos
+ );
+ return null;
+ }
+ let info = await PlacesUtils.bookmarks.fetch(this.props.guid);
+ if (info.type != PlacesUtils.bookmarks.TYPE_SEPARATOR) {
+ Logger.logPotentialError(
+ "Places item at position " + expected_pos + " is not a separator"
+ );
+ return null;
+ }
+ return this.props.guid;
+ },
+
+ /**
+ * Update
+ *
+ * Updates this separator's properties according the properties on this
+ * object's 'updateProps' property.
+ *
+ * @return nothing
+ */
+ async Update() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Update");
+ await this.SetLocation(this.updateProps.location);
+ await this.SetPosition(this.updateProps.position);
+ return true;
+ },
+
+ /**
+ * Remove
+ *
+ * Removes this separator. The separator should have been located
+ * previously by a call to Find.
+ *
+ * @return nothing
+ */
+ async Remove() {
+ Logger.AssertTrue(this.props.guid, "Invalid guid during Update");
+ await PlacesUtils.bookmarks.remove(this.props.guid);
+ },
+};
+
+extend(Separator, PlacesItem);
diff --git a/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs
new file mode 100644
index 0000000000..587d7668f4
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs
@@ -0,0 +1,128 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ * ChromeUtils.import() and acts as a singleton. Only the following
+ * listed symbols will exposed on import, and only when and where imported.
+ */
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ OSKeyStore: "resource://gre/modules/OSKeyStore.sys.mjs",
+ formAutofillStorage: "resource://autofill/FormAutofillStorage.sys.mjs",
+});
+
+class FormAutofillBase {
+ constructor(props, subStorageName, fields) {
+ this._subStorageName = subStorageName;
+ this._fields = fields;
+
+ this.props = {};
+ this.updateProps = null;
+ if ("changes" in props) {
+ this.updateProps = props.changes;
+ }
+ for (const field of this._fields) {
+ this.props[field] = field in props ? props[field] : null;
+ }
+ }
+
+ async getStorage() {
+ await lazy.formAutofillStorage.initialize();
+ return lazy.formAutofillStorage[this._subStorageName];
+ }
+
+ async Create() {
+ const storage = await this.getStorage();
+ await storage.add(this.props);
+ }
+
+ async Find() {
+ const storage = await this.getStorage();
+ return storage._data.find(entry =>
+ this._fields.every(field => entry[field] === this.props[field])
+ );
+ }
+
+ async Update() {
+ const storage = await this.getStorage();
+ const { guid } = await this.Find();
+ await storage.update(guid, this.updateProps, true);
+ }
+
+ async Remove() {
+ const storage = await this.getStorage();
+ const { guid } = await this.Find();
+ storage.remove(guid);
+ }
+}
+
+async function DumpStorage(subStorageName) {
+ await lazy.formAutofillStorage.initialize();
+ Logger.logInfo(`\ndumping ${subStorageName} list\n`, true);
+ const entries = lazy.formAutofillStorage[subStorageName]._data;
+ for (const entry of entries) {
+ Logger.logInfo(JSON.stringify(entry), true);
+ }
+ Logger.logInfo(`\n\nend ${subStorageName} list\n`, true);
+}
+
+const ADDRESS_FIELDS = [
+ "given-name",
+ "additional-name",
+ "family-name",
+ "organization",
+ "street-address",
+ "address-level2",
+ "address-level1",
+ "postal-code",
+ "country",
+ "tel",
+ "email",
+];
+
+export class Address extends FormAutofillBase {
+ constructor(props) {
+ super(props, "addresses", ADDRESS_FIELDS);
+ }
+}
+
+export async function DumpAddresses() {
+ await DumpStorage("addresses");
+}
+
+const CREDIT_CARD_FIELDS = [
+ "cc-name",
+ "cc-number",
+ "cc-exp-month",
+ "cc-exp-year",
+];
+
+export class CreditCard extends FormAutofillBase {
+ constructor(props) {
+ super(props, "creditCards", CREDIT_CARD_FIELDS);
+ }
+
+ async Find() {
+ const storage = await this.getStorage();
+ await Promise.all(
+ storage._data.map(
+ async entry =>
+ (entry["cc-number"] = await lazy.OSKeyStore.decrypt(
+ entry["cc-number-encrypted"]
+ ))
+ )
+ );
+ return storage._data.find(entry => {
+ return this._fields.every(field => entry[field] === this.props[field]);
+ });
+ }
+}
+
+export async function DumpCreditCards() {
+ await DumpStorage("creditCards");
+}
diff --git a/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs
new file mode 100644
index 0000000000..35b5f5c03b
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs
@@ -0,0 +1,205 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ ChromeUtils.import() and acts as a singleton. Only the following
+ listed symbols will exposed on import, and only when and where imported.
+ */
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+import { FormHistory } from "resource://gre/modules/FormHistory.sys.mjs";
+
+/**
+ * FormDB
+ *
+ * Helper object containing methods to interact with the FormHistory module.
+ */
+var FormDB = {
+ async _update(data) {
+ await FormHistory.update(data);
+ },
+
+ /**
+ * insertValue
+ *
+ * Adds the specified value for the specified fieldname into form history.
+ *
+ * @param fieldname The form fieldname to insert
+ * @param value The form value to insert
+ * @param us The time, in microseconds, to use for the lastUsed
+ * and firstUsed columns
+ * @return Promise<undefined>
+ */
+ insertValue(fieldname, value, us) {
+ let data = {
+ op: "add",
+ fieldname,
+ value,
+ timesUsed: 1,
+ firstUsed: us,
+ lastUsed: us,
+ };
+ return this._update(data);
+ },
+
+ /**
+ * updateValue
+ *
+ * Updates a row in the moz_formhistory table with a new value.
+ *
+ * @param id The id of the row to update
+ * @param newvalue The new value to set
+ * @return Promise<undefined>
+ */
+ updateValue(id, newvalue) {
+ return this._update({ op: "update", guid: id, value: newvalue });
+ },
+
+ /**
+ * getDataForValue
+ *
+ * Retrieves a set of values for a row in the database that
+ * corresponds to the given fieldname and value.
+ *
+ * @param fieldname The fieldname of the row to query
+ * @param value The value of the row to query
+ * @return Promise<null if no row is found with the specified fieldname and value,
+ * or an object containing the row's guid, lastUsed, and firstUsed
+ * values>
+ */
+ async getDataForValue(fieldname, value) {
+ let results = await FormHistory.search(["guid", "lastUsed", "firstUsed"], {
+ fieldname,
+ value,
+ });
+ if (results.length > 1) {
+ throw new Error("more than 1 result for this query");
+ }
+ return results;
+ },
+
+ /**
+ * remove
+ *
+ * Removes the specified GUID from the database.
+ *
+ * @param guid The guid of the item to delete
+ * @return Promise<>
+ */
+ remove(guid) {
+ return this._update({ op: "remove", guid });
+ },
+};
+
+/**
+ * FormData class constructor
+ *
+ * Initializes instance properties.
+ */
+export function FormData(props, msSinceEpoch) {
+ this.fieldname = null;
+ this.value = null;
+ this.date = 0;
+ this.newvalue = null;
+ this.usSinceEpoch = msSinceEpoch * 1000;
+
+ for (var prop in props) {
+ if (prop in this) {
+ this[prop] = props[prop];
+ }
+ }
+}
+
+/**
+ * FormData instance methods
+ */
+FormData.prototype = {
+ /**
+ * hours_to_us
+ *
+ * Converts hours since present to microseconds since epoch.
+ *
+ * @param hours The number of hours since the present time (e.g., 0 is
+ * 'now', and -1 is 1 hour ago)
+ * @return the corresponding number of microseconds since the epoch
+ */
+ hours_to_us(hours) {
+ return this.usSinceEpoch + hours * 60 * 60 * 1000 * 1000;
+ },
+
+ /**
+ * Create
+ *
+ * If this FormData object doesn't exist in the moz_formhistory database,
+ * add it. Throws on error.
+ *
+ * @return nothing
+ */
+ Create() {
+ Logger.AssertTrue(
+ this.fieldname != null && this.value != null,
+ "Must specify both fieldname and value"
+ );
+
+ return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => {
+ if (!formdata) {
+ // this item doesn't exist yet in the db, so we need to insert it
+ return FormDB.insertValue(
+ this.fieldname,
+ this.value,
+ this.hours_to_us(this.date)
+ );
+ }
+ /* Right now, we ignore this case. If bug 552531 is ever fixed,
+ we might need to add code here to update the firstUsed or
+ lastUsed fields, as appropriate.
+ */
+ return null;
+ });
+ },
+
+ /**
+ * Find
+ *
+ * Attempts to locate an entry in the moz_formhistory database that
+ * matches the fieldname and value for this FormData object.
+ *
+ * @return true if this entry exists in the database, otherwise false
+ */
+ Find() {
+ return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => {
+ let status = formdata != null;
+ if (status) {
+ /*
+ //form history dates currently not synced! bug 552531
+ let us = this.hours_to_us(this.date);
+ status = Logger.AssertTrue(
+ us >= formdata.firstUsed && us <= formdata.lastUsed,
+ "No match for with that date value");
+
+ if (status)
+ */
+ this.id = formdata.guid;
+ }
+ return status;
+ });
+ },
+
+ /**
+ * Remove
+ *
+ * Removes the row represented by this FormData instance from the
+ * moz_formhistory database.
+ *
+ * @return nothing
+ */
+ async Remove() {
+ const formdata = await FormDB.getDataForValue(this.fieldname, this.value);
+ if (!formdata) {
+ return;
+ }
+ await FormDB.remove(formdata.guid);
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs
new file mode 100644
index 0000000000..845bab3aa9
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs
@@ -0,0 +1,158 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ * ChromeUtils.import() and acts as a singleton. Only the following
+ * listed symbols will exposed on import, and only when and where imported.
+ */
+
+import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs";
+
+import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs";
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+export var DumpHistory = async function TPS_History__DumpHistory() {
+ let query = PlacesUtils.history.getNewQuery();
+ let options = PlacesUtils.history.getNewQueryOptions();
+ let root = PlacesUtils.history.executeQuery(query, options).root;
+ root.containerOpen = true;
+ Logger.logInfo("\n\ndumping history\n", true);
+ for (var i = 0; i < root.childCount; i++) {
+ let node = root.getChild(i);
+ let uri = node.uri;
+ let guid = await PlacesSyncUtils.history
+ .fetchGuidForURL(uri)
+ .catch(() => "?".repeat(12));
+ let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(uri);
+ for (var visit of curvisits) {
+ Logger.logInfo(
+ `GUID: ${guid}, URI: ${uri}, type=${visit.type}, date=${visit.date}`,
+ true
+ );
+ }
+ }
+ root.containerOpen = false;
+ Logger.logInfo("\nend history dump\n", true);
+};
+
+/**
+ * HistoryEntry object
+ *
+ * Contains methods for manipulating browser history entries.
+ */
+export var HistoryEntry = {
+ /**
+ * Add
+ *
+ * Adds visits for a uri to the history database. Throws on error.
+ *
+ * @param item An object representing one or more visits to a specific uri
+ * @param usSinceEpoch The number of microseconds from Epoch to
+ * the time the current Crossweave run was started
+ * @return nothing
+ */
+ async Add(item, msSinceEpoch) {
+ Logger.AssertTrue(
+ "visits" in item && "uri" in item,
+ "History entry in test file must have both 'visits' " +
+ "and 'uri' properties"
+ );
+ let place = {
+ url: item.uri,
+ visits: [],
+ };
+ for (let visit of item.visits) {
+ let date = new Date(
+ Math.round(msSinceEpoch + visit.date * 60 * 60 * 1000)
+ );
+ place.visits.push({ date, transition: visit.type });
+ }
+ if ("title" in item) {
+ place.title = item.title;
+ }
+ return PlacesUtils.history.insert(place);
+ },
+
+ /**
+ * Find
+ *
+ * Finds visits for a uri to the history database. Throws on error.
+ *
+ * @param item An object representing one or more visits to a specific uri
+ * @param usSinceEpoch The number of microseconds from Epoch to
+ * the time the current Crossweave run was started
+ * @return true if all the visits for the uri are found, otherwise false
+ */
+ async Find(item, msSinceEpoch) {
+ Logger.AssertTrue(
+ "visits" in item && "uri" in item,
+ "History entry in test file must have both 'visits' " +
+ "and 'uri' properties"
+ );
+ let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(item.uri);
+ for (let visit of curvisits) {
+ for (let itemvisit of item.visits) {
+ // Note: in microseconds.
+ let expectedDate =
+ itemvisit.date * 60 * 60 * 1000 * 1000 + msSinceEpoch * 1000;
+ if (visit.type == itemvisit.type) {
+ if (itemvisit.date === undefined || visit.date == expectedDate) {
+ itemvisit.found = true;
+ }
+ }
+ }
+ }
+
+ let all_items_found = true;
+ for (let itemvisit of item.visits) {
+ all_items_found = all_items_found && "found" in itemvisit;
+ Logger.logInfo(
+ `History entry for ${item.uri}, type: ${itemvisit.type}, date: ${itemvisit.date}` +
+ `(${
+ itemvisit.date * 60 * 60 * 1000 * 1000
+ }), found = ${!!itemvisit.found}`
+ );
+ }
+ return all_items_found;
+ },
+
+ /**
+ * Delete
+ *
+ * Removes visits from the history database. Throws on error.
+ *
+ * @param item An object representing items to delete
+ * @param usSinceEpoch The number of microseconds from Epoch to
+ * the time the current Crossweave run was started
+ * @return nothing
+ */
+ async Delete(item, msSinceEpoch) {
+ if ("uri" in item) {
+ let removedAny = await PlacesUtils.history.remove(item.uri);
+ if (!removedAny) {
+ Logger.log("Warning: Removed 0 history visits for uri " + item.uri);
+ }
+ } else if ("host" in item) {
+ await PlacesUtils.history.removeByFilter({ host: item.host });
+ } else if ("begin" in item && "end" in item) {
+ let filter = {
+ beginDate: new Date(msSinceEpoch + item.begin * 60 * 60 * 1000),
+ endDate: new Date(msSinceEpoch + item.end * 60 * 60 * 1000),
+ };
+ let removedAny = await PlacesUtils.history.removeVisitsByFilter(filter);
+ if (!removedAny) {
+ Logger.log(
+ "Warning: Removed 0 history visits with " +
+ JSON.stringify({ item, filter })
+ );
+ }
+ } else {
+ Logger.AssertTrue(
+ false,
+ "invalid entry in delete history " + JSON.stringify(item)
+ );
+ }
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs
new file mode 100644
index 0000000000..12c6d90e98
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs
@@ -0,0 +1,187 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ * ChromeUtils.import() and acts as a singleton. Only the following
+ * listed symbols will exposed on import, and only when and where imported.
+ */
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+var nsLoginInfo = new Components.Constructor(
+ "@mozilla.org/login-manager/loginInfo;1",
+ Ci.nsILoginInfo,
+ "init"
+);
+
+export var DumpPasswords = function TPS__Passwords__DumpPasswords() {
+ let logins = Services.logins.getAllLogins();
+ Logger.logInfo("\ndumping password list\n", true);
+ for (var i = 0; i < logins.length; i++) {
+ Logger.logInfo(
+ "* origin=" +
+ logins[i].origin +
+ ", formActionOrigin=" +
+ logins[i].formActionOrigin +
+ ", realm=" +
+ logins[i].httpRealm +
+ ", password=" +
+ logins[i].password +
+ ", passwordField=" +
+ logins[i].passwordField +
+ ", username=" +
+ logins[i].username +
+ ", usernameField=" +
+ logins[i].usernameField,
+ true
+ );
+ }
+ Logger.logInfo("\n\nend password list\n", true);
+};
+
+/**
+ * PasswordProps object; holds password properties.
+ */
+function PasswordProps(props) {
+ this.hostname = null;
+ this.submitURL = null;
+ this.realm = null;
+ this.username = "";
+ this.password = "";
+ this.usernameField = "";
+ this.passwordField = "";
+ this.delete = false;
+
+ for (var prop in props) {
+ if (prop in this) {
+ this[prop] = props[prop];
+ }
+ }
+}
+
+/**
+ * Password class constructor. Initializes instance properties.
+ */
+export function Password(props) {
+ this.props = new PasswordProps(props);
+ if ("changes" in props) {
+ this.updateProps = new PasswordProps(props);
+ for (var prop in props.changes) {
+ if (prop in this.updateProps) {
+ this.updateProps[prop] = props.changes[prop];
+ }
+ }
+ } else {
+ this.updateProps = null;
+ }
+}
+
+/**
+ * Password instance methods.
+ */
+Password.prototype = {
+ /**
+ * Create
+ *
+ * Adds a password entry to the login manager for the password
+ * represented by this object's properties. Throws on error.
+ *
+ * @return the new login guid
+ */
+ async Create() {
+ let login = new nsLoginInfo(
+ this.props.hostname,
+ this.props.submitURL,
+ this.props.realm,
+ this.props.username,
+ this.props.password,
+ this.props.usernameField,
+ this.props.passwordField
+ );
+ await Services.logins.addLoginAsync(login);
+ login.QueryInterface(Ci.nsILoginMetaInfo);
+ return login.guid;
+ },
+
+ /**
+ * Find
+ *
+ * Finds a password entry in the login manager, for the password
+ * represented by this object's properties.
+ *
+ * @return the guid of the password if found, otherwise -1
+ */
+ Find() {
+ let logins = Services.logins.findLogins(
+ this.props.hostname,
+ this.props.submitURL,
+ this.props.realm
+ );
+ for (var i = 0; i < logins.length; i++) {
+ if (
+ logins[i].username == this.props.username &&
+ logins[i].password == this.props.password &&
+ logins[i].usernameField == this.props.usernameField &&
+ logins[i].passwordField == this.props.passwordField
+ ) {
+ logins[i].QueryInterface(Ci.nsILoginMetaInfo);
+ return logins[i].guid;
+ }
+ }
+ return -1;
+ },
+
+ /**
+ * Update
+ *
+ * Updates an existing password entry in the login manager with
+ * new properties. Throws on error. The 'old' properties are this
+ * object's properties, the 'new' properties are the properties in
+ * this object's 'updateProps' object.
+ *
+ * @return nothing
+ */
+ Update() {
+ let oldlogin = new nsLoginInfo(
+ this.props.hostname,
+ this.props.submitURL,
+ this.props.realm,
+ this.props.username,
+ this.props.password,
+ this.props.usernameField,
+ this.props.passwordField
+ );
+ let newlogin = new nsLoginInfo(
+ this.updateProps.hostname,
+ this.updateProps.submitURL,
+ this.updateProps.realm,
+ this.updateProps.username,
+ this.updateProps.password,
+ this.updateProps.usernameField,
+ this.updateProps.passwordField
+ );
+ Services.logins.modifyLogin(oldlogin, newlogin);
+ },
+
+ /**
+ * Remove
+ *
+ * Removes an entry from the login manager for a password which
+ * matches this object's properties. Throws on error.
+ *
+ * @return nothing
+ */
+ Remove() {
+ let login = new nsLoginInfo(
+ this.props.hostname,
+ this.props.submitURL,
+ this.props.realm,
+ this.props.username,
+ this.props.password,
+ this.props.usernameField,
+ this.props.passwordField
+ );
+ Services.logins.removeLogin(login);
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs
new file mode 100644
index 0000000000..9f6c423a40
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs
@@ -0,0 +1,122 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ ChromeUtils.import() and acts as a singleton.
+ Only the following listed symbols will exposed on import, and only when
+ and where imported. */
+
+const WEAVE_PREF_PREFIX = "services.sync.prefs.sync.";
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+/**
+ * Preference class constructor
+ *
+ * Initializes instance properties.
+ */
+export function Preference(props) {
+ Logger.AssertTrue(
+ "name" in props && "value" in props,
+ "Preference must have both name and value"
+ );
+
+ this.name = props.name;
+ this.value = props.value;
+}
+
+/**
+ * Preference instance methods
+ */
+Preference.prototype = {
+ /**
+ * Modify
+ *
+ * Sets the value of the preference this.name to this.value.
+ * Throws on error.
+ *
+ * @return nothing
+ */
+ Modify() {
+ // Determine if this pref is actually something Weave even looks at.
+ let weavepref = WEAVE_PREF_PREFIX + this.name;
+ try {
+ let syncPref = Services.prefs.getBoolPref(weavepref);
+ if (!syncPref) {
+ Services.prefs.setBoolPref(weavepref, true);
+ }
+ } catch (e) {
+ Logger.AssertTrue(false, "Weave doesn't sync pref " + this.name);
+ }
+
+ // Modify the pref; throw an exception if the pref type is different
+ // than the value type specified in the test.
+ let prefType = Services.prefs.getPrefType(this.name);
+ switch (prefType) {
+ case Ci.nsIPrefBranch.PREF_INT:
+ Logger.AssertEqual(
+ typeof this.value,
+ "number",
+ "Wrong type used for preference value"
+ );
+ Services.prefs.setIntPref(this.name, this.value);
+ break;
+ case Ci.nsIPrefBranch.PREF_STRING:
+ Logger.AssertEqual(
+ typeof this.value,
+ "string",
+ "Wrong type used for preference value"
+ );
+ Services.prefs.setCharPref(this.name, this.value);
+ break;
+ case Ci.nsIPrefBranch.PREF_BOOL:
+ Logger.AssertEqual(
+ typeof this.value,
+ "boolean",
+ "Wrong type used for preference value"
+ );
+ Services.prefs.setBoolPref(this.name, this.value);
+ break;
+ }
+ },
+
+ /**
+ * Find
+ *
+ * Verifies that the preference this.name has the value
+ * this.value. Throws on error, or if the pref's type or value
+ * doesn't match.
+ *
+ * @return nothing
+ */
+ Find() {
+ // Read the pref value.
+ let value;
+ try {
+ let prefType = Services.prefs.getPrefType(this.name);
+ switch (prefType) {
+ case Ci.nsIPrefBranch.PREF_INT:
+ value = Services.prefs.getIntPref(this.name);
+ break;
+ case Ci.nsIPrefBranch.PREF_STRING:
+ value = Services.prefs.getCharPref(this.name);
+ break;
+ case Ci.nsIPrefBranch.PREF_BOOL:
+ value = Services.prefs.getBoolPref(this.name);
+ break;
+ }
+ } catch (e) {
+ Logger.AssertTrue(false, "Error accessing pref " + this.name);
+ }
+
+ // Throw an exception if the current and expected values aren't of
+ // the same type, or don't have the same values.
+ Logger.AssertEqual(
+ typeof value,
+ typeof this.value,
+ "Value types don't match"
+ );
+ Logger.AssertEqual(value, this.value, "Preference values don't match");
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs
new file mode 100644
index 0000000000..8ea8f3b780
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs
@@ -0,0 +1,92 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ ChromeUtils.import() and acts as a singleton.
+ Only the following listed symbols will exposed on import, and only when
+ and where imported. */
+
+import { Weave } from "resource://services-sync/main.sys.mjs";
+
+import { Logger } from "resource://tps/logger.sys.mjs";
+
+// Unfortunately, due to where TPS is run, we can't directly reuse the logic from
+// BrowserTestUtils.sys.mjs. Moreover, we can't resolve the URI it loads the content
+// frame script from ("chrome://mochikit/content/tests/BrowserTestUtils/content-utils.js"),
+// hence the hackiness here and in BrowserTabs.Add.
+Services.mm.loadFrameScript(
+ "data:application/javascript;charset=utf-8," +
+ encodeURIComponent(`
+ addEventListener("load", function(event) {
+ let subframe = event.target != content.document;
+ sendAsyncMessage("tps:loadEvent", {subframe: subframe, url: event.target.documentURI});
+ }, true)`),
+ true,
+ true
+);
+
+export var BrowserTabs = {
+ /**
+ * Add
+ *
+ * Opens a new tab in the current browser window for the
+ * given uri. Rejects on error.
+ *
+ * @param uri The uri to load in the new tab
+ * @return Promise
+ */
+ async Add(uri) {
+ let mainWindow = Services.wm.getMostRecentWindow("navigator:browser");
+ let browser = mainWindow.gBrowser;
+ let newtab = browser.addTrustedTab(uri);
+
+ // Wait for the tab to load.
+ await new Promise(resolve => {
+ let mm = browser.ownerGlobal.messageManager;
+ mm.addMessageListener("tps:loadEvent", function onLoad(msg) {
+ mm.removeMessageListener("tps:loadEvent", onLoad);
+ resolve();
+ });
+ });
+
+ browser.selectedTab = newtab;
+ },
+
+ /**
+ * Find
+ *
+ * Finds the specified uri and title in Weave's list of remote tabs
+ * for the specified profile.
+ *
+ * @param uri The uri of the tab to find
+ * @param title The page title of the tab to find
+ * @param profile The profile to search for tabs
+ * @return true if the specified tab could be found, otherwise false
+ */
+ async Find(uri, title, profile) {
+ // Find the uri in Weave's list of tabs for the given profile.
+ let tabEngine = Weave.Service.engineManager.get("tabs");
+ for (let client of Weave.Service.clientsEngine.remoteClients) {
+ let tabClients = await tabEngine.getAllClients();
+ let tabClient = tabClients.find(x => x.id === client.id);
+ if (!tabClient || !tabClient.tabs) {
+ continue;
+ }
+ for (let key in tabClient.tabs) {
+ let tab = tabClient.tabs[key];
+ let weaveTabUrl = tab.urlHistory[0];
+ if (uri == weaveTabUrl && profile == client.name) {
+ if (title == undefined || title == tab.title) {
+ return true;
+ }
+ }
+ }
+ Logger.logInfo(
+ `Dumping tabs for ${client.name}...\n` +
+ JSON.stringify(tabClient.tabs, null, 2)
+ );
+ }
+ return false;
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs
new file mode 100644
index 0000000000..b0798b9031
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs
@@ -0,0 +1,32 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ ChromeUtils.import() and acts as a singleton.
+ Only the following listed symbols will exposed on import, and only when
+ and where imported. */
+
+export var BrowserWindows = {
+ /**
+ * Add
+ *
+ * Opens a new window. Throws on error.
+ *
+ * @param aPrivate The private option.
+ * @return nothing
+ */
+ Add(aPrivate, fn) {
+ return new Promise(resolve => {
+ let mainWindow = Services.wm.getMostRecentWindow("navigator:browser");
+ let win = mainWindow.OpenBrowserWindow({ private: aPrivate });
+ win.addEventListener(
+ "load",
+ function () {
+ resolve(win);
+ },
+ { once: true }
+ );
+ });
+ },
+};
diff --git a/services/sync/tps/extensions/tps/resource/quit.sys.mjs b/services/sync/tps/extensions/tps/resource/quit.sys.mjs
new file mode 100644
index 0000000000..e2cb8d8c22
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/quit.sys.mjs
@@ -0,0 +1,38 @@
+/* -*- indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ From mozilla/toolkit/content
+ These files did not have a license
+*/
+function canQuitApplication() {
+ try {
+ var cancelQuit = Cc["@mozilla.org/supports-PRBool;1"].createInstance(
+ Ci.nsISupportsPRBool
+ );
+ Services.obs.notifyObservers(cancelQuit, "quit-application-requested");
+
+ // Something aborted the quit process.
+ if (cancelQuit.data) {
+ return false;
+ }
+ } catch (ex) {}
+
+ return true;
+}
+
+export function goQuitApplication() {
+ if (!canQuitApplication()) {
+ return false;
+ }
+
+ try {
+ Services.startup.quit(Ci.nsIAppStartup.eForceQuit);
+ } catch (ex) {
+ throw new Error(`goQuitApplication: ${ex.message}`);
+ }
+
+ return true;
+}
diff --git a/services/sync/tps/extensions/tps/resource/tps.jsm b/services/sync/tps/extensions/tps/resource/tps.jsm
new file mode 100644
index 0000000000..dfecd229aa
--- /dev/null
+++ b/services/sync/tps/extensions/tps/resource/tps.jsm
@@ -0,0 +1,1614 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This is a JavaScript module (JSM) to be imported via
+ * ChromeUtils.import() and acts as a singleton. Only the following
+ * listed symbols will exposed on import, and only when and where imported.
+ */
+
+var EXPORTED_SYMBOLS = [
+ "ACTIONS",
+ "Addons",
+ "Addresses",
+ "Bookmarks",
+ "CreditCards",
+ "ExtensionStorage",
+ "Formdata",
+ "History",
+ "Passwords",
+ "Prefs",
+ "Tabs",
+ "TPS",
+ "Windows",
+];
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+const { PromiseUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/PromiseUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Addon: "resource://tps/modules/addons.sys.mjs",
+ AddonValidator: "resource://services-sync/engines/addons.sys.mjs",
+ Address: "resource://tps/modules/formautofill.sys.mjs",
+ Async: "resource://services-common/async.sys.mjs",
+ Authentication: "resource://tps/auth/fxaccounts.sys.mjs",
+ Bookmark: "resource://tps/modules/bookmarks.sys.mjs",
+ BookmarkFolder: "resource://tps/modules/bookmarks.sys.mjs",
+ BookmarkValidator: "resource://tps/modules/bookmarkValidator.sys.mjs",
+ BrowserTabs: "resource://tps/modules/tabs.sys.mjs",
+ BrowserWindows: "resource://tps/modules/windows.sys.mjs",
+ CommonUtils: "resource://services-common/utils.sys.mjs",
+ CreditCard: "resource://tps/modules/formautofill.sys.mjs",
+ DumpAddresses: "resource://tps/modules/formautofill.sys.mjs",
+ DumpBookmarks: "resource://tps/modules/bookmarks.sys.mjs",
+ DumpCreditCards: "resource://tps/modules/formautofill.sys.mjs",
+ DumpHistory: "resource://tps/modules/history.sys.mjs",
+ DumpPasswords: "resource://tps/modules/passwords.sys.mjs",
+ FileUtils: "resource://gre/modules/FileUtils.sys.mjs",
+ FormData: "resource://tps/modules/forms.sys.mjs",
+ FormValidator: "resource://services-sync/engines/forms.sys.mjs",
+ HistoryEntry: "resource://tps/modules/history.sys.mjs",
+ JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs",
+ Livemark: "resource://tps/modules/bookmarks.sys.mjs",
+ Log: "resource://gre/modules/Log.sys.mjs",
+ Logger: "resource://tps/logger.sys.mjs",
+ Password: "resource://tps/modules/passwords.sys.mjs",
+ PasswordValidator: "resource://services-sync/engines/passwords.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+ Preference: "resource://tps/modules/prefs.sys.mjs",
+ STATUS_OK: "resource://services-sync/constants.sys.mjs",
+ Separator: "resource://tps/modules/bookmarks.sys.mjs",
+ SessionStore: "resource:///modules/sessionstore/SessionStore.sys.mjs",
+ Svc: "resource://services-sync/util.sys.mjs",
+ SyncTelemetry: "resource://services-sync/telemetry.sys.mjs",
+ WEAVE_VERSION: "resource://services-sync/constants.sys.mjs",
+ Weave: "resource://services-sync/main.sys.mjs",
+ extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs",
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "fileProtocolHandler", () => {
+ let fileHandler = Services.io.getProtocolHandler("file");
+ return fileHandler.QueryInterface(Ci.nsIFileProtocolHandler);
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "gTextDecoder", () => {
+ return new TextDecoder();
+});
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "NetUtil",
+ "resource://gre/modules/NetUtil.jsm"
+);
+
+// Options for wiping data during a sync
+const SYNC_RESET_CLIENT = "resetClient";
+const SYNC_WIPE_CLIENT = "wipeClient";
+const SYNC_WIPE_REMOTE = "wipeRemote";
+
+// Actions a test can perform
+const ACTION_ADD = "add";
+const ACTION_DELETE = "delete";
+const ACTION_MODIFY = "modify";
+const ACTION_SET_ENABLED = "set-enabled";
+const ACTION_SYNC = "sync";
+const ACTION_SYNC_RESET_CLIENT = SYNC_RESET_CLIENT;
+const ACTION_SYNC_WIPE_CLIENT = SYNC_WIPE_CLIENT;
+const ACTION_SYNC_WIPE_REMOTE = SYNC_WIPE_REMOTE;
+const ACTION_VERIFY = "verify";
+const ACTION_VERIFY_NOT = "verify-not";
+
+const ACTIONS = [
+ ACTION_ADD,
+ ACTION_DELETE,
+ ACTION_MODIFY,
+ ACTION_SET_ENABLED,
+ ACTION_SYNC,
+ ACTION_SYNC_RESET_CLIENT,
+ ACTION_SYNC_WIPE_CLIENT,
+ ACTION_SYNC_WIPE_REMOTE,
+ ACTION_VERIFY,
+ ACTION_VERIFY_NOT,
+];
+
+const OBSERVER_TOPICS = [
+ "fxaccounts:onlogin",
+ "fxaccounts:onlogout",
+ "profile-before-change",
+ "weave:service:tracking-started",
+ "weave:service:tracking-stopped",
+ "weave:service:login:error",
+ "weave:service:setup-complete",
+ "weave:service:sync:finish",
+ "weave:service:sync:delayed",
+ "weave:service:sync:error",
+ "weave:service:sync:start",
+ "weave:service:resyncs-finished",
+ "places-browser-init-complete",
+];
+
+var TPS = {
+ _currentAction: -1,
+ _currentPhase: -1,
+ _enabledEngines: null,
+ _errors: 0,
+ _isTracking: false,
+ _phaseFinished: false,
+ _phaselist: {},
+ _setupComplete: false,
+ _syncActive: false,
+ _syncCount: 0,
+ _syncsReportedViaTelemetry: 0,
+ _syncErrors: 0,
+ _syncWipeAction: null,
+ _tabsAdded: 0,
+ _tabsFinished: 0,
+ _test: null,
+ _triggeredSync: false,
+ _msSinceEpoch: 0,
+ _requestedQuit: false,
+ shouldValidateAddons: false,
+ shouldValidateBookmarks: false,
+ shouldValidatePasswords: false,
+ shouldValidateForms: false,
+ _placesInitDeferred: PromiseUtils.defer(),
+
+ _init: function TPS__init() {
+ this.delayAutoSync();
+
+ OBSERVER_TOPICS.forEach(function (aTopic) {
+ Services.obs.addObserver(this, aTopic, true);
+ }, this);
+
+ // Some engines bump their score during their sync, which then causes
+ // another sync immediately (notably, prefs and addons). We don't want
+ // this to happen, and there's no obvious preference to kill it - so
+ // we do this nasty hack to ensure the global score is always zero.
+ Services.prefs.addObserver("services.sync.globalScore", () => {
+ if (lazy.Weave.Service.scheduler.globalScore != 0) {
+ lazy.Weave.Service.scheduler.globalScore = 0;
+ }
+ });
+ },
+
+ DumpError(msg, exc = null) {
+ this._errors++;
+ let errInfo;
+ if (exc) {
+ errInfo = lazy.Log.exceptionStr(exc); // includes details and stack-trace.
+ } else {
+ // always write a stack even if no error passed.
+ errInfo = lazy.Log.stackTrace(new Error());
+ }
+ lazy.Logger.logError(`[phase ${this._currentPhase}] ${msg} - ${errInfo}`);
+ this.quit();
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ observe: function TPS__observe(subject, topic, data) {
+ try {
+ lazy.Logger.logInfo("----------event observed: " + topic);
+
+ switch (topic) {
+ case "profile-before-change":
+ OBSERVER_TOPICS.forEach(function (topic) {
+ Services.obs.removeObserver(this, topic);
+ }, this);
+
+ lazy.Logger.close();
+
+ break;
+
+ case "places-browser-init-complete":
+ this._placesInitDeferred.resolve();
+ break;
+
+ case "weave:service:setup-complete":
+ this._setupComplete = true;
+
+ if (this._syncWipeAction) {
+ lazy.Weave.Svc.Prefs.set("firstSync", this._syncWipeAction);
+ this._syncWipeAction = null;
+ }
+
+ break;
+
+ case "weave:service:sync:error":
+ this._syncActive = false;
+
+ this.delayAutoSync();
+
+ // If this is the first sync error, retry...
+ if (this._syncErrors === 0) {
+ lazy.Logger.logInfo("Sync error; retrying...");
+ this._syncErrors++;
+ lazy.CommonUtils.nextTick(() => {
+ this.RunNextTestAction().catch(err => {
+ this.DumpError("RunNextTestActionFailed", err);
+ });
+ });
+ } else {
+ this._triggeredSync = false;
+ this.DumpError("Sync error; aborting test");
+ return;
+ }
+
+ break;
+
+ case "weave:service:resyncs-finished":
+ this._syncActive = false;
+ this._syncErrors = 0;
+ this._triggeredSync = false;
+
+ this.delayAutoSync();
+ break;
+
+ case "weave:service:sync:start":
+ // Ensure that the sync operation has been started by TPS
+ if (!this._triggeredSync) {
+ this.DumpError(
+ "Automatic sync got triggered, which is not allowed."
+ );
+ }
+
+ this._syncActive = true;
+ break;
+
+ case "weave:service:tracking-started":
+ this._isTracking = true;
+ break;
+
+ case "weave:service:tracking-stopped":
+ this._isTracking = false;
+ break;
+
+ case "fxaccounts:onlogin":
+ // A user signed in - for TPS that always means sync - so configure
+ // that.
+ lazy.Weave.Service.configure().catch(e => {
+ this.DumpError("Configuring sync failed.", e);
+ });
+ break;
+
+ default:
+ lazy.Logger.logInfo(`unhandled event: ${topic}`);
+ }
+ } catch (e) {
+ this.DumpError("Observer failed", e);
+ }
+ },
+
+ /**
+ * Given that we cannot completely disable the automatic sync operations, we
+ * massively delay the next sync. Sync operations have to only happen when
+ * directly called via TPS.Sync()!
+ */
+ delayAutoSync: function TPS_delayAutoSync() {
+ lazy.Weave.Svc.Prefs.set("scheduler.immediateInterval", 7200);
+ lazy.Weave.Svc.Prefs.set("scheduler.idleInterval", 7200);
+ lazy.Weave.Svc.Prefs.set("scheduler.activeInterval", 7200);
+ lazy.Weave.Svc.Prefs.set("syncThreshold", 10000000);
+ },
+
+ quit: function TPS__quit() {
+ lazy.Logger.logInfo("quitting");
+ this._requestedQuit = true;
+ this.goQuitApplication();
+ },
+
+ async HandleWindows(aWindow, action) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on window " +
+ JSON.stringify(aWindow)
+ );
+ switch (action) {
+ case ACTION_ADD:
+ await lazy.BrowserWindows.Add(aWindow.private);
+ break;
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on windows"
+ );
+ },
+
+ async HandleTabs(tabs, action) {
+ for (let tab of tabs) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on tab " +
+ JSON.stringify(tab)
+ );
+ switch (action) {
+ case ACTION_ADD:
+ await lazy.BrowserTabs.Add(tab.uri);
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(
+ typeof tab.profile != "undefined",
+ "profile must be defined when verifying tabs"
+ );
+ lazy.Logger.AssertTrue(
+ await lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile),
+ "error locating tab"
+ );
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ typeof tab.profile != "undefined",
+ "profile must be defined when verifying tabs"
+ );
+ lazy.Logger.AssertTrue(
+ await !lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile),
+ "tab found which was expected to be absent"
+ );
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on tabs"
+ );
+ },
+
+ async HandlePrefs(prefs, action) {
+ for (let pref of prefs) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on pref " +
+ JSON.stringify(pref)
+ );
+ let preference = new lazy.Preference(pref);
+ switch (action) {
+ case ACTION_MODIFY:
+ preference.Modify();
+ break;
+ case ACTION_VERIFY:
+ preference.Find();
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on pref"
+ );
+ },
+
+ async HandleForms(data, action) {
+ this.shouldValidateForms = true;
+ for (let datum of data) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on form entry " +
+ JSON.stringify(datum)
+ );
+ let formdata = new lazy.FormData(datum, this._msSinceEpoch);
+ switch (action) {
+ case ACTION_ADD:
+ await formdata.Create();
+ break;
+ case ACTION_DELETE:
+ await formdata.Remove();
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(await formdata.Find(), "form data not found");
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ !(await formdata.Find()),
+ "form data found, but it shouldn't be present"
+ );
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on formdata"
+ );
+ },
+
+ async HandleHistory(entries, action) {
+ try {
+ for (let entry of entries) {
+ const entryString = JSON.stringify(entry);
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on history entry " +
+ entryString
+ );
+ switch (action) {
+ case ACTION_ADD:
+ await lazy.HistoryEntry.Add(entry, this._msSinceEpoch);
+ break;
+ case ACTION_DELETE:
+ await lazy.HistoryEntry.Delete(entry, this._msSinceEpoch);
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(
+ await lazy.HistoryEntry.Find(entry, this._msSinceEpoch),
+ "Uri visits not found in history database: " + entryString
+ );
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ !(await lazy.HistoryEntry.Find(entry, this._msSinceEpoch)),
+ "Uri visits found in history database, but they shouldn't be: " +
+ entryString
+ );
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on history"
+ );
+ } catch (e) {
+ await lazy.DumpHistory();
+ throw e;
+ }
+ },
+
+ async HandlePasswords(passwords, action) {
+ this.shouldValidatePasswords = true;
+ try {
+ for (let password of passwords) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on password " +
+ JSON.stringify(password)
+ );
+ let passwordOb = new lazy.Password(password);
+ switch (action) {
+ case ACTION_ADD:
+ lazy.Logger.AssertTrue(
+ (await passwordOb.Create()) > -1,
+ "error adding password"
+ );
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(
+ passwordOb.Find() != -1,
+ "password not found"
+ );
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ passwordOb.Find() == -1,
+ "password found, but it shouldn't exist"
+ );
+ break;
+ case ACTION_DELETE:
+ lazy.Logger.AssertTrue(
+ passwordOb.Find() != -1,
+ "password not found"
+ );
+ passwordOb.Remove();
+ break;
+ case ACTION_MODIFY:
+ if (passwordOb.updateProps != null) {
+ lazy.Logger.AssertTrue(
+ passwordOb.Find() != -1,
+ "password not found"
+ );
+ passwordOb.Update();
+ }
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on passwords"
+ );
+ } catch (e) {
+ lazy.DumpPasswords();
+ throw e;
+ }
+ },
+
+ async HandleAddons(addons, action, state) {
+ this.shouldValidateAddons = true;
+ for (let entry of addons) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on addon " +
+ JSON.stringify(entry)
+ );
+ let addon = new lazy.Addon(this, entry);
+ switch (action) {
+ case ACTION_ADD:
+ await addon.install();
+ break;
+ case ACTION_DELETE:
+ await addon.uninstall();
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(
+ await addon.find(state),
+ "addon " + addon.id + " not found"
+ );
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertFalse(
+ await addon.find(state),
+ "addon " + addon.id + " is present, but it shouldn't be"
+ );
+ break;
+ case ACTION_SET_ENABLED:
+ lazy.Logger.AssertTrue(
+ await addon.setEnabled(state),
+ "addon " + addon.id + " not found"
+ );
+ break;
+ default:
+ throw new Error("Unknown action for add-on: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on addons"
+ );
+ },
+
+ async HandleBookmarks(bookmarks, action) {
+ // wait for default bookmarks to be created.
+ await this._placesInitDeferred.promise;
+ this.shouldValidateBookmarks = true;
+ try {
+ let items = [];
+ for (let folder in bookmarks) {
+ let last_item_pos = -1;
+ for (let bookmark of bookmarks[folder]) {
+ lazy.Logger.clearPotentialError();
+ let placesItem;
+ bookmark.location = folder;
+
+ if (last_item_pos != -1) {
+ bookmark.last_item_pos = last_item_pos;
+ }
+ let itemGuid = null;
+
+ if (action != ACTION_MODIFY && action != ACTION_DELETE) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on bookmark " +
+ JSON.stringify(bookmark)
+ );
+ }
+
+ if ("uri" in bookmark) {
+ placesItem = new lazy.Bookmark(bookmark);
+ } else if ("folder" in bookmark) {
+ placesItem = new lazy.BookmarkFolder(bookmark);
+ } else if ("livemark" in bookmark) {
+ placesItem = new lazy.Livemark(bookmark);
+ } else if ("separator" in bookmark) {
+ placesItem = new lazy.Separator(bookmark);
+ }
+
+ if (action == ACTION_ADD) {
+ itemGuid = await placesItem.Create();
+ } else {
+ itemGuid = await placesItem.Find();
+ if (action == ACTION_VERIFY_NOT) {
+ lazy.Logger.AssertTrue(
+ itemGuid == null,
+ "places item exists but it shouldn't: " +
+ JSON.stringify(bookmark)
+ );
+ } else {
+ lazy.Logger.AssertTrue(itemGuid, "places item not found", true);
+ }
+ }
+
+ last_item_pos = await placesItem.GetItemIndex();
+ items.push(placesItem);
+ }
+ }
+
+ if (action == ACTION_DELETE || action == ACTION_MODIFY) {
+ for (let item of items) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on bookmark " +
+ JSON.stringify(item)
+ );
+ switch (action) {
+ case ACTION_DELETE:
+ await item.Remove();
+ break;
+ case ACTION_MODIFY:
+ if (item.updateProps != null) {
+ await item.Update();
+ }
+ break;
+ }
+ }
+ }
+
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on bookmarks"
+ );
+ } catch (e) {
+ await lazy.DumpBookmarks();
+ throw e;
+ }
+ },
+
+ async HandleAddresses(addresses, action) {
+ try {
+ for (let address of addresses) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on address " +
+ JSON.stringify(address)
+ );
+ let addressOb = new lazy.Address(address);
+ switch (action) {
+ case ACTION_ADD:
+ await addressOb.Create();
+ break;
+ case ACTION_MODIFY:
+ await addressOb.Update();
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(await addressOb.Find(), "address not found");
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ !(await addressOb.Find()),
+ "address found, but it shouldn't exist"
+ );
+ break;
+ case ACTION_DELETE:
+ lazy.Logger.AssertTrue(await addressOb.Find(), "address not found");
+ await addressOb.Remove();
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on addresses"
+ );
+ } catch (e) {
+ await lazy.DumpAddresses();
+ throw e;
+ }
+ },
+
+ async HandleCreditCards(creditCards, action) {
+ try {
+ for (let creditCard of creditCards) {
+ lazy.Logger.logInfo(
+ "executing action " +
+ action.toUpperCase() +
+ " on creditCard " +
+ JSON.stringify(creditCard)
+ );
+ let creditCardOb = new lazy.CreditCard(creditCard);
+ switch (action) {
+ case ACTION_ADD:
+ await creditCardOb.Create();
+ break;
+ case ACTION_MODIFY:
+ await creditCardOb.Update();
+ break;
+ case ACTION_VERIFY:
+ lazy.Logger.AssertTrue(
+ await creditCardOb.Find(),
+ "creditCard not found"
+ );
+ break;
+ case ACTION_VERIFY_NOT:
+ lazy.Logger.AssertTrue(
+ !(await creditCardOb.Find()),
+ "creditCard found, but it shouldn't exist"
+ );
+ break;
+ case ACTION_DELETE:
+ lazy.Logger.AssertTrue(
+ await creditCardOb.Find(),
+ "creditCard not found"
+ );
+ await creditCardOb.Remove();
+ break;
+ default:
+ lazy.Logger.AssertTrue(false, "invalid action: " + action);
+ }
+ }
+ lazy.Logger.logPass(
+ "executing action " + action.toUpperCase() + " on creditCards"
+ );
+ } catch (e) {
+ await lazy.DumpCreditCards();
+ throw e;
+ }
+ },
+
+ async Cleanup() {
+ try {
+ await this.WipeServer();
+ } catch (ex) {
+ lazy.Logger.logError(
+ "Failed to wipe server: " + lazy.Log.exceptionStr(ex)
+ );
+ }
+ try {
+ if (await lazy.Authentication.isLoggedIn()) {
+ // signout and wait for Sync to completely reset itself.
+ lazy.Logger.logInfo("signing out");
+ let waiter = this.promiseObserver("weave:service:start-over:finish");
+ await lazy.Authentication.signOut();
+ await waiter;
+ lazy.Logger.logInfo("signout complete");
+ }
+ await lazy.Authentication.deleteEmail(this.config.fx_account.username);
+ } catch (e) {
+ lazy.Logger.logError("Failed to sign out: " + lazy.Log.exceptionStr(e));
+ }
+ },
+
+ /**
+ * Use Sync's bookmark validation code to see if we've corrupted the tree.
+ */
+ async ValidateBookmarks() {
+ let getServerBookmarkState = async () => {
+ let bookmarkEngine = lazy.Weave.Service.engineManager.get("bookmarks");
+ let collection = bookmarkEngine.itemSource();
+ let collectionKey =
+ bookmarkEngine.service.collectionKeys.keyForCollection(
+ bookmarkEngine.name
+ );
+ collection.full = true;
+ let items = [];
+ let resp = await collection.get();
+ for (let json of resp.obj) {
+ let record = new collection._recordObj();
+ record.deserialize(json);
+ await record.decrypt(collectionKey);
+ items.push(record.cleartext);
+ }
+ return items;
+ };
+ let serverRecordDumpStr;
+ try {
+ lazy.Logger.logInfo("About to perform bookmark validation");
+ let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", {
+ includeItemIds: true,
+ });
+ let serverRecords = await getServerBookmarkState();
+ // We can't wait until catch to stringify this, since at that point it will have cycles.
+ serverRecordDumpStr = JSON.stringify(serverRecords);
+
+ let validator = new lazy.BookmarkValidator();
+ let { problemData } = await validator.compareServerWithClient(
+ serverRecords,
+ clientTree
+ );
+
+ for (let { name, count } of problemData.getSummary()) {
+ // Exclude mobile showing up on the server hackily so that we don't
+ // report it every time, see bug 1273234 and 1274394 for more information.
+ if (
+ name === "serverUnexpected" &&
+ problemData.serverUnexpected.includes("mobile")
+ ) {
+ --count;
+ }
+ if (count) {
+ // Log this out before we assert. This is useful in the context of TPS logs, since we
+ // can see the IDs in the test files.
+ lazy.Logger.logInfo(
+ `Validation problem: "${name}": ${JSON.stringify(
+ problemData[name]
+ )}`
+ );
+ }
+ lazy.Logger.AssertEqual(
+ count,
+ 0,
+ `Bookmark validation error of type ${name}`
+ );
+ }
+ } catch (e) {
+ // Dump the client records (should always be doable)
+ lazy.DumpBookmarks();
+ // Dump the server records if gotten them already.
+ if (serverRecordDumpStr) {
+ lazy.Logger.logInfo(
+ "Server bookmark records:\n" + serverRecordDumpStr + "\n"
+ );
+ }
+ this.DumpError("Bookmark validation failed", e);
+ }
+ lazy.Logger.logInfo("Bookmark validation finished");
+ },
+
+ async ValidateCollection(engineName, ValidatorType) {
+ let serverRecordDumpStr;
+ let clientRecordDumpStr;
+ try {
+ lazy.Logger.logInfo(`About to perform validation for "${engineName}"`);
+ let engine = lazy.Weave.Service.engineManager.get(engineName);
+ let validator = new ValidatorType(engine);
+ let serverRecords = await validator.getServerItems(engine);
+ let clientRecords = await validator.getClientItems();
+ try {
+ // This substantially improves the logs for addons while not making a
+ // substantial difference for the other two
+ clientRecordDumpStr = JSON.stringify(
+ clientRecords.map(r => {
+ let res = validator.normalizeClientItem(r);
+ delete res.original; // Try and prevent cyclic references
+ return res;
+ })
+ );
+ } catch (e) {
+ // ignore the error, the dump string is just here to make debugging easier.
+ clientRecordDumpStr = "<Cyclic value>";
+ }
+ try {
+ serverRecordDumpStr = JSON.stringify(serverRecords);
+ } catch (e) {
+ // as above
+ serverRecordDumpStr = "<Cyclic value>";
+ }
+ let { problemData } = await validator.compareClientWithServer(
+ clientRecords,
+ serverRecords
+ );
+ for (let { name, count } of problemData.getSummary()) {
+ if (count) {
+ lazy.Logger.logInfo(
+ `Validation problem: "${name}": ${JSON.stringify(
+ problemData[name]
+ )}`
+ );
+ }
+ lazy.Logger.AssertEqual(
+ count,
+ 0,
+ `Validation error for "${engineName}" of type "${name}"`
+ );
+ }
+ } catch (e) {
+ // Dump the client records if possible
+ if (clientRecordDumpStr) {
+ lazy.Logger.logInfo(
+ `Client state for ${engineName}:\n${clientRecordDumpStr}\n`
+ );
+ }
+ // Dump the server records if gotten them already.
+ if (serverRecordDumpStr) {
+ lazy.Logger.logInfo(
+ `Server state for ${engineName}:\n${serverRecordDumpStr}\n`
+ );
+ }
+ this.DumpError(`Validation failed for ${engineName}`, e);
+ }
+ lazy.Logger.logInfo(`Validation finished for ${engineName}`);
+ },
+
+ ValidatePasswords() {
+ return this.ValidateCollection("passwords", lazy.PasswordValidator);
+ },
+
+ ValidateForms() {
+ return this.ValidateCollection("forms", lazy.FormValidator);
+ },
+
+ ValidateAddons() {
+ return this.ValidateCollection("addons", lazy.AddonValidator);
+ },
+
+ async RunNextTestAction() {
+ lazy.Logger.logInfo("Running next test action");
+ try {
+ if (this._currentAction >= this._phaselist[this._currentPhase].length) {
+ // Run necessary validations and then finish up
+ lazy.Logger.logInfo("No more actions - running validations...");
+ if (this.shouldValidateBookmarks) {
+ await this.ValidateBookmarks();
+ }
+ if (this.shouldValidatePasswords) {
+ await this.ValidatePasswords();
+ }
+ if (this.shouldValidateForms) {
+ await this.ValidateForms();
+ }
+ if (this.shouldValidateAddons) {
+ await this.ValidateAddons();
+ }
+ // Force this early so that we run the validation and detect missing pings
+ // *before* we start shutting down, since if we do it after, the python
+ // code won't notice the failure.
+ lazy.SyncTelemetry.shutdown();
+ // we're all done
+ lazy.Logger.logInfo(
+ "test phase " +
+ this._currentPhase +
+ ": " +
+ (this._errors ? "FAIL" : "PASS")
+ );
+ this._phaseFinished = true;
+ this.quit();
+ return;
+ }
+ this.seconds_since_epoch = Services.prefs.getIntPref(
+ "tps.seconds_since_epoch"
+ );
+ if (this.seconds_since_epoch) {
+ // Places dislikes it if we add visits in the future. We pretend the
+ // real time is 1 minute ago to avoid issues caused by places using a
+ // different clock than the one that set the seconds_since_epoch pref.
+ this._msSinceEpoch = (this.seconds_since_epoch - 60) * 1000;
+ } else {
+ this.DumpError("seconds-since-epoch not set");
+ return;
+ }
+
+ let phase = this._phaselist[this._currentPhase];
+ let action = phase[this._currentAction];
+ lazy.Logger.logInfo("starting action: " + action[0].name);
+ await action[0].apply(this, action.slice(1));
+
+ this._currentAction++;
+ } catch (e) {
+ if (lazy.Async.isShutdownException(e)) {
+ if (this._requestedQuit) {
+ lazy.Logger.logInfo("Sync aborted due to requested shutdown");
+ } else {
+ this.DumpError(
+ "Sync aborted due to shutdown, but we didn't request it"
+ );
+ }
+ } else {
+ this.DumpError("RunNextTestAction failed", e);
+ }
+ return;
+ }
+ await this.RunNextTestAction();
+ },
+
+ _getFileRelativeToSourceRoot(testFileURL, relativePath) {
+ let file = lazy.fileProtocolHandler.getFileFromURLSpec(testFileURL);
+ let root = file.parent.parent.parent.parent.parent; // <root>/services/sync/tests/tps/test_foo.js // <root>/services/sync/tests/tps // <root>/services/sync/tests // <root>/services/sync // <root>/services // <root>
+ root.appendRelativePath(relativePath);
+ root.normalize();
+ return root;
+ },
+
+ _pingValidator: null,
+
+ // Default ping validator that always says the ping passes. This should be
+ // overridden unless the `testing.tps.skipPingValidation` pref is true.
+ get pingValidator() {
+ return this._pingValidator
+ ? this._pingValidator
+ : {
+ validate() {
+ lazy.Logger.logInfo(
+ "Not validating ping -- disabled by pref or failure to load schema"
+ );
+ return { valid: true, errors: [] };
+ },
+ };
+ },
+
+ // Attempt to load the sync_ping_schema.json and initialize `this.pingValidator`
+ // based on the source of the tps file. Assumes that it's at "../unit/sync_ping_schema.json"
+ // relative to the directory the tps test file (testFile) is contained in.
+ _tryLoadPingSchema(testFile) {
+ if (Services.prefs.getBoolPref("testing.tps.skipPingValidation", false)) {
+ return;
+ }
+ try {
+ let schemaFile = this._getFileRelativeToSourceRoot(
+ testFile,
+ "services/sync/tests/unit/sync_ping_schema.json"
+ );
+
+ let stream = Cc[
+ "@mozilla.org/network/file-input-stream;1"
+ ].createInstance(Ci.nsIFileInputStream);
+
+ stream.init(
+ schemaFile,
+ lazy.FileUtils.MODE_RDONLY,
+ lazy.FileUtils.PERMS_FILE,
+ 0
+ );
+
+ let bytes = lazy.NetUtil.readInputStream(stream, stream.available());
+ let schema = JSON.parse(lazy.gTextDecoder.decode(bytes));
+ lazy.Logger.logInfo("Successfully loaded schema");
+
+ this._pingValidator = new lazy.JsonSchema.Validator(schema);
+ } catch (e) {
+ this.DumpError(
+ `Failed to load ping schema relative to "${testFile}".`,
+ e
+ );
+ }
+ },
+
+ /**
+ * Runs a single test phase.
+ *
+ * This is the main entry point for each phase of a test. The TPS command
+ * line driver loads this module and calls into the function with the
+ * arguments from the command line.
+ *
+ * When a phase is executed, the file is loaded as JavaScript into the
+ * current object.
+ *
+ * The following keys in the options argument have meaning:
+ *
+ * - ignoreUnusedEngines If true, unused engines will be unloaded from
+ * Sync. This makes output easier to parse and is
+ * useful for debugging test failures.
+ *
+ * @param file
+ * String URI of the file to open.
+ * @param phase
+ * String name of the phase to run.
+ * @param logpath
+ * String path of the log file to write to.
+ * @param options
+ * Object defining addition run-time options.
+ */
+ async RunTestPhase(file, phase, logpath, options) {
+ try {
+ let settings = options || {};
+
+ lazy.Logger.init(logpath);
+ lazy.Logger.logInfo("Sync version: " + lazy.WEAVE_VERSION);
+ lazy.Logger.logInfo("Firefox buildid: " + Services.appinfo.appBuildID);
+ lazy.Logger.logInfo("Firefox version: " + Services.appinfo.version);
+ lazy.Logger.logInfo(
+ "Firefox source revision: " +
+ (AppConstants.SOURCE_REVISION_URL || "unknown")
+ );
+ lazy.Logger.logInfo("Firefox platform: " + AppConstants.platform);
+
+ // do some sync housekeeping
+ if (lazy.Weave.Service.isLoggedIn) {
+ this.DumpError("Sync logged in on startup...profile may be dirty");
+ return;
+ }
+
+ // Wait for Sync service to become ready.
+ if (!lazy.Weave.Status.ready) {
+ this.waitForEvent("weave:service:ready");
+ }
+
+ await lazy.Weave.Service.promiseInitialized;
+
+ // We only want to do this if we modified the bookmarks this phase.
+ this.shouldValidateBookmarks = false;
+
+ // Always give Sync an extra tick to initialize. If we waited for the
+ // service:ready event, this is required to ensure all handlers have
+ // executed.
+ await lazy.Async.promiseYield();
+ await this._executeTestPhase(file, phase, settings);
+ } catch (e) {
+ this.DumpError("RunTestPhase failed", e);
+ }
+ },
+
+ /**
+ * Executes a single test phase.
+ *
+ * This is called by RunTestPhase() after the environment is validated.
+ */
+ async _executeTestPhase(file, phase, settings) {
+ try {
+ this.config = JSON.parse(Services.prefs.getCharPref("tps.config"));
+ // parse the test file
+ Services.scriptloader.loadSubScript(file, this);
+ this._currentPhase = phase;
+ // cleanup phases are in the format `cleanup-${profileName}`.
+ if (this._currentPhase.startsWith("cleanup-")) {
+ let profileToClean = this._currentPhase.slice("cleanup-".length);
+ this.phases[this._currentPhase] = profileToClean;
+ this.Phase(this._currentPhase, [[this.Cleanup]]);
+ } else {
+ // Don't bother doing this for cleanup phases.
+ this._tryLoadPingSchema(file);
+ }
+ let this_phase = this._phaselist[this._currentPhase];
+
+ if (this_phase == undefined) {
+ this.DumpError("invalid phase " + this._currentPhase);
+ return;
+ }
+
+ if (this.phases[this._currentPhase] == undefined) {
+ this.DumpError("no profile defined for phase " + this._currentPhase);
+ return;
+ }
+
+ // If we have restricted the active engines, unregister engines we don't
+ // care about.
+ if (settings.ignoreUnusedEngines && Array.isArray(this._enabledEngines)) {
+ let names = {};
+ for (let name of this._enabledEngines) {
+ names[name] = true;
+ }
+ for (let engine of lazy.Weave.Service.engineManager.getEnabled()) {
+ if (!(engine.name in names)) {
+ lazy.Logger.logInfo("Unregistering unused engine: " + engine.name);
+ await lazy.Weave.Service.engineManager.unregister(engine);
+ }
+ }
+ }
+ lazy.Logger.logInfo("Starting phase " + this._currentPhase);
+
+ lazy.Logger.logInfo(
+ "setting client.name to " + this.phases[this._currentPhase]
+ );
+ lazy.Weave.Svc.Prefs.set("client.name", this.phases[this._currentPhase]);
+
+ this._interceptSyncTelemetry();
+
+ // start processing the test actions
+ this._currentAction = 0;
+ await lazy.SessionStore.promiseAllWindowsRestored;
+ await this.RunNextTestAction();
+ } catch (e) {
+ this.DumpError("_executeTestPhase failed", e);
+ }
+ },
+
+ /**
+ * Override sync telemetry functions so that we can detect errors generating
+ * the sync ping, and count how many pings we report.
+ */
+ _interceptSyncTelemetry() {
+ let originalObserve = lazy.SyncTelemetry.observe;
+ let self = this;
+ lazy.SyncTelemetry.observe = function () {
+ try {
+ originalObserve.apply(this, arguments);
+ } catch (e) {
+ self.DumpError("Error when generating sync telemetry", e);
+ }
+ };
+ lazy.SyncTelemetry.submit = record => {
+ lazy.Logger.logInfo(
+ "Intercepted sync telemetry submission: " + JSON.stringify(record)
+ );
+ this._syncsReportedViaTelemetry +=
+ record.syncs.length + (record.discarded || 0);
+ if (record.discarded) {
+ if (record.syncs.length != lazy.SyncTelemetry.maxPayloadCount) {
+ this.DumpError(
+ "Syncs discarded from ping before maximum payload count reached"
+ );
+ }
+ }
+ // If this is the shutdown ping, check and see that the telemetry saw all the syncs.
+ if (record.why === "shutdown") {
+ // If we happen to sync outside of tps manually causing it, its not an
+ // error in the telemetry, so we only complain if we didn't see all of them.
+ if (this._syncsReportedViaTelemetry < this._syncCount) {
+ this.DumpError(
+ `Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.`
+ );
+ }
+ }
+ if (!record.syncs.length) {
+ // Note: we're overwriting submit, so this is called even for pings that
+ // may have no data (which wouldn't be submitted to telemetry and would
+ // fail validation).
+ return;
+ }
+ // Our ping may have some undefined values, which we rely on JSON stripping
+ // out as part of the ping submission - but our validator fails with them,
+ // so round-trip via JSON here to avoid that.
+ record = JSON.parse(JSON.stringify(record));
+ const result = this.pingValidator.validate(record);
+ if (!result.valid) {
+ // Note that we already logged the record.
+ this.DumpError(
+ "Sync ping validation failed with errors: " +
+ JSON.stringify(result.errors)
+ );
+ }
+ };
+ },
+
+ /**
+ * Register a single phase with the test harness.
+ *
+ * This is called when loading individual test files.
+ *
+ * @param phasename
+ * String name of the phase being loaded.
+ * @param fnlist
+ * Array of functions/actions to perform.
+ */
+ Phase: function Test__Phase(phasename, fnlist) {
+ if (Object.keys(this._phaselist).length === 0) {
+ // This is the first phase we should force a log in
+ fnlist.unshift([this.Login]);
+ }
+ this._phaselist[phasename] = fnlist;
+ },
+
+ /**
+ * Restrict enabled Sync engines to a specified set.
+ *
+ * This can be called by a test to limit what engines are enabled. It is
+ * recommended to call it to reduce the overhead and log clutter for the
+ * test.
+ *
+ * The "clients" engine is special and is always enabled, so there is no
+ * need to specify it.
+ *
+ * @param names
+ * Array of Strings for engines to make active during the test.
+ */
+ EnableEngines: function EnableEngines(names) {
+ if (!Array.isArray(names)) {
+ throw new Error(
+ "Argument to RestrictEngines() is not an array: " + typeof names
+ );
+ }
+
+ this._enabledEngines = names;
+ },
+
+ /**
+ * Returns a promise that resolves when a specific observer notification is
+ * resolved. This is similar to the various waitFor* functions, although is
+ * typically safer if you need to do some other work that may make the event
+ * fire.
+ *
+ * eg:
+ * doSomething(); // causes the event to be fired.
+ * await promiseObserver("something");
+ * is risky as the call to doSomething may trigger the event before the
+ * promiseObserver call is made. Contrast with:
+ *
+ * let waiter = promiseObserver("something");
+ * doSomething(); // causes the event to be fired.
+ * await waiter; // will return as soon as the event fires, even if it fires
+ * // before this function is called.
+ *
+ * @param aEventName
+ * String event to wait for.
+ */
+ promiseObserver(aEventName) {
+ return new Promise(resolve => {
+ lazy.Logger.logInfo("Setting up wait for " + aEventName + "...");
+ let handler = () => {
+ lazy.Logger.logInfo("Observed " + aEventName);
+ lazy.Svc.Obs.remove(aEventName, handler);
+ resolve();
+ };
+ lazy.Svc.Obs.add(aEventName, handler);
+ });
+ },
+
+ /**
+ * Wait for the named event to be observed.
+ *
+ * Note that in general, you should probably use promiseObserver unless you
+ * are 100% sure that the event being waited on can only be sent after this
+ * call adds the listener.
+ *
+ * @param aEventName
+ * String event to wait for.
+ */
+ async waitForEvent(aEventName) {
+ await this.promiseObserver(aEventName);
+ },
+
+ /**
+ * Waits for Sync to logged in before returning
+ */
+ async waitForSetupComplete() {
+ if (!this._setupComplete) {
+ await this.waitForEvent("weave:service:setup-complete");
+ }
+ },
+
+ /**
+ * Waits for Sync to be finished before returning
+ */
+ async waitForSyncFinished() {
+ if (lazy.Weave.Service.locked) {
+ await this.waitForEvent("weave:service:resyncs-finished");
+ }
+ },
+
+ /**
+ * Waits for Sync to start tracking before returning.
+ */
+ async waitForTracking() {
+ if (!this._isTracking) {
+ await this.waitForEvent("weave:service:tracking-started");
+ }
+ },
+
+ /**
+ * Login on the server
+ */
+ async Login() {
+ if (await lazy.Authentication.isReady()) {
+ return;
+ }
+
+ lazy.Logger.logInfo("Setting client credentials and login.");
+ await lazy.Authentication.signIn(this.config.fx_account);
+ await this.waitForSetupComplete();
+ lazy.Logger.AssertEqual(
+ lazy.Weave.Status.service,
+ lazy.STATUS_OK,
+ "Weave status OK"
+ );
+ await this.waitForTracking();
+ },
+
+ /**
+ * Triggers a sync operation
+ *
+ * @param {String} [wipeAction]
+ * Type of wipe to perform (resetClient, wipeClient, wipeRemote)
+ *
+ */
+ async Sync(wipeAction) {
+ if (this._syncActive) {
+ this.DumpError("Sync currently active which should be impossible");
+ return;
+ }
+ lazy.Logger.logInfo(
+ "Executing Sync" + (wipeAction ? ": " + wipeAction : "")
+ );
+
+ // Force a wipe action if requested. In case of an initial sync the pref
+ // will be overwritten by Sync itself (see bug 992198), so ensure that we
+ // also handle it via the "weave:service:setup-complete" notification.
+ if (wipeAction) {
+ this._syncWipeAction = wipeAction;
+ lazy.Weave.Svc.Prefs.set("firstSync", wipeAction);
+ } else {
+ lazy.Weave.Svc.Prefs.reset("firstSync");
+ }
+ if (!(await lazy.Weave.Service.login())) {
+ // We need to complete verification.
+ lazy.Logger.logInfo("Logging in before performing sync");
+ await this.Login();
+ }
+ ++this._syncCount;
+
+ lazy.Logger.logInfo(
+ "Executing Sync" + (wipeAction ? ": " + wipeAction : "")
+ );
+ this._triggeredSync = true;
+ await lazy.Weave.Service.sync();
+ lazy.Logger.logInfo("Sync is complete");
+ // wait a second for things to settle...
+ await new Promise(resolve => {
+ lazy.CommonUtils.namedTimer(resolve, 1000, this, "postsync");
+ });
+ },
+
+ async WipeServer() {
+ lazy.Logger.logInfo("Wiping data from server.");
+
+ await this.Login();
+ await lazy.Weave.Service.login();
+ await lazy.Weave.Service.wipeServer();
+ },
+
+ /**
+ * Action which ensures changes are being tracked before returning.
+ */
+ async EnsureTracking() {
+ await this.Login();
+ await this.waitForTracking();
+ },
+};
+
+var Addons = {
+ async install(addons) {
+ await TPS.HandleAddons(addons, ACTION_ADD);
+ },
+ async setEnabled(addons, state) {
+ await TPS.HandleAddons(addons, ACTION_SET_ENABLED, state);
+ },
+ async uninstall(addons) {
+ await TPS.HandleAddons(addons, ACTION_DELETE);
+ },
+ async verify(addons, state) {
+ await TPS.HandleAddons(addons, ACTION_VERIFY, state);
+ },
+ async verifyNot(addons) {
+ await TPS.HandleAddons(addons, ACTION_VERIFY_NOT);
+ },
+ skipValidation() {
+ TPS.shouldValidateAddons = false;
+ },
+};
+
+var Addresses = {
+ async add(addresses) {
+ await this.HandleAddresses(addresses, ACTION_ADD);
+ },
+ async modify(addresses) {
+ await this.HandleAddresses(addresses, ACTION_MODIFY);
+ },
+ async delete(addresses) {
+ await this.HandleAddresses(addresses, ACTION_DELETE);
+ },
+ async verify(addresses) {
+ await this.HandleAddresses(addresses, ACTION_VERIFY);
+ },
+ async verifyNot(addresses) {
+ await this.HandleAddresses(addresses, ACTION_VERIFY_NOT);
+ },
+};
+
+var Bookmarks = {
+ async add(bookmarks) {
+ await TPS.HandleBookmarks(bookmarks, ACTION_ADD);
+ },
+ async modify(bookmarks) {
+ await TPS.HandleBookmarks(bookmarks, ACTION_MODIFY);
+ },
+ async delete(bookmarks) {
+ await TPS.HandleBookmarks(bookmarks, ACTION_DELETE);
+ },
+ async verify(bookmarks) {
+ await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY);
+ },
+ async verifyNot(bookmarks) {
+ await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY_NOT);
+ },
+ skipValidation() {
+ TPS.shouldValidateBookmarks = false;
+ },
+};
+
+var CreditCards = {
+ async add(creditCards) {
+ await this.HandleCreditCards(creditCards, ACTION_ADD);
+ },
+ async modify(creditCards) {
+ await this.HandleCreditCards(creditCards, ACTION_MODIFY);
+ },
+ async delete(creditCards) {
+ await this.HandleCreditCards(creditCards, ACTION_DELETE);
+ },
+ async verify(creditCards) {
+ await this.HandleCreditCards(creditCards, ACTION_VERIFY);
+ },
+ async verifyNot(creditCards) {
+ await this.HandleCreditCards(creditCards, ACTION_VERIFY_NOT);
+ },
+};
+
+var Formdata = {
+ async add(formdata) {
+ await this.HandleForms(formdata, ACTION_ADD);
+ },
+ async delete(formdata) {
+ await this.HandleForms(formdata, ACTION_DELETE);
+ },
+ async verify(formdata) {
+ await this.HandleForms(formdata, ACTION_VERIFY);
+ },
+ async verifyNot(formdata) {
+ await this.HandleForms(formdata, ACTION_VERIFY_NOT);
+ },
+};
+
+var History = {
+ async add(history) {
+ await this.HandleHistory(history, ACTION_ADD);
+ },
+ async delete(history) {
+ await this.HandleHistory(history, ACTION_DELETE);
+ },
+ async verify(history) {
+ await this.HandleHistory(history, ACTION_VERIFY);
+ },
+ async verifyNot(history) {
+ await this.HandleHistory(history, ACTION_VERIFY_NOT);
+ },
+};
+
+var Passwords = {
+ async add(passwords) {
+ await this.HandlePasswords(passwords, ACTION_ADD);
+ },
+ async modify(passwords) {
+ await this.HandlePasswords(passwords, ACTION_MODIFY);
+ },
+ async delete(passwords) {
+ await this.HandlePasswords(passwords, ACTION_DELETE);
+ },
+ async verify(passwords) {
+ await this.HandlePasswords(passwords, ACTION_VERIFY);
+ },
+ async verifyNot(passwords) {
+ await this.HandlePasswords(passwords, ACTION_VERIFY_NOT);
+ },
+ skipValidation() {
+ TPS.shouldValidatePasswords = false;
+ },
+};
+
+var Prefs = {
+ async modify(prefs) {
+ await TPS.HandlePrefs(prefs, ACTION_MODIFY);
+ },
+ async verify(prefs) {
+ await TPS.HandlePrefs(prefs, ACTION_VERIFY);
+ },
+};
+
+var Tabs = {
+ async add(tabs) {
+ await TPS.HandleTabs(tabs, ACTION_ADD);
+ },
+ async verify(tabs) {
+ await TPS.HandleTabs(tabs, ACTION_VERIFY);
+ },
+ async verifyNot(tabs) {
+ await TPS.HandleTabs(tabs, ACTION_VERIFY_NOT);
+ },
+};
+
+var Windows = {
+ async add(aWindow) {
+ await TPS.HandleWindows(aWindow, ACTION_ADD);
+ },
+};
+
+// Jumping through loads of hoops via calling back into a "HandleXXX" method
+// and adding an ACTION_XXX indirection adds no value - let's KISS!
+// eslint-disable-next-line no-unused-vars
+var ExtStorage = {
+ async set(id, data) {
+ lazy.Logger.logInfo(`setting data for '${id}': ${data}`);
+ await lazy.extensionStorageSync.set({ id }, data);
+ },
+ async verify(id, keys, data) {
+ let got = await lazy.extensionStorageSync.get({ id }, keys);
+ lazy.Logger.AssertEqual(got, data, `data for '${id}'/${keys}`);
+ },
+};
+
+// Initialize TPS
+TPS._init();
diff --git a/services/sync/tps/extensions/tps/schema.json b/services/sync/tps/extensions/tps/schema.json
new file mode 100644
index 0000000000..fe51488c70
--- /dev/null
+++ b/services/sync/tps/extensions/tps/schema.json
@@ -0,0 +1 @@
+[]