diff options
Diffstat (limited to '')
17 files changed, 5047 insertions, 0 deletions
diff --git a/services/sync/tps/extensions/tps/api.js b/services/sync/tps/extensions/tps/api.js new file mode 100644 index 0000000000..686abdc6fd --- /dev/null +++ b/services/sync/tps/extensions/tps/api.js @@ -0,0 +1,77 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +/* globals ExtensionAPI, Services, XPCOMUtils */ + +XPCOMUtils.defineLazyServiceGetter( + this, + "resProto", + "@mozilla.org/network/protocol;1?name=resource", + "nsISubstitutingProtocolHandler" +); + +async function tpsStartup() { + try { + var { TPS } = ChromeUtils.import("resource://tps/tps.jsm"); + let { goQuitApplication } = ChromeUtils.importESModule( + "resource://tps/quit.sys.mjs" + ); + TPS.goQuitApplication = goQuitApplication; + + let testFile = Services.prefs.getStringPref("testing.tps.testFile", ""); + let testPhase = Services.prefs.getStringPref("testing.tps.testPhase", ""); + if (!testFile || !testPhase) { + // Note: this quits. + TPS.DumpError( + "TPS no longer takes arguments from the command line. " + + "instead you need to pass preferences `testing.tps.{testFile,testPhase}` " + + "and optionally `testing.tps.{logFile,ignoreUnusedEngines}`.\n" + ); + } + + let logFile = Services.prefs.getStringPref("testing.tps.logFile", ""); + let ignoreUnusedEngines = Services.prefs.getBoolPref( + "testing.tps.ignoreUnusedEngines", + false + ); + let options = { ignoreUnusedEngines }; + let testFileUri = Services.io.newFileURI(new FileUtils.File(testFile)).spec; + + try { + await TPS.RunTestPhase(testFileUri, testPhase, logFile, options); + } catch (err) { + TPS.DumpError("TestPhase failed", err); + } + } catch (e) { + if (typeof TPS != "undefined") { + // Note: This calls quit() under the hood + TPS.DumpError("Test initialization failed", e); + } + dump(`TPS test initialization failed: ${e} - ${e.stack}\n`); + // Try and quit right away, no reason to wait around for python + // to kill us if initialization failed. + Services.startup.quit(Ci.nsIAppStartup.eForceQuit); + } +} + +this.tps = class extends ExtensionAPI { + onStartup() { + resProto.setSubstitution( + "tps", + Services.io.newURI("resource/", null, this.extension.rootURI) + ); + /* Ignore the platform's online/offline status while running tests. */ + Services.io.manageOfflineStatus = false; + Services.io.offline = false; + tpsStartup(); + } + + onShutdown() { + resProto.setSubstitution("tps", null); + } +}; diff --git a/services/sync/tps/extensions/tps/manifest.json b/services/sync/tps/extensions/tps/manifest.json new file mode 100644 index 0000000000..c961e76506 --- /dev/null +++ b/services/sync/tps/extensions/tps/manifest.json @@ -0,0 +1,23 @@ +{ + "manifest_version": 2, + "name": "TPS", + "version": "1.0", + + "browser_specific_settings": { + "gecko": { + "id": "tps@mozilla.org" + } + }, + + "experiment_apis": { + "tps": { + "schema": "schema.json", + "parent": { + "scopes": ["addon_parent"], + "script": "api.js", + "paths": [["tps"]], + "events": ["startup"] + } + } + } +} diff --git a/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs new file mode 100644 index 0000000000..81c0fd578a --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/auth/fxaccounts.sys.mjs @@ -0,0 +1,209 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Log } from "resource://gre/modules/Log.sys.mjs"; +import { clearTimeout, setTimeout } from "resource://gre/modules/Timer.sys.mjs"; + +import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs"; + +const fxAccounts = getFxAccountsSingleton(); +import { FxAccountsClient } from "resource://gre/modules/FxAccountsClient.sys.mjs"; +import { FxAccountsConfig } from "resource://gre/modules/FxAccountsConfig.sys.mjs"; +import { Logger } from "resource://tps/logger.sys.mjs"; + +/** + * Helper object for Firefox Accounts authentication + */ +export var Authentication = { + /** + * Check if an user has been logged in + */ + async isLoggedIn() { + return !!(await this.getSignedInUser()); + }, + + async isReady() { + let user = await this.getSignedInUser(); + return user && user.verified; + }, + + _getRestmailUsername(user) { + const restmailSuffix = "@restmail.net"; + if (user.toLowerCase().endsWith(restmailSuffix)) { + return user.slice(0, -restmailSuffix.length); + } + return null; + }, + + async shortWaitForVerification(ms) { + let userData = await this.getSignedInUser(); + let timeoutID; + let timeoutPromise = new Promise(resolve => { + timeoutID = setTimeout(() => { + Logger.logInfo(`Warning: no verification after ${ms}ms.`); + resolve(); + }, ms); + }); + await Promise.race([ + fxAccounts.whenVerified(userData).finally(() => clearTimeout(timeoutID)), + timeoutPromise, + ]); + userData = await this.getSignedInUser(); + return userData && userData.verified; + }, + + async _openVerificationPage(uri) { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let newtab = mainWindow.gBrowser.addWebTab(uri); + let win = mainWindow.gBrowser.getBrowserForTab(newtab); + await new Promise(resolve => { + win.addEventListener("loadend", resolve, { once: true }); + }); + let didVerify = await this.shortWaitForVerification(10000); + mainWindow.gBrowser.removeTab(newtab); + return didVerify; + }, + + async _completeVerification(user) { + let username = this._getRestmailUsername(user); + if (!username) { + Logger.logInfo( + `Username "${user}" isn't a restmail username so can't complete verification` + ); + return false; + } + Logger.logInfo("Fetching mail (from restmail) for user " + username); + let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent( + username + )}`; + let triedAlready = new Set(); + const tries = 10; + const normalWait = 2000; + for (let i = 0; i < tries; ++i) { + let resp = await fetch(restmailURI); + let messages = await resp.json(); + // Sort so that the most recent emails are first. + messages.sort((a, b) => new Date(b.receivedAt) - new Date(a.receivedAt)); + for (let m of messages) { + // We look for a link that has a x-link that we haven't yet tried. + if (!m.headers["x-link"] || triedAlready.has(m.headers["x-link"])) { + continue; + } + let confirmLink = m.headers["x-link"]; + triedAlready.add(confirmLink); + Logger.logInfo("Trying confirmation link " + confirmLink); + try { + if (await this._openVerificationPage(confirmLink)) { + return true; + } + } catch (e) { + Logger.logInfo( + "Warning: Failed to follow confirmation link: " + + Log.exceptionStr(e) + ); + } + } + if (i === 0) { + // first time through after failing we'll do this. + await fxAccounts.resendVerificationEmail(); + } + if (await this.shortWaitForVerification(normalWait)) { + return true; + } + } + // One last try. + return this.shortWaitForVerification(normalWait); + }, + + async deleteEmail(user) { + let username = this._getRestmailUsername(user); + if (!username) { + Logger.logInfo("Not a restmail username, can't delete"); + return false; + } + Logger.logInfo("Deleting mail (from restmail) for user " + username); + let restmailURI = `https://www.restmail.net/mail/${encodeURIComponent( + username + )}`; + try { + // Clean up after ourselves. + let deleteResult = await fetch(restmailURI, { method: "DELETE" }); + if (!deleteResult.ok) { + Logger.logInfo( + `Warning: Got non-success status ${deleteResult.status} when deleting emails` + ); + return false; + } + } catch (e) { + Logger.logInfo( + "Warning: Failed to delete old emails: " + Log.exceptionStr(e) + ); + return false; + } + return true; + }, + + /** + * Wrapper to retrieve the currently signed in user + * + * @returns Information about the currently signed in user + */ + async getSignedInUser() { + try { + return await fxAccounts.getSignedInUser(); + } catch (error) { + Logger.logError( + "getSignedInUser() failed with: " + JSON.stringify(error) + ); + throw error; + } + }, + + /** + * Wrapper to synchronize the login of a user + * + * @param account + * Account information of the user to login + * @param account.username + * The username for the account (utf8) + * @param account.password + * The user's password + */ + async signIn(account) { + Logger.AssertTrue(account.username, "Username has been found"); + Logger.AssertTrue(account.password, "Password has been found"); + + Logger.logInfo("Login user: " + account.username); + + try { + // Required here since we don't go through the real login page + await FxAccountsConfig.ensureConfigured(); + + let client = new FxAccountsClient(); + let credentials = await client.signIn( + account.username, + account.password, + true + ); + await fxAccounts._internal.setSignedInUser(credentials); + if (!credentials.verified) { + await this._completeVerification(account.username); + } + + return true; + } catch (error) { + throw new Error("signIn() failed with: " + error.message); + } + }, + + /** + * Sign out of Firefox Accounts. + */ + async signOut() { + if (await Authentication.isLoggedIn()) { + // Note: This will clean up the device ID. + await fxAccounts.signOut(); + } + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/logger.sys.mjs b/services/sync/tps/extensions/tps/resource/logger.sys.mjs new file mode 100644 index 0000000000..5d1a0d9ce5 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/logger.sys.mjs @@ -0,0 +1,170 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +XPCOMUtils.defineLazyModuleGetters(lazy, { + ObjectUtils: "resource://gre/modules/ObjectUtils.jsm", +}); + +export var Logger = { + _foStream: null, + _converter: null, + _potentialError: null, + + init(path) { + if (this._converter != null) { + // we're already open! + return; + } + + if (path) { + Services.prefs.setCharPref("tps.logfile", path); + } else { + path = Services.prefs.getCharPref("tps.logfile"); + } + + this._file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + this._file.initWithPath(path); + var exists = this._file.exists(); + + // Make a file output stream and converter to handle it. + this._foStream = Cc[ + "@mozilla.org/network/file-output-stream;1" + ].createInstance(Ci.nsIFileOutputStream); + // If the file already exists, append it, otherwise create it. + var fileflags = exists ? 0x02 | 0x08 | 0x10 : 0x02 | 0x08 | 0x20; + + this._foStream.init(this._file, fileflags, 0o666, 0); + this._converter = Cc[ + "@mozilla.org/intl/converter-output-stream;1" + ].createInstance(Ci.nsIConverterOutputStream); + this._converter.init(this._foStream, "UTF-8"); + }, + + write(data) { + if (this._converter == null) { + console.error("TPS Logger.write called with _converter == null!"); + return; + } + this._converter.writeString(data); + }, + + close() { + if (this._converter != null) { + this._converter.close(); + this._converter = null; + this._foStream = null; + } + }, + + AssertTrue(bool, msg, showPotentialError) { + if (bool) { + return; + } + + if (showPotentialError && this._potentialError) { + msg += "; " + this._potentialError; + this._potentialError = null; + } + throw new Error("ASSERTION FAILED! " + msg); + }, + + AssertFalse(bool, msg, showPotentialError) { + return this.AssertTrue(!bool, msg, showPotentialError); + }, + + AssertEqual(got, expected, msg) { + if (!lazy.ObjectUtils.deepEqual(got, expected)) { + throw new Error( + "ASSERTION FAILED! " + + msg + + "; expected " + + JSON.stringify(expected) + + ", got " + + JSON.stringify(got) + ); + } + }, + + log(msg, withoutPrefix) { + dump(msg + "\n"); + if (withoutPrefix) { + this.write(msg + "\n"); + } else { + function pad(n, len) { + let s = "0000" + n; + return s.slice(-len); + } + + let now = new Date(); + let year = pad(now.getFullYear(), 4); + let month = pad(now.getMonth() + 1, 2); + let day = pad(now.getDate(), 2); + let hour = pad(now.getHours(), 2); + let minutes = pad(now.getMinutes(), 2); + let seconds = pad(now.getSeconds(), 2); + let ms = pad(now.getMilliseconds(), 3); + + this.write( + year + + "-" + + month + + "-" + + day + + " " + + hour + + ":" + + minutes + + ":" + + seconds + + "." + + ms + + " " + + msg + + "\n" + ); + } + }, + + clearPotentialError() { + this._potentialError = null; + }, + + logPotentialError(msg) { + this._potentialError = msg; + }, + + logLastPotentialError(msg) { + var message = msg; + if (this._potentialError) { + message = this._poentialError; + this._potentialError = null; + } + this.log("CROSSWEAVE ERROR: " + message); + }, + + logError(msg) { + this.log("CROSSWEAVE ERROR: " + msg); + }, + + logInfo(msg, withoutPrefix) { + if (withoutPrefix) { + this.log(msg, true); + } else { + this.log("CROSSWEAVE INFO: " + msg); + } + }, + + logPass(msg) { + this.log("CROSSWEAVE TEST PASS: " + msg); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs new file mode 100644 index 0000000000..596f942a06 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/addons.sys.mjs @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AddonManager } from "resource://gre/modules/AddonManager.sys.mjs"; +import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs"; +import { Logger } from "resource://tps/logger.sys.mjs"; + +export const STATE_ENABLED = 1; +export const STATE_DISABLED = 2; + +export function Addon(TPS, id) { + this.TPS = TPS; + this.id = id; +} + +Addon.prototype = { + addon: null, + + async uninstall() { + // find our addon locally + let addon = await AddonManager.getAddonByID(this.id); + Logger.AssertTrue( + !!addon, + "could not find addon " + this.id + " to uninstall" + ); + await AddonUtils.uninstallAddon(addon); + }, + + async find(state) { + let addon = await AddonManager.getAddonByID(this.id); + + if (!addon) { + Logger.logInfo("Could not find add-on with ID: " + this.id); + return false; + } + + this.addon = addon; + + Logger.logInfo( + "add-on found: " + addon.id + ", enabled: " + !addon.userDisabled + ); + if (state == STATE_ENABLED) { + Logger.AssertFalse(addon.userDisabled, "add-on is disabled: " + addon.id); + return true; + } else if (state == STATE_DISABLED) { + Logger.AssertTrue(addon.userDisabled, "add-on is enabled: " + addon.id); + return true; + } else if (state) { + throw new Error("Don't know how to handle state: " + state); + } else { + // No state, so just checking that it exists. + return true; + } + }, + + async install() { + // For Install, the id parameter initially passed is really the filename + // for the addon's install .xml; we'll read the actual id from the .xml. + + const result = await AddonUtils.installAddons([ + { id: this.id, requireSecureURI: false }, + ]); + + Logger.AssertEqual( + 1, + result.installedIDs.length, + "Exactly 1 add-on was installed." + ); + Logger.AssertEqual( + this.id, + result.installedIDs[0], + "Add-on was installed successfully: " + this.id + ); + }, + + async setEnabled(flag) { + Logger.AssertTrue(await this.find(), "Add-on is available."); + + let userDisabled; + if (flag == STATE_ENABLED) { + userDisabled = false; + } else if (flag == STATE_DISABLED) { + userDisabled = true; + } else { + throw new Error("Unknown flag to setEnabled: " + flag); + } + + AddonUtils.updateUserDisabled(this.addon, userDisabled); + + return true; + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs new file mode 100644 index 0000000000..e075b55149 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.sys.mjs @@ -0,0 +1,1065 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file was moved to tps from the main production code as it was unused +// after removal of the non-mirror bookmarks engine. +// It used to have a test before it was moved: +// https://searchfox.org/mozilla-central/rev/b1a5802e0f73bfd6d2096e5fefc2b47831a50b2d/services/sync/tests/unit/test_bookmark_validator.js + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +const QUERY_PROTOCOL = "place:"; + +function areURLsEqual(a, b) { + if (a === b) { + return true; + } + if (a.startsWith(QUERY_PROTOCOL) != b.startsWith(QUERY_PROTOCOL)) { + return false; + } + // Tag queries are special because we rewrite them to point to the + // local tag folder ID. It's expected that the folders won't match, + // but all other params should. + let aParams = new URLSearchParams(a.slice(QUERY_PROTOCOL.length)); + let aType = +aParams.get("type"); + if (aType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + return false; + } + let bParams = new URLSearchParams(b.slice(QUERY_PROTOCOL.length)); + let bType = +bParams.get("type"); + if (bType != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + return false; + } + let aKeys = new Set(aParams.keys()); + let bKeys = new Set(bParams.keys()); + if (aKeys.size != bKeys.size) { + return false; + } + // Tag queries shouldn't reference multiple folders, or named folders like + // "TOOLBAR" or "BOOKMARKS_MENU". Just in case, we make sure all folder IDs + // are numeric. If they are, we ignore them when comparing the query params. + if (aKeys.has("folder") && aParams.getAll("folder").every(isFinite)) { + aKeys.delete("folder"); + } + if (bKeys.has("folder") && bParams.getAll("folder").every(isFinite)) { + bKeys.delete("folder"); + } + for (let key of aKeys) { + if (!bKeys.has(key)) { + return false; + } + if ( + !CommonUtils.arrayEqual( + aParams.getAll(key).sort(), + bParams.getAll(key).sort() + ) + ) { + return false; + } + } + for (let key of bKeys) { + if (!aKeys.has(key)) { + return false; + } + } + return true; +} + +const BOOKMARK_VALIDATOR_VERSION = 1; + +/** + * Result of bookmark validation. Contains the following fields which describe + * server-side problems unless otherwise specified. + * + * - missingIDs (number): # of objects with missing ids + * - duplicates (array of ids): ids seen more than once + * - parentChildMismatches (array of {parent: parentid, child: childid}): + * instances where the child's parentid and the parent's children array + * do not match + * - cycles (array of array of ids). List of cycles found in the server-side tree. + * - clientCycles (array of array of ids). List of cycles found in the client-side tree. + * - orphans (array of {id: string, parent: string}): List of nodes with + * either no parentid, or where the parent could not be found. + * - missingChildren (array of {parent: id, child: id}): + * List of parent/children where the child id couldn't be found + * - deletedChildren (array of { parent: id, child: id }): + * List of parent/children where child id was a deleted item (but still showed up + * in the children array) + * - multipleParents (array of {child: id, parents: array of ids}): + * List of children that were part of multiple parent arrays + * - deletedParents (array of ids) : List of records that aren't deleted but + * had deleted parents + * - childrenOnNonFolder (array of ids): list of non-folders that still have + * children arrays + * - duplicateChildren (array of ids): list of records who have the same + * child listed multiple times in their children array + * - parentNotFolder (array of ids): list of records that have parents that + * aren't folders + * - rootOnServer (boolean): true if the root came from the server + * - badClientRoots (array of ids): Contains any client-side root ids where + * the root is missing or isn't a (direct) child of the places root. + * + * - clientMissing: Array of ids on the server missing from the client + * - serverMissing: Array of ids on the client missing from the server + * - serverDeleted: Array of ids on the client that the server had marked as deleted. + * - serverUnexpected: Array of ids that appear on the server but shouldn't + * because the client attempts to never upload them. + * - differences: Array of {id: string, differences: string array} recording + * the non-structural properties that are differente between the client and server + * - structuralDifferences: As above, but contains the items where the differences were + * structural, that is, they contained childGUIDs or parentid + */ +export class BookmarkProblemData { + constructor() { + this.rootOnServer = false; + this.missingIDs = 0; + + this.duplicates = []; + this.parentChildMismatches = []; + this.cycles = []; + this.clientCycles = []; + this.orphans = []; + this.missingChildren = []; + this.deletedChildren = []; + this.multipleParents = []; + this.deletedParents = []; + this.childrenOnNonFolder = []; + this.duplicateChildren = []; + this.parentNotFolder = []; + + this.badClientRoots = []; + this.clientMissing = []; + this.serverMissing = []; + this.serverDeleted = []; + this.serverUnexpected = []; + this.differences = []; + this.structuralDifferences = []; + } + + /** + * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into + * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc. + */ + _summarizeDifferences(prefix, diffs) { + let diffCounts = new Map(); + for (let { differences } of diffs) { + for (let type of differences) { + let name = prefix + ":" + type; + let count = diffCounts.get(name) || 0; + diffCounts.set(name, count + 1); + } + } + return [...diffCounts].map(([name, count]) => ({ name, count })); + } + + /** + * Produce a list summarizing problems found. Each entry contains {name, count}, + * where name is the field name for the problem, and count is the number of times + * the problem was encountered. + * + * Validation has failed if all counts are not 0. + * + * If the `full` argument is truthy, we also include information about which + * properties we saw structural differences in. Currently, this means either + * "sdiff:parentid" and "sdiff:childGUIDS" may be present. + */ + getSummary(full) { + let result = [ + { name: "clientMissing", count: this.clientMissing.length }, + { name: "serverMissing", count: this.serverMissing.length }, + { name: "serverDeleted", count: this.serverDeleted.length }, + { name: "serverUnexpected", count: this.serverUnexpected.length }, + + { + name: "structuralDifferences", + count: this.structuralDifferences.length, + }, + { name: "differences", count: this.differences.length }, + + { name: "missingIDs", count: this.missingIDs }, + { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 }, + + { name: "duplicates", count: this.duplicates.length }, + { + name: "parentChildMismatches", + count: this.parentChildMismatches.length, + }, + { name: "cycles", count: this.cycles.length }, + { name: "clientCycles", count: this.clientCycles.length }, + { name: "badClientRoots", count: this.badClientRoots.length }, + { name: "orphans", count: this.orphans.length }, + { name: "missingChildren", count: this.missingChildren.length }, + { name: "deletedChildren", count: this.deletedChildren.length }, + { name: "multipleParents", count: this.multipleParents.length }, + { name: "deletedParents", count: this.deletedParents.length }, + { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length }, + { name: "duplicateChildren", count: this.duplicateChildren.length }, + { name: "parentNotFolder", count: this.parentNotFolder.length }, + ]; + if (full) { + let structural = this._summarizeDifferences( + "sdiff", + this.structuralDifferences + ); + result.push.apply(result, structural); + } + return result; + } +} + +// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon. +XPCOMUtils.defineLazyGetter(lazy, "SYNCED_ROOTS", () => [ + lazy.PlacesUtils.bookmarks.menuGuid, + lazy.PlacesUtils.bookmarks.toolbarGuid, + lazy.PlacesUtils.bookmarks.unfiledGuid, + lazy.PlacesUtils.bookmarks.mobileGuid, +]); + +// Maps root GUIDs to their query folder names from +// toolkit/components/places/nsNavHistoryQuery.cpp. We follow queries that +// reference existing folders in the client tree, and detect cycles where a +// query references its containing folder. +XPCOMUtils.defineLazyGetter(lazy, "ROOT_GUID_TO_QUERY_FOLDER_NAME", () => ({ + [lazy.PlacesUtils.bookmarks.rootGuid]: "PLACES_ROOT", + [lazy.PlacesUtils.bookmarks.menuGuid]: "BOOKMARKS_MENU", + + // Tags should never show up in our client tree, and never form cycles, but we + // report them just in case. + [lazy.PlacesUtils.bookmarks.tagsGuid]: "TAGS", + + [lazy.PlacesUtils.bookmarks.unfiledGuid]: "UNFILED_BOOKMARKS", + [lazy.PlacesUtils.bookmarks.toolbarGuid]: "TOOLBAR", + [lazy.PlacesUtils.bookmarks.mobileGuid]: "MOBILE_BOOKMARKS", +})); + +async function detectCycles(records) { + // currentPath and pathLookup contain the same data. pathLookup is faster to + // query, but currentPath gives is the order of traversal that we need in + // order to report the members of the cycles. + let pathLookup = new Set(); + let currentPath = []; + let cycles = []; + let seenEver = new Set(); + const yieldState = lazy.Async.yieldState(); + + const traverse = async node => { + if (pathLookup.has(node)) { + let cycleStart = currentPath.lastIndexOf(node); + let cyclePath = currentPath.slice(cycleStart).map(n => n.id); + cycles.push(cyclePath); + return; + } else if (seenEver.has(node)) { + // If we're checking the server, this is a problem, but it should already be reported. + // On the client, this could happen due to including `node.concrete` in the child list. + return; + } + seenEver.add(node); + let children = node.children || []; + if (node.concreteItems) { + children.push(...node.concreteItems); + } + if (children.length) { + pathLookup.add(node); + currentPath.push(node); + await lazy.Async.yieldingForEach(children, traverse, yieldState); + currentPath.pop(); + pathLookup.delete(node); + } + }; + + await lazy.Async.yieldingForEach( + records, + async record => { + if (!seenEver.has(record)) { + await traverse(record); + } + }, + yieldState + ); + + return cycles; +} + +class ServerRecordInspection { + constructor() { + this.serverRecords = null; + this.liveRecords = []; + + this.folders = []; + + this.root = null; + + this.idToRecord = new Map(); + + this.deletedIds = new Set(); + this.deletedRecords = []; + + this.problemData = new BookmarkProblemData(); + + // These are handled outside of problemData + this._orphans = new Map(); + this._multipleParents = new Map(); + + this.yieldState = lazy.Async.yieldState(); + } + + static async create(records) { + return new ServerRecordInspection().performInspection(records); + } + + async performInspection(records) { + await this._setRecords(records); + await this._linkParentIDs(); + await this._linkChildren(); + await this._findOrphans(); + await this._finish(); + return this; + } + + // We don't set orphans in this.problemData. Instead, we walk the tree at the + // end to find unreachable items. + _noteOrphan(id, parentId = undefined) { + // This probably shouldn't be called with a parentId twice, but if it + // happens we take the most recent one. + if (parentId || !this._orphans.has(id)) { + this._orphans.set(id, parentId); + } + } + + noteParent(child, parent) { + let parents = this._multipleParents.get(child); + if (!parents) { + this._multipleParents.set(child, [parent]); + } else { + parents.push(parent); + } + } + + noteMismatch(child, parent) { + let exists = this.problemData.parentChildMismatches.some( + match => match.child == child && match.parent == parent + ); + if (!exists) { + this.problemData.parentChildMismatches.push({ child, parent }); + } + } + + // - Populates `this.deletedIds`, `this.folders`, and `this.idToRecord` + // - calls `_initRoot` (thus initializing `this.root`). + async _setRecords(records) { + if (this.serverRecords) { + // In general this class is expected to be created, have + // `performInspection` called, and then only read from from that point on. + throw new Error("Bug: ServerRecordInspection can't `setRecords` twice"); + } + this.serverRecords = records; + let rootChildren = []; + + await lazy.Async.yieldingForEach( + this.serverRecords, + async record => { + if (!record.id) { + ++this.problemData.missingIDs; + return; + } + + if (record.deleted) { + this.deletedIds.add(record.id); + } + if (this.idToRecord.has(record.id)) { + this.problemData.duplicates.push(record.id); + return; + } + + this.idToRecord.set(record.id, record); + + if (!record.deleted) { + this.liveRecords.push(record); + + if (record.parentid == "places") { + rootChildren.push(record); + } + } + + if (!record.children) { + return; + } + + if (record.type != "folder") { + // Due to implementation details in engines/bookmarks.js, (Livemark + // subclassing BookmarkFolder) Livemarks will have a children array, + // but it should still be empty. + if (!record.children.length) { + return; + } + // Otherwise we mark it as an error and still try to resolve the children + this.problemData.childrenOnNonFolder.push(record.id); + } + + this.folders.push(record); + + if (new Set(record.children).size !== record.children.length) { + this.problemData.duplicateChildren.push(record.id); + } + + // After we're through with them, folder records store 3 (ugh) arrays that + // represent their folder information. The final fields looks like: + // + // - childGUIDs: The original `children` array, which is an array of + // record IDs. + // + // - unfilteredChildren: Contains more or less `childGUIDs.map(id => + // idToRecord.get(id))`, without the nulls for missing children. It will + // still have deleted, duplicate, mismatching, etc. children. + // + // - children: This is the 'cleaned' version of the child records that are + // safe to iterate over, etc.. If there are no reported problems, it should + // be identical to unfilteredChildren. + // + // The last two are left alone until later `this._linkChildren`, however. + record.childGUIDs = record.children; + + await lazy.Async.yieldingForEach( + record.childGUIDs, + id => { + this.noteParent(id, record.id); + }, + this.yieldState + ); + + record.children = []; + }, + this.yieldState + ); + + // Finish up some parts we can easily do now that we have idToRecord. + this.deletedRecords = Array.from(this.deletedIds, id => + this.idToRecord.get(id) + ); + + this._initRoot(rootChildren); + } + + _initRoot(rootChildren) { + let serverRoot = this.idToRecord.get("places"); + if (serverRoot) { + this.root = serverRoot; + this.problemData.rootOnServer = true; + return; + } + + // Fabricate a root. We want to be able to remember that it's fake, but + // would like to avoid it needing too much special casing, so we come up + // with children for it too (we just get these while we're iterating over + // the records to avoid needing two passes over a potentially large number + // of records). + + this.root = { + id: "places", + fake: true, + children: rootChildren, + childGUIDs: rootChildren.map(record => record.id), + type: "folder", + title: "", + }; + this.liveRecords.push(this.root); + this.idToRecord.set("places", this.root); + } + + // Adds `parent` to all records it can that have `parentid` + async _linkParentIDs() { + await lazy.Async.yieldingForEach( + this.idToRecord, + ([id, record]) => { + if (record == this.root || record.deleted) { + return false; + } + + // Check and update our orphan map. + let parentID = record.parentid; + let parent = this.idToRecord.get(parentID); + if (!parentID || !parent) { + this._noteOrphan(id, parentID); + return false; + } + + record.parent = parent; + + if (parent.deleted) { + this.problemData.deletedParents.push(id); + return true; + } else if (parent.type != "folder") { + this.problemData.parentNotFolder.push(record.id); + return true; + } + + if (parent.id !== "place" || this.problemData.rootOnServer) { + if (!parent.childGUIDs.includes(record.id)) { + this.noteMismatch(record.id, parent.id); + } + } + + if (parent.deleted && !record.deleted) { + this.problemData.deletedParents.push(record.id); + } + + // Note: We used to check if the parentName on the server matches the + // actual local parent name, but given this is used only for de-duping a + // record the first time it is seen and expensive to keep up-to-date, we + // decided to just stop recording it. See bug 1276969 for more. + return false; + }, + this.yieldState + ); + } + + // Build the children and unfilteredChildren arrays, (which are of record + // objects, not ids) + async _linkChildren() { + // Check that we aren't missing any children. + await lazy.Async.yieldingForEach( + this.folders, + async folder => { + folder.children = []; + folder.unfilteredChildren = []; + + let idsThisFolder = new Set(); + + await lazy.Async.yieldingForEach( + folder.childGUIDs, + childID => { + let child = this.idToRecord.get(childID); + + if (!child) { + this.problemData.missingChildren.push({ + parent: folder.id, + child: childID, + }); + return; + } + + if (child.deleted) { + this.problemData.deletedChildren.push({ + parent: folder.id, + child: childID, + }); + return; + } + + if (child.parentid != folder.id) { + this.noteMismatch(childID, folder.id); + return; + } + + if (idsThisFolder.has(childID)) { + // Already recorded earlier, we just don't want to mess up `children` + return; + } + folder.children.push(child); + }, + this.yieldState + ); + }, + this.yieldState + ); + } + + // Finds the orphans in the tree using something similar to a `mark and sweep` + // strategy. That is, we iterate over the children from the root, remembering + // which items we've seen. Then, we iterate all items, and know the ones we + // haven't seen are orphans. + async _findOrphans() { + let seen = new Set([this.root.id]); + + const inCycle = await lazy.Async.yieldingForEach( + Utils.walkTree(this.root), + ([node]) => { + if (seen.has(node.id)) { + // We're in an infloop due to a cycle. + // Return early to avoid reporting false positives for orphans. + return true; + } + seen.add(node.id); + + return false; + }, + this.yieldState + ); + + if (inCycle) { + return; + } + + await lazy.Async.yieldingForEach( + this.liveRecords, + (record, i) => { + if (!seen.has(record.id)) { + // We intentionally don't record the parentid here, since we only record + // that if the record refers to a parent that doesn't exist, which we + // have already handled (when linking parentid's). + this._noteOrphan(record.id); + } + }, + this.yieldState + ); + + await lazy.Async.yieldingForEach( + this._orphans, + ([id, parent]) => { + this.problemData.orphans.push({ id, parent }); + }, + this.yieldState + ); + } + + async _finish() { + this.problemData.cycles = await detectCycles(this.liveRecords); + + for (const [child, recordedParents] of this._multipleParents) { + let parents = new Set(recordedParents); + if (parents.size > 1) { + this.problemData.multipleParents.push({ child, parents: [...parents] }); + } + } + // Dedupe simple arrays in the problem data, so that we don't have to worry + // about it in the code + const idArrayProps = [ + "duplicates", + "deletedParents", + "childrenOnNonFolder", + "duplicateChildren", + "parentNotFolder", + ]; + for (let prop of idArrayProps) { + this.problemData[prop] = [...new Set(this.problemData[prop])]; + } + } +} + +export class BookmarkValidator { + constructor() { + this.yieldState = lazy.Async.yieldState(); + } + + async canValidate() { + return !(await lazy.PlacesSyncUtils.bookmarks.havePendingChanges()); + } + + async _followQueries(recordsByQueryId) { + await lazy.Async.yieldingForEach( + recordsByQueryId.values(), + entry => { + if ( + entry.type !== "query" && + (!entry.bmkUri || !entry.bmkUri.startsWith(QUERY_PROTOCOL)) + ) { + return; + } + let params = new URLSearchParams( + entry.bmkUri.slice(QUERY_PROTOCOL.length) + ); + // Queries with `excludeQueries` won't form cycles because they'll + // exclude all queries, including themselves, from the result set. + let excludeQueries = params.get("excludeQueries"); + if (excludeQueries === "1" || excludeQueries === "true") { + // `nsNavHistoryQuery::ParseQueryBooleanString` allows `1` and `true`. + return; + } + entry.concreteItems = []; + let queryIds = params.getAll("folder"); + for (let queryId of queryIds) { + let concreteItem = recordsByQueryId.get(queryId); + if (concreteItem) { + entry.concreteItems.push(concreteItem); + } + } + }, + this.yieldState + ); + } + + async createClientRecordsFromTree(clientTree) { + // Iterate over the treeNode, converting it to something more similar to what + // the server stores. + let records = []; + // A map of local IDs and well-known query folder names to records. Unlike + // GUIDs, local IDs aren't synced, since they're not stable across devices. + // New Places APIs use GUIDs to refer to bookmarks, but the legacy APIs + // still use local IDs. We use this mapping to parse `place:` queries that + // refer to folders via their local IDs. + let recordsByQueryId = new Map(); + let syncedRoots = lazy.SYNCED_ROOTS; + + const traverse = async (treeNode, synced) => { + if (!synced) { + synced = syncedRoots.includes(treeNode.guid); + } + let localId = treeNode.id; + let guid = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(treeNode.guid); + let itemType = "item"; + treeNode.ignored = !synced; + treeNode.id = guid; + switch (treeNode.type) { + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE: + if (treeNode.uri.startsWith(QUERY_PROTOCOL)) { + itemType = "query"; + } else { + itemType = "bookmark"; + } + break; + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: + let isLivemark = false; + if (treeNode.annos) { + for (let anno of treeNode.annos) { + if (anno.name === lazy.PlacesUtils.LMANNO_FEEDURI) { + isLivemark = true; + treeNode.feedUri = anno.value; + } else if (anno.name === lazy.PlacesUtils.LMANNO_SITEURI) { + isLivemark = true; + treeNode.siteUri = anno.value; + } + } + } + itemType = isLivemark ? "livemark" : "folder"; + break; + case lazy.PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: + itemType = "separator"; + break; + } + + if (treeNode.tags) { + treeNode.tags = treeNode.tags.split(","); + } else { + treeNode.tags = []; + } + treeNode.type = itemType; + treeNode.pos = treeNode.index; + treeNode.bmkUri = treeNode.uri; + records.push(treeNode); + if (treeNode.guid in lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME) { + let queryId = lazy.ROOT_GUID_TO_QUERY_FOLDER_NAME[treeNode.guid]; + recordsByQueryId.set(queryId, treeNode); + } + if (localId) { + // Always add the ID, since it's still possible for a query to + // reference a root without using the well-known name. For example, + // `place:folder=${PlacesUtils.mobileFolderId}` and + // `place:folder=MOBILE_BOOKMARKS` are equivalent. + recordsByQueryId.set(localId.toString(10), treeNode); + } + if (treeNode.type === "folder") { + treeNode.childGUIDs = []; + if (!treeNode.children) { + treeNode.children = []; + } + + await lazy.Async.yieldingForEach( + treeNode.children, + async child => { + await traverse(child, synced); + child.parent = treeNode; + child.parentid = guid; + treeNode.childGUIDs.push(child.guid); + }, + this.yieldState + ); + } + }; + + await traverse(clientTree, false); + + clientTree.id = "places"; + await this._followQueries(recordsByQueryId); + return records; + } + + /** + * Process the server-side list. Mainly this builds the records into a tree, + * but it also records information about problems, and produces arrays of the + * deleted and non-deleted nodes. + * + * Returns an object containing: + * - records:Array of non-deleted records. Each record contains the following + * properties + * - childGUIDs (array of strings, only present if type is 'folder'): the + * list of child GUIDs stored on the server. + * - children (array of records, only present if type is 'folder'): + * each record has these same properties. This may differ in content + * from what you may expect from the childGUIDs list, as it won't + * contain any records that could not be found. + * - parent (record): The parent to this record. + * - Unchanged properties send down from the server: id, title, type, + * parentName, parentid, bmkURI, keyword, tags, pos, queryId + * - root: Root of the server-side bookmark tree. Has the same properties as + * above. + * - deletedRecords: As above, but only contains items that the server sent + * where it also sent indication that the item should be deleted. + * - problemData: a BookmarkProblemData object, with the caveat that + * the fields describing client/server relationship will not have been filled + * out yet. + */ + async inspectServerRecords(serverRecords) { + const data = await ServerRecordInspection.create(serverRecords); + return { + deletedRecords: data.deletedRecords, + records: data.liveRecords, + problemData: data.problemData, + root: data.root, + }; + } + + // Perform client-side sanity checking that doesn't involve server data + async _validateClient(problemData, clientRecords) { + problemData.clientCycles = await detectCycles(clientRecords); + for (let rootGUID of lazy.SYNCED_ROOTS) { + let record = clientRecords.find(record => record.guid === rootGUID); + if (!record || record.parentid !== "places") { + problemData.badClientRoots.push(rootGUID); + } + } + } + + async _computeUnifiedRecordMap(serverRecords, clientRecords) { + let allRecords = new Map(); + await lazy.Async.yieldingForEach( + serverRecords, + sr => { + if (sr.fake) { + return; + } + allRecords.set(sr.id, { client: null, server: sr }); + }, + this.yieldState + ); + + await lazy.Async.yieldingForEach( + clientRecords, + cr => { + let unified = allRecords.get(cr.id); + if (!unified) { + allRecords.set(cr.id, { client: cr, server: null }); + } else { + unified.client = cr; + } + }, + this.yieldState + ); + + return allRecords; + } + + _recordMissing(problems, id, clientRecord, serverRecord, serverTombstones) { + if (!clientRecord && serverRecord) { + problems.clientMissing.push(id); + } + if (!serverRecord && clientRecord) { + if (serverTombstones.has(id)) { + problems.serverDeleted.push(id); + } else if (!clientRecord.ignored && clientRecord.id != "places") { + problems.serverMissing.push(id); + } + } + } + + _compareRecords(client, server) { + let structuralDifferences = []; + let differences = []; + + // Don't bother comparing titles of roots. It's okay if locally it's + // "Mobile Bookmarks", but the server thinks it's "mobile". + // TODO: We probably should be handing other localized bookmarks (e.g. + // default bookmarks) here as well, see bug 1316041. + if (!lazy.SYNCED_ROOTS.includes(client.guid)) { + // We want to treat undefined, null and an empty string as identical + if ((client.title || "") !== (server.title || "")) { + differences.push("title"); + } + } + + if (client.parentid || server.parentid) { + if (client.parentid !== server.parentid) { + structuralDifferences.push("parentid"); + } + } + + if (client.tags || server.tags) { + let cl = client.tags ? [...client.tags].sort() : []; + let sl = server.tags ? [...server.tags].sort() : []; + if (!CommonUtils.arrayEqual(cl, sl)) { + differences.push("tags"); + } + } + + let sameType = client.type === server.type; + if (!sameType) { + if ( + server.type === "query" && + client.type === "bookmark" && + client.bmkUri.startsWith(QUERY_PROTOCOL) + ) { + sameType = true; + } + } + + if (!sameType) { + differences.push("type"); + } else { + switch (server.type) { + case "bookmark": + case "query": + if (!areURLsEqual(server.bmkUri, client.bmkUri)) { + differences.push("bmkUri"); + } + break; + case "separator": + if (server.pos != client.pos) { + differences.push("pos"); + } + break; + case "livemark": + if (server.feedUri != client.feedUri) { + differences.push("feedUri"); + } + if (server.siteUri != client.siteUri) { + differences.push("siteUri"); + } + break; + case "folder": + if (server.id === "places" && server.fake) { + // It's the fabricated places root. It won't have the GUIDs, but + // it doesn't matter. + break; + } + if (client.childGUIDs || server.childGUIDs) { + let cl = client.childGUIDs || []; + let sl = server.childGUIDs || []; + if (!CommonUtils.arrayEqual(cl, sl)) { + structuralDifferences.push("childGUIDs"); + } + } + break; + } + } + return { differences, structuralDifferences }; + } + + /** + * Compare the list of server records with the client tree. + * + * Returns the same data as described in the inspectServerRecords comment, + * with the following additional fields. + * - clientRecords: an array of client records in a similar format to + * the .records (ie, server records) entry. + * - problemData is the same as for inspectServerRecords, except all properties + * will be filled out. + */ + async compareServerWithClient(serverRecords, clientTree) { + let clientRecords = await this.createClientRecordsFromTree(clientTree); + let inspectionInfo = await this.inspectServerRecords(serverRecords); + inspectionInfo.clientRecords = clientRecords; + + // Mainly do this to remove deleted items and normalize child guids. + serverRecords = inspectionInfo.records; + let problemData = inspectionInfo.problemData; + + await this._validateClient(problemData, clientRecords); + + let allRecords = await this._computeUnifiedRecordMap( + serverRecords, + clientRecords + ); + + let serverDeleted = new Set(inspectionInfo.deletedRecords.map(r => r.id)); + + await lazy.Async.yieldingForEach( + allRecords, + ([id, { client, server }]) => { + if (!client || !server) { + this._recordMissing(problemData, id, client, server, serverDeleted); + return; + } + if (server && client && client.ignored) { + problemData.serverUnexpected.push(id); + } + let { differences, structuralDifferences } = this._compareRecords( + client, + server + ); + + if (differences.length) { + problemData.differences.push({ id, differences }); + } + if (structuralDifferences.length) { + problemData.structuralDifferences.push({ + id, + differences: structuralDifferences, + }); + } + }, + this.yieldState + ); + + return inspectionInfo; + } + + async _getServerState(engine) { + let collection = engine.itemSource(); + let collectionKey = engine.service.collectionKeys.keyForCollection( + engine.name + ); + collection.full = true; + let result = await collection.getBatched(); + if (!result.response.success) { + throw result.response; + } + let cleartexts = []; + await lazy.Async.yieldingForEach( + result.records, + async record => { + await record.decrypt(collectionKey); + cleartexts.push(record.cleartext); + }, + this.yieldState + ); + return cleartexts; + } + + async validate(engine) { + let start = Date.now(); + let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", { + includeItemIds: true, + }); + let serverState = await this._getServerState(engine); + let serverRecordCount = serverState.length; + let result = await this.compareServerWithClient(serverState, clientTree); + let end = Date.now(); + let duration = end - start; + + engine._log.debug(`Validated bookmarks in ${duration}ms`); + engine._log.debug(`Problem summary`); + for (let { name, count } of result.problemData.getSummary()) { + engine._log.debug(` ${name}: ${count}`); + } + + return { + duration, + version: this.version, + problems: result.problemData, + recordCount: serverRecordCount, + }; + } +} + +BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION; diff --git a/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs new file mode 100644 index 0000000000..e4aac948b5 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/bookmarks.sys.mjs @@ -0,0 +1,833 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { PlacesBackups } from "resource://gre/modules/PlacesBackups.sys.mjs"; + +import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs"; +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +export async function DumpBookmarks() { + let [bookmarks] = await PlacesBackups.getBookmarksTree(); + Logger.logInfo( + "Dumping Bookmarks...\n" + JSON.stringify(bookmarks, undefined, 2) + "\n\n" + ); +} + +/** + * extend, causes a child object to inherit from a parent + */ +function extend(child, supertype) { + Object.setPrototypeOf(child.prototype, supertype.prototype); +} +/** + * PlacesItemProps object, holds properties for places items + */ +function PlacesItemProps(props) { + this.location = null; + this.uri = null; + this.keyword = null; + this.title = null; + this.after = null; + this.before = null; + this.folder = null; + this.position = null; + this.delete = false; + this.tags = null; + this.last_item_pos = null; + this.type = null; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * PlacesItem object. Base class for places items. + */ +export function PlacesItem(props) { + this.props = new PlacesItemProps(props); + if (this.props.location == null) { + this.props.location = "menu"; + } + if ("changes" in props) { + this.updateProps = new PlacesItemProps(props.changes); + } else { + this.updateProps = null; + } +} + +/** + * Instance methods for generic places items. + */ +PlacesItem.prototype = { + // an array of possible root folders for places items + _bookmarkFolders: { + places: PlacesUtils.bookmarks.rootGuid, + menu: PlacesUtils.bookmarks.menuGuid, + tags: PlacesUtils.bookmarks.tagsGuid, + unfiled: PlacesUtils.bookmarks.unfiledGuid, + toolbar: PlacesUtils.bookmarks.toolbarGuid, + mobile: PlacesUtils.bookmarks.mobileGuid, + }, + + _typeMap: new Map([ + [PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER, PlacesUtils.bookmarks.TYPE_FOLDER], + [ + PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR, + PlacesUtils.bookmarks.TYPE_SEPARATOR, + ], + [PlacesUtils.TYPE_X_MOZ_PLACE, PlacesUtils.bookmarks.TYPE_BOOKMARK], + ]), + + toString() { + var that = this; + var props = ["uri", "title", "location", "folder"]; + var string = + (this.props.type ? this.props.type + " " : "") + + "(" + + (function () { + var ret = []; + for (var i in props) { + if (that.props[props[i]]) { + ret.push(props[i] + ": " + that.props[props[i]]); + } + } + return ret; + })().join(", ") + + ")"; + return string; + }, + + /** + * GetPlacesChildGuid + * + * Finds the guid of the an item with the specified properties in the places + * database under the specified parent. + * + * @param folder The guid of the folder to search + * @param type The type of the item to find, or null to match any item; + * this is one of the PlacesUtils.bookmarks.TYPE_* values + * @param title The title of the item to find, or null to match any title + * @param uri The uri of the item to find, or null to match any uri + * + * @return the node id if the item was found, otherwise null + */ + async GetPlacesChildGuid(folder, type, title, uri) { + let children = (await PlacesUtils.promiseBookmarksTree(folder)).children; + if (!children) { + return null; + } + let guid = null; + for (let node of children) { + if (node.title == title) { + let nodeType = this._typeMap.get(node.type); + if (type == null || type == undefined || nodeType == type) { + if (uri == undefined || uri == null || node.uri.spec == uri.spec) { + // Note that this is suspect as we return the *last* matching + // child, which some tests rely on (ie, an early-return here causes + // at least 1 test to fail). But that's a yak for another day. + guid = node.guid; + } + } + } + } + return guid; + }, + + /** + * IsAdjacentTo + * + * Determines if this object is immediately adjacent to another. + * + * @param itemName The name of the other object; this may be any kind of + * places item + * @param relativePos The relative position of the other object. If -1, + * it means the other object should precede this one, if +1, + * the other object should come after this one + * @return true if this object is immediately adjacent to the other object, + * otherwise false + */ + async IsAdjacentTo(itemName, relativePos) { + Logger.AssertTrue( + this.props.folder_id != -1 && this.props.guid != null, + "Either folder_id or guid was invalid" + ); + let otherGuid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + itemName + ); + Logger.AssertTrue(otherGuid, "item " + itemName + " not found"); + let other_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index; + let this_pos = (await PlacesUtils.bookmarks.fetch(this.props.guid)).index; + if (other_pos + relativePos != this_pos) { + Logger.logPotentialError( + "Invalid position - " + + (this.props.title ? this.props.title : this.props.folder) + + " not " + + (relativePos == 1 ? "after " : "before ") + + itemName + + " for " + + this.toString() + ); + return false; + } + return true; + }, + + /** + * GetItemIndex + * + * Gets the item index for this places item. + * + * @return the item index, or -1 if there's an error + */ + async GetItemIndex() { + if (this.props.guid == null) { + return -1; + } + return (await PlacesUtils.bookmarks.fetch(this.props.guid)).index; + }, + + /** + * GetFolder + * + * Gets the folder guid for the specified bookmark folder + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder guid if the folder is found, otherwise null + */ + async GetFolder(location) { + let folder_parts = location.split("/"); + if (!(folder_parts[0] in this._bookmarkFolders)) { + return null; + } + let folderGuid = this._bookmarkFolders[folder_parts[0]]; + for (let i = 1; i < folder_parts.length; i++) { + let guid = await this.GetPlacesChildGuid( + folderGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + folder_parts[i] + ); + if (guid == null) { + return null; + } + folderGuid = guid; + } + return folderGuid; + }, + + /** + * CreateFolder + * + * Creates a bookmark folder. + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder id if the folder was created, otherwise -1 + */ + async CreateFolder(location) { + let folder_parts = location.split("/"); + if (!(folder_parts[0] in this._bookmarkFolders)) { + return -1; + } + let folderGuid = this._bookmarkFolders[folder_parts[0]]; + for (let i = 1; i < folder_parts.length; i++) { + let subfolderGuid = await this.GetPlacesChildGuid( + folderGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + folder_parts[i] + ); + if (subfolderGuid == null) { + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: folderGuid, + name: folder_parts[i], + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + folderGuid = guid; + } else { + folderGuid = subfolderGuid; + } + } + return folderGuid; + }, + + /** + * GetOrCreateFolder + * + * Locates the specified folder; if not found it is created. + * + * @param location The full path of the folder, which must begin + * with one of the bookmark root folders + * @return the folder id if the folder was found or created, otherwise -1 + */ + async GetOrCreateFolder(location) { + let parentGuid = await this.GetFolder(location); + if (parentGuid == null) { + parentGuid = await this.CreateFolder(location); + } + return parentGuid; + }, + + /** + * CheckPosition + * + * Verifies the position of this places item. + * + * @param before The name of the places item that this item should be + before, or null if this check should be skipped + * @param after The name of the places item that this item should be + after, or null if this check should be skipped + * @param last_item_pos The index of the places item above this one, + * or null if this check should be skipped + * @return true if this item is in the correct position, otherwise false + */ + async CheckPosition(before, after, last_item_pos) { + if (after) { + if (!(await this.IsAdjacentTo(after, 1))) { + return false; + } + } + if (before) { + if (!(await this.IsAdjacentTo(before, -1))) { + return false; + } + } + if (last_item_pos != null && last_item_pos > -1) { + let index = await this.GetItemIndex(); + if (index != last_item_pos + 1) { + Logger.logPotentialError( + "Item not found at the expected index, got " + + index + + ", expected " + + (last_item_pos + 1) + + " for " + + this.toString() + ); + return false; + } + } + return true; + }, + + /** + * SetLocation + * + * Moves this places item to a different folder. + * + * @param location The full path of the folder to which to move this + * places item, which must begin with one of the bookmark root + * folders; if null, no changes are made + * @return nothing if successful, otherwise an exception is thrown + */ + async SetLocation(location) { + if (location != null) { + let newfolderGuid = await this.GetOrCreateFolder(location); + Logger.AssertTrue( + newfolderGuid, + "Location " + location + " doesn't exist; can't change item's location" + ); + await PlacesUtils.bookmarks.update({ + guid: this.props.guid, + parentGuid: newfolderGuid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + this.props.parentGuid = newfolderGuid; + } + }, + + /** + * SetPosition + * + * Updates the position of this places item within this item's current + * folder. Use SetLocation to change folders. + * + * @param position The new index this item should be moved to; if null, + * no changes are made; if -1, this item is moved to the bottom of + * the current folder. Otherwise, must be a string which is the + * title of an existing item in the folder, who's current position + * is used as the index. + * @return nothing if successful, otherwise an exception is thrown + */ + async SetPosition(position) { + if (position == null) { + return; + } + let index = -1; + if (position != -1) { + let existingGuid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + position + ); + if (existingGuid) { + index = (await PlacesUtils.bookmarks.fetch(existingGuid)).index; + } + Logger.AssertTrue( + index != -1, + "position " + position + " is invalid; unable to change position" + ); + } + await PlacesUtils.bookmarks.update({ guid: this.props.guid, index }); + }, + + /** + * Update the title of this places item + * + * @param title The new title to set for this item; if null, no changes + * are made + * @return nothing + */ + async SetTitle(title) { + if (title != null) { + await PlacesUtils.bookmarks.update({ guid: this.props.guid, title }); + } + }, +}; + +/** + * Bookmark class constructor. Initializes instance properties. + */ +export function Bookmark(props) { + PlacesItem.call(this, props); + if (this.props.title == null) { + this.props.title = this.props.uri; + } + this.props.type = "bookmark"; +} + +/** + * Bookmark instance methods. + */ +Bookmark.prototype = { + /** + * SetKeyword + * + * Update this bookmark's keyword. + * + * @param keyword The keyword to set for this bookmark; if null, no + * changes are made + * @return nothing + */ + async SetKeyword(keyword) { + if (keyword != null) { + // Mirror logic from PlacesSyncUtils's updateBookmarkMetadata + let entry = await PlacesUtils.keywords.fetch({ url: this.props.uri }); + if (entry) { + await PlacesUtils.keywords.remove(entry); + } + await PlacesUtils.keywords.insert({ keyword, url: this.props.uri }); + } + }, + + /** + * SetUri + * + * Updates this bookmark's URI. + * + * @param uri The new URI to set for this boomark; if null, no changes + * are made + * @return nothing + */ + async SetUri(uri) { + if (uri) { + let url = Services.io.newURI(uri); + await PlacesUtils.bookmarks.update({ guid: this.props.guid, url }); + } + }, + + /** + * SetTags + * + * Updates this bookmark's tags. + * + * @param tags An array of tags which should be associated with this + * bookmark; any previous tags are removed; if this param is null, + * no changes are made. If this param is an empty array, all + * tags are removed from this bookmark. + * @return nothing + */ + SetTags(tags) { + if (tags != null) { + let URI = Services.io.newURI(this.props.uri); + PlacesUtils.tagging.untagURI(URI, null); + if (tags.length) { + PlacesUtils.tagging.tagURI(URI, tags); + } + } + }, + + /** + * Create + * + * Creates the bookmark described by this object's properties. + * + * @return the id of the created bookmark + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "bookmark, error creating folder " + + this.props.location + ); + let bookmarkURI = Services.io.newURI(this.props.uri); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + url: bookmarkURI, + title: this.props.title, + }); + this.props.guid = guid; + await this.SetKeyword(this.props.keyword); + await this.SetTags(this.props.tags); + return this.props.guid; + }, + + /** + * Update + * + * Updates this bookmark's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetTitle(this.updateProps.title); + await this.SetUri(this.updateProps.uri); + await this.SetKeyword(this.updateProps.keyword); + await this.SetTags(this.updateProps.tags); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + }, + + /** + * Find + * + * Locates the bookmark which corresponds to this object's properties. + * + * @return the bookmark guid if the bookmark was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + let bookmarkTitle = this.props.title; + this.props.guid = await this.GetPlacesChildGuid( + this.props.parentGuid, + null, + bookmarkTitle, + this.props.uri + ); + + if (!this.props.guid) { + Logger.logPotentialError(this.toString() + " not found"); + return null; + } + if (this.props.keyword != null) { + let { keyword } = await PlacesSyncUtils.bookmarks.fetch(this.props.guid); + if (keyword != this.props.keyword) { + Logger.logPotentialError( + "Incorrect keyword - expected: " + + this.props.keyword + + ", actual: " + + keyword + + " for " + + this.toString() + ); + return null; + } + } + if (this.props.tags != null) { + try { + let URI = Services.io.newURI(this.props.uri); + let tags = PlacesUtils.tagging.getTagsForURI(URI); + tags.sort(); + this.props.tags.sort(); + if (JSON.stringify(tags) != JSON.stringify(this.props.tags)) { + Logger.logPotentialError( + "Wrong tags - expected: " + + JSON.stringify(this.props.tags) + + ", actual: " + + JSON.stringify(tags) + + " for " + + this.toString() + ); + return null; + } + } catch (e) { + Logger.logPotentialError("error processing tags " + e); + return null; + } + } + if ( + !(await this.CheckPosition( + this.props.before, + this.props.after, + this.props.last_item_pos + )) + ) { + return null; + } + return this.props.guid; + }, + + /** + * Remove + * + * Removes this bookmark. The bookmark should have been located previously + * by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Remove"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, +}; + +extend(Bookmark, PlacesItem); + +/** + * BookmarkFolder class constructor. Initializes instance properties. + */ +export function BookmarkFolder(props) { + PlacesItem.call(this, props); + this.props.type = "folder"; +} + +/** + * BookmarkFolder instance methods + */ +BookmarkFolder.prototype = { + /** + * Create + * + * Creates the bookmark folder described by this object's properties. + * + * @return the id of the created bookmark folder + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "folder, error creating parent folder " + + this.props.location + ); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + title: this.props.folder, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + this.props.guid = guid; + return this.props.parentGuid; + }, + + /** + * Find + * + * Locates the bookmark folder which corresponds to this object's + * properties. + * + * @return the folder guid if the folder was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + this.props.guid = await this.GetPlacesChildGuid( + this.props.parentGuid, + PlacesUtils.bookmarks.TYPE_FOLDER, + this.props.folder + ); + if (this.props.guid == null) { + return null; + } + if ( + !(await this.CheckPosition( + this.props.before, + this.props.after, + this.props.last_item_pos + )) + ) { + return null; + } + return this.props.guid; + }, + + /** + * Remove + * + * Removes this folder. The folder should have been located previously + * by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Remove"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, + + /** + * Update + * + * Updates this bookmark's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + await this.SetTitle(this.updateProps.folder); + }, +}; + +extend(BookmarkFolder, PlacesItem); + +/** + * Separator class constructor. Initializes instance properties. + */ +export function Separator(props) { + PlacesItem.call(this, props); + this.props.type = "separator"; +} + +/** + * Separator instance methods. + */ +Separator.prototype = { + /** + * Create + * + * Creates the bookmark separator described by this object's properties. + * + * @return the id of the created separator + */ + async Create() { + this.props.parentGuid = await this.GetOrCreateFolder(this.props.location); + Logger.AssertTrue( + this.props.parentGuid, + "Unable to create " + + "folder, error creating parent folder " + + this.props.location + ); + let { guid } = await PlacesUtils.bookmarks.insert({ + parentGuid: this.props.parentGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + this.props.guid = guid; + return guid; + }, + + /** + * Find + * + * Locates the bookmark separator which corresponds to this object's + * properties. + * + * @return the item guid if the separator was found, otherwise null + */ + async Find() { + this.props.parentGuid = await this.GetFolder(this.props.location); + if (this.props.parentGuid == null) { + Logger.logError("Unable to find folder " + this.props.location); + return null; + } + if (this.props.before == null && this.props.last_item_pos == null) { + Logger.logPotentialError( + "Separator requires 'before' attribute if it's the" + + "first item in the list" + ); + return null; + } + let expected_pos = -1; + if (this.props.before) { + let otherGuid = this.GetPlacesChildGuid( + this.props.parentGuid, + null, + this.props.before + ); + if (otherGuid == null) { + Logger.logPotentialError( + "Can't find places item " + + this.props.before + + " for locating separator" + ); + return null; + } + expected_pos = (await PlacesUtils.bookmarks.fetch(otherGuid)).index - 1; + } else { + expected_pos = this.props.last_item_pos + 1; + } + // Note these are IDs instead of GUIDs. + let children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + this.props.parentGuid + ); + this.props.guid = children[expected_pos]; + if (this.props.guid == null) { + Logger.logPotentialError( + "No separator found at position " + expected_pos + ); + return null; + } + let info = await PlacesUtils.bookmarks.fetch(this.props.guid); + if (info.type != PlacesUtils.bookmarks.TYPE_SEPARATOR) { + Logger.logPotentialError( + "Places item at position " + expected_pos + " is not a separator" + ); + return null; + } + return this.props.guid; + }, + + /** + * Update + * + * Updates this separator's properties according the properties on this + * object's 'updateProps' property. + * + * @return nothing + */ + async Update() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await this.SetLocation(this.updateProps.location); + await this.SetPosition(this.updateProps.position); + return true; + }, + + /** + * Remove + * + * Removes this separator. The separator should have been located + * previously by a call to Find. + * + * @return nothing + */ + async Remove() { + Logger.AssertTrue(this.props.guid, "Invalid guid during Update"); + await PlacesUtils.bookmarks.remove(this.props.guid); + }, +}; + +extend(Separator, PlacesItem); diff --git a/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs new file mode 100644 index 0000000000..587d7668f4 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/formautofill.sys.mjs @@ -0,0 +1,128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + OSKeyStore: "resource://gre/modules/OSKeyStore.sys.mjs", + formAutofillStorage: "resource://autofill/FormAutofillStorage.sys.mjs", +}); + +class FormAutofillBase { + constructor(props, subStorageName, fields) { + this._subStorageName = subStorageName; + this._fields = fields; + + this.props = {}; + this.updateProps = null; + if ("changes" in props) { + this.updateProps = props.changes; + } + for (const field of this._fields) { + this.props[field] = field in props ? props[field] : null; + } + } + + async getStorage() { + await lazy.formAutofillStorage.initialize(); + return lazy.formAutofillStorage[this._subStorageName]; + } + + async Create() { + const storage = await this.getStorage(); + await storage.add(this.props); + } + + async Find() { + const storage = await this.getStorage(); + return storage._data.find(entry => + this._fields.every(field => entry[field] === this.props[field]) + ); + } + + async Update() { + const storage = await this.getStorage(); + const { guid } = await this.Find(); + await storage.update(guid, this.updateProps, true); + } + + async Remove() { + const storage = await this.getStorage(); + const { guid } = await this.Find(); + storage.remove(guid); + } +} + +async function DumpStorage(subStorageName) { + await lazy.formAutofillStorage.initialize(); + Logger.logInfo(`\ndumping ${subStorageName} list\n`, true); + const entries = lazy.formAutofillStorage[subStorageName]._data; + for (const entry of entries) { + Logger.logInfo(JSON.stringify(entry), true); + } + Logger.logInfo(`\n\nend ${subStorageName} list\n`, true); +} + +const ADDRESS_FIELDS = [ + "given-name", + "additional-name", + "family-name", + "organization", + "street-address", + "address-level2", + "address-level1", + "postal-code", + "country", + "tel", + "email", +]; + +export class Address extends FormAutofillBase { + constructor(props) { + super(props, "addresses", ADDRESS_FIELDS); + } +} + +export async function DumpAddresses() { + await DumpStorage("addresses"); +} + +const CREDIT_CARD_FIELDS = [ + "cc-name", + "cc-number", + "cc-exp-month", + "cc-exp-year", +]; + +export class CreditCard extends FormAutofillBase { + constructor(props) { + super(props, "creditCards", CREDIT_CARD_FIELDS); + } + + async Find() { + const storage = await this.getStorage(); + await Promise.all( + storage._data.map( + async entry => + (entry["cc-number"] = await lazy.OSKeyStore.decrypt( + entry["cc-number-encrypted"] + )) + ) + ); + return storage._data.find(entry => { + return this._fields.every(field => entry[field] === this.props[field]); + }); + } +} + +export async function DumpCreditCards() { + await DumpStorage("creditCards"); +} diff --git a/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs new file mode 100644 index 0000000000..35b5f5c03b --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/forms.sys.mjs @@ -0,0 +1,205 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. Only the following + listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +import { FormHistory } from "resource://gre/modules/FormHistory.sys.mjs"; + +/** + * FormDB + * + * Helper object containing methods to interact with the FormHistory module. + */ +var FormDB = { + async _update(data) { + await FormHistory.update(data); + }, + + /** + * insertValue + * + * Adds the specified value for the specified fieldname into form history. + * + * @param fieldname The form fieldname to insert + * @param value The form value to insert + * @param us The time, in microseconds, to use for the lastUsed + * and firstUsed columns + * @return Promise<undefined> + */ + insertValue(fieldname, value, us) { + let data = { + op: "add", + fieldname, + value, + timesUsed: 1, + firstUsed: us, + lastUsed: us, + }; + return this._update(data); + }, + + /** + * updateValue + * + * Updates a row in the moz_formhistory table with a new value. + * + * @param id The id of the row to update + * @param newvalue The new value to set + * @return Promise<undefined> + */ + updateValue(id, newvalue) { + return this._update({ op: "update", guid: id, value: newvalue }); + }, + + /** + * getDataForValue + * + * Retrieves a set of values for a row in the database that + * corresponds to the given fieldname and value. + * + * @param fieldname The fieldname of the row to query + * @param value The value of the row to query + * @return Promise<null if no row is found with the specified fieldname and value, + * or an object containing the row's guid, lastUsed, and firstUsed + * values> + */ + async getDataForValue(fieldname, value) { + let results = await FormHistory.search(["guid", "lastUsed", "firstUsed"], { + fieldname, + value, + }); + if (results.length > 1) { + throw new Error("more than 1 result for this query"); + } + return results; + }, + + /** + * remove + * + * Removes the specified GUID from the database. + * + * @param guid The guid of the item to delete + * @return Promise<> + */ + remove(guid) { + return this._update({ op: "remove", guid }); + }, +}; + +/** + * FormData class constructor + * + * Initializes instance properties. + */ +export function FormData(props, msSinceEpoch) { + this.fieldname = null; + this.value = null; + this.date = 0; + this.newvalue = null; + this.usSinceEpoch = msSinceEpoch * 1000; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * FormData instance methods + */ +FormData.prototype = { + /** + * hours_to_us + * + * Converts hours since present to microseconds since epoch. + * + * @param hours The number of hours since the present time (e.g., 0 is + * 'now', and -1 is 1 hour ago) + * @return the corresponding number of microseconds since the epoch + */ + hours_to_us(hours) { + return this.usSinceEpoch + hours * 60 * 60 * 1000 * 1000; + }, + + /** + * Create + * + * If this FormData object doesn't exist in the moz_formhistory database, + * add it. Throws on error. + * + * @return nothing + */ + Create() { + Logger.AssertTrue( + this.fieldname != null && this.value != null, + "Must specify both fieldname and value" + ); + + return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => { + if (!formdata) { + // this item doesn't exist yet in the db, so we need to insert it + return FormDB.insertValue( + this.fieldname, + this.value, + this.hours_to_us(this.date) + ); + } + /* Right now, we ignore this case. If bug 552531 is ever fixed, + we might need to add code here to update the firstUsed or + lastUsed fields, as appropriate. + */ + return null; + }); + }, + + /** + * Find + * + * Attempts to locate an entry in the moz_formhistory database that + * matches the fieldname and value for this FormData object. + * + * @return true if this entry exists in the database, otherwise false + */ + Find() { + return FormDB.getDataForValue(this.fieldname, this.value).then(formdata => { + let status = formdata != null; + if (status) { + /* + //form history dates currently not synced! bug 552531 + let us = this.hours_to_us(this.date); + status = Logger.AssertTrue( + us >= formdata.firstUsed && us <= formdata.lastUsed, + "No match for with that date value"); + + if (status) + */ + this.id = formdata.guid; + } + return status; + }); + }, + + /** + * Remove + * + * Removes the row represented by this FormData instance from the + * moz_formhistory database. + * + * @return nothing + */ + async Remove() { + const formdata = await FormDB.getDataForValue(this.fieldname, this.value); + if (!formdata) { + return; + } + await FormDB.remove(formdata.guid); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs new file mode 100644 index 0000000000..845bab3aa9 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/history.sys.mjs @@ -0,0 +1,158 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +import { PlacesSyncUtils } from "resource://gre/modules/PlacesSyncUtils.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +export var DumpHistory = async function TPS_History__DumpHistory() { + let query = PlacesUtils.history.getNewQuery(); + let options = PlacesUtils.history.getNewQueryOptions(); + let root = PlacesUtils.history.executeQuery(query, options).root; + root.containerOpen = true; + Logger.logInfo("\n\ndumping history\n", true); + for (var i = 0; i < root.childCount; i++) { + let node = root.getChild(i); + let uri = node.uri; + let guid = await PlacesSyncUtils.history + .fetchGuidForURL(uri) + .catch(() => "?".repeat(12)); + let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(uri); + for (var visit of curvisits) { + Logger.logInfo( + `GUID: ${guid}, URI: ${uri}, type=${visit.type}, date=${visit.date}`, + true + ); + } + } + root.containerOpen = false; + Logger.logInfo("\nend history dump\n", true); +}; + +/** + * HistoryEntry object + * + * Contains methods for manipulating browser history entries. + */ +export var HistoryEntry = { + /** + * Add + * + * Adds visits for a uri to the history database. Throws on error. + * + * @param item An object representing one or more visits to a specific uri + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return nothing + */ + async Add(item, msSinceEpoch) { + Logger.AssertTrue( + "visits" in item && "uri" in item, + "History entry in test file must have both 'visits' " + + "and 'uri' properties" + ); + let place = { + url: item.uri, + visits: [], + }; + for (let visit of item.visits) { + let date = new Date( + Math.round(msSinceEpoch + visit.date * 60 * 60 * 1000) + ); + place.visits.push({ date, transition: visit.type }); + } + if ("title" in item) { + place.title = item.title; + } + return PlacesUtils.history.insert(place); + }, + + /** + * Find + * + * Finds visits for a uri to the history database. Throws on error. + * + * @param item An object representing one or more visits to a specific uri + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return true if all the visits for the uri are found, otherwise false + */ + async Find(item, msSinceEpoch) { + Logger.AssertTrue( + "visits" in item && "uri" in item, + "History entry in test file must have both 'visits' " + + "and 'uri' properties" + ); + let curvisits = await PlacesSyncUtils.history.fetchVisitsForURL(item.uri); + for (let visit of curvisits) { + for (let itemvisit of item.visits) { + // Note: in microseconds. + let expectedDate = + itemvisit.date * 60 * 60 * 1000 * 1000 + msSinceEpoch * 1000; + if (visit.type == itemvisit.type) { + if (itemvisit.date === undefined || visit.date == expectedDate) { + itemvisit.found = true; + } + } + } + } + + let all_items_found = true; + for (let itemvisit of item.visits) { + all_items_found = all_items_found && "found" in itemvisit; + Logger.logInfo( + `History entry for ${item.uri}, type: ${itemvisit.type}, date: ${itemvisit.date}` + + `(${ + itemvisit.date * 60 * 60 * 1000 * 1000 + }), found = ${!!itemvisit.found}` + ); + } + return all_items_found; + }, + + /** + * Delete + * + * Removes visits from the history database. Throws on error. + * + * @param item An object representing items to delete + * @param usSinceEpoch The number of microseconds from Epoch to + * the time the current Crossweave run was started + * @return nothing + */ + async Delete(item, msSinceEpoch) { + if ("uri" in item) { + let removedAny = await PlacesUtils.history.remove(item.uri); + if (!removedAny) { + Logger.log("Warning: Removed 0 history visits for uri " + item.uri); + } + } else if ("host" in item) { + await PlacesUtils.history.removeByFilter({ host: item.host }); + } else if ("begin" in item && "end" in item) { + let filter = { + beginDate: new Date(msSinceEpoch + item.begin * 60 * 60 * 1000), + endDate: new Date(msSinceEpoch + item.end * 60 * 60 * 1000), + }; + let removedAny = await PlacesUtils.history.removeVisitsByFilter(filter); + if (!removedAny) { + Logger.log( + "Warning: Removed 0 history visits with " + + JSON.stringify({ item, filter }) + ); + } + } else { + Logger.AssertTrue( + false, + "invalid entry in delete history " + JSON.stringify(item) + ); + } + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs new file mode 100644 index 0000000000..12c6d90e98 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/passwords.sys.mjs @@ -0,0 +1,187 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +import { Logger } from "resource://tps/logger.sys.mjs"; + +var nsLoginInfo = new Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +export var DumpPasswords = function TPS__Passwords__DumpPasswords() { + let logins = Services.logins.getAllLogins(); + Logger.logInfo("\ndumping password list\n", true); + for (var i = 0; i < logins.length; i++) { + Logger.logInfo( + "* origin=" + + logins[i].origin + + ", formActionOrigin=" + + logins[i].formActionOrigin + + ", realm=" + + logins[i].httpRealm + + ", password=" + + logins[i].password + + ", passwordField=" + + logins[i].passwordField + + ", username=" + + logins[i].username + + ", usernameField=" + + logins[i].usernameField, + true + ); + } + Logger.logInfo("\n\nend password list\n", true); +}; + +/** + * PasswordProps object; holds password properties. + */ +function PasswordProps(props) { + this.hostname = null; + this.submitURL = null; + this.realm = null; + this.username = ""; + this.password = ""; + this.usernameField = ""; + this.passwordField = ""; + this.delete = false; + + for (var prop in props) { + if (prop in this) { + this[prop] = props[prop]; + } + } +} + +/** + * Password class constructor. Initializes instance properties. + */ +export function Password(props) { + this.props = new PasswordProps(props); + if ("changes" in props) { + this.updateProps = new PasswordProps(props); + for (var prop in props.changes) { + if (prop in this.updateProps) { + this.updateProps[prop] = props.changes[prop]; + } + } + } else { + this.updateProps = null; + } +} + +/** + * Password instance methods. + */ +Password.prototype = { + /** + * Create + * + * Adds a password entry to the login manager for the password + * represented by this object's properties. Throws on error. + * + * @return the new login guid + */ + async Create() { + let login = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + await Services.logins.addLoginAsync(login); + login.QueryInterface(Ci.nsILoginMetaInfo); + return login.guid; + }, + + /** + * Find + * + * Finds a password entry in the login manager, for the password + * represented by this object's properties. + * + * @return the guid of the password if found, otherwise -1 + */ + Find() { + let logins = Services.logins.findLogins( + this.props.hostname, + this.props.submitURL, + this.props.realm + ); + for (var i = 0; i < logins.length; i++) { + if ( + logins[i].username == this.props.username && + logins[i].password == this.props.password && + logins[i].usernameField == this.props.usernameField && + logins[i].passwordField == this.props.passwordField + ) { + logins[i].QueryInterface(Ci.nsILoginMetaInfo); + return logins[i].guid; + } + } + return -1; + }, + + /** + * Update + * + * Updates an existing password entry in the login manager with + * new properties. Throws on error. The 'old' properties are this + * object's properties, the 'new' properties are the properties in + * this object's 'updateProps' object. + * + * @return nothing + */ + Update() { + let oldlogin = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + let newlogin = new nsLoginInfo( + this.updateProps.hostname, + this.updateProps.submitURL, + this.updateProps.realm, + this.updateProps.username, + this.updateProps.password, + this.updateProps.usernameField, + this.updateProps.passwordField + ); + Services.logins.modifyLogin(oldlogin, newlogin); + }, + + /** + * Remove + * + * Removes an entry from the login manager for a password which + * matches this object's properties. Throws on error. + * + * @return nothing + */ + Remove() { + let login = new nsLoginInfo( + this.props.hostname, + this.props.submitURL, + this.props.realm, + this.props.username, + this.props.password, + this.props.usernameField, + this.props.passwordField + ); + Services.logins.removeLogin(login); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs new file mode 100644 index 0000000000..9f6c423a40 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/prefs.sys.mjs @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +const WEAVE_PREF_PREFIX = "services.sync.prefs.sync."; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +/** + * Preference class constructor + * + * Initializes instance properties. + */ +export function Preference(props) { + Logger.AssertTrue( + "name" in props && "value" in props, + "Preference must have both name and value" + ); + + this.name = props.name; + this.value = props.value; +} + +/** + * Preference instance methods + */ +Preference.prototype = { + /** + * Modify + * + * Sets the value of the preference this.name to this.value. + * Throws on error. + * + * @return nothing + */ + Modify() { + // Determine if this pref is actually something Weave even looks at. + let weavepref = WEAVE_PREF_PREFIX + this.name; + try { + let syncPref = Services.prefs.getBoolPref(weavepref); + if (!syncPref) { + Services.prefs.setBoolPref(weavepref, true); + } + } catch (e) { + Logger.AssertTrue(false, "Weave doesn't sync pref " + this.name); + } + + // Modify the pref; throw an exception if the pref type is different + // than the value type specified in the test. + let prefType = Services.prefs.getPrefType(this.name); + switch (prefType) { + case Ci.nsIPrefBranch.PREF_INT: + Logger.AssertEqual( + typeof this.value, + "number", + "Wrong type used for preference value" + ); + Services.prefs.setIntPref(this.name, this.value); + break; + case Ci.nsIPrefBranch.PREF_STRING: + Logger.AssertEqual( + typeof this.value, + "string", + "Wrong type used for preference value" + ); + Services.prefs.setCharPref(this.name, this.value); + break; + case Ci.nsIPrefBranch.PREF_BOOL: + Logger.AssertEqual( + typeof this.value, + "boolean", + "Wrong type used for preference value" + ); + Services.prefs.setBoolPref(this.name, this.value); + break; + } + }, + + /** + * Find + * + * Verifies that the preference this.name has the value + * this.value. Throws on error, or if the pref's type or value + * doesn't match. + * + * @return nothing + */ + Find() { + // Read the pref value. + let value; + try { + let prefType = Services.prefs.getPrefType(this.name); + switch (prefType) { + case Ci.nsIPrefBranch.PREF_INT: + value = Services.prefs.getIntPref(this.name); + break; + case Ci.nsIPrefBranch.PREF_STRING: + value = Services.prefs.getCharPref(this.name); + break; + case Ci.nsIPrefBranch.PREF_BOOL: + value = Services.prefs.getBoolPref(this.name); + break; + } + } catch (e) { + Logger.AssertTrue(false, "Error accessing pref " + this.name); + } + + // Throw an exception if the current and expected values aren't of + // the same type, or don't have the same values. + Logger.AssertEqual( + typeof value, + typeof this.value, + "Value types don't match" + ); + Logger.AssertEqual(value, this.value, "Preference values don't match"); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs new file mode 100644 index 0000000000..8ea8f3b780 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/tabs.sys.mjs @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +import { Weave } from "resource://services-sync/main.sys.mjs"; + +import { Logger } from "resource://tps/logger.sys.mjs"; + +// Unfortunately, due to where TPS is run, we can't directly reuse the logic from +// BrowserTestUtils.sys.mjs. Moreover, we can't resolve the URI it loads the content +// frame script from ("chrome://mochikit/content/tests/BrowserTestUtils/content-utils.js"), +// hence the hackiness here and in BrowserTabs.Add. +Services.mm.loadFrameScript( + "data:application/javascript;charset=utf-8," + + encodeURIComponent(` + addEventListener("load", function(event) { + let subframe = event.target != content.document; + sendAsyncMessage("tps:loadEvent", {subframe: subframe, url: event.target.documentURI}); + }, true)`), + true, + true +); + +export var BrowserTabs = { + /** + * Add + * + * Opens a new tab in the current browser window for the + * given uri. Rejects on error. + * + * @param uri The uri to load in the new tab + * @return Promise + */ + async Add(uri) { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let browser = mainWindow.gBrowser; + let newtab = browser.addTrustedTab(uri); + + // Wait for the tab to load. + await new Promise(resolve => { + let mm = browser.ownerGlobal.messageManager; + mm.addMessageListener("tps:loadEvent", function onLoad(msg) { + mm.removeMessageListener("tps:loadEvent", onLoad); + resolve(); + }); + }); + + browser.selectedTab = newtab; + }, + + /** + * Find + * + * Finds the specified uri and title in Weave's list of remote tabs + * for the specified profile. + * + * @param uri The uri of the tab to find + * @param title The page title of the tab to find + * @param profile The profile to search for tabs + * @return true if the specified tab could be found, otherwise false + */ + async Find(uri, title, profile) { + // Find the uri in Weave's list of tabs for the given profile. + let tabEngine = Weave.Service.engineManager.get("tabs"); + for (let client of Weave.Service.clientsEngine.remoteClients) { + let tabClients = await tabEngine.getAllClients(); + let tabClient = tabClients.find(x => x.id === client.id); + if (!tabClient || !tabClient.tabs) { + continue; + } + for (let key in tabClient.tabs) { + let tab = tabClient.tabs[key]; + let weaveTabUrl = tab.urlHistory[0]; + if (uri == weaveTabUrl && profile == client.name) { + if (title == undefined || title == tab.title) { + return true; + } + } + } + Logger.logInfo( + `Dumping tabs for ${client.name}...\n` + + JSON.stringify(tabClient.tabs, null, 2) + ); + } + return false; + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs new file mode 100644 index 0000000000..b0798b9031 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/modules/windows.sys.mjs @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + ChromeUtils.import() and acts as a singleton. + Only the following listed symbols will exposed on import, and only when + and where imported. */ + +export var BrowserWindows = { + /** + * Add + * + * Opens a new window. Throws on error. + * + * @param aPrivate The private option. + * @return nothing + */ + Add(aPrivate, fn) { + return new Promise(resolve => { + let mainWindow = Services.wm.getMostRecentWindow("navigator:browser"); + let win = mainWindow.OpenBrowserWindow({ private: aPrivate }); + win.addEventListener( + "load", + function () { + resolve(win); + }, + { once: true } + ); + }); + }, +}; diff --git a/services/sync/tps/extensions/tps/resource/quit.sys.mjs b/services/sync/tps/extensions/tps/resource/quit.sys.mjs new file mode 100644 index 0000000000..e2cb8d8c22 --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/quit.sys.mjs @@ -0,0 +1,38 @@ +/* -*- indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + From mozilla/toolkit/content + These files did not have a license +*/ +function canQuitApplication() { + try { + var cancelQuit = Cc["@mozilla.org/supports-PRBool;1"].createInstance( + Ci.nsISupportsPRBool + ); + Services.obs.notifyObservers(cancelQuit, "quit-application-requested"); + + // Something aborted the quit process. + if (cancelQuit.data) { + return false; + } + } catch (ex) {} + + return true; +} + +export function goQuitApplication() { + if (!canQuitApplication()) { + return false; + } + + try { + Services.startup.quit(Ci.nsIAppStartup.eForceQuit); + } catch (ex) { + throw new Error(`goQuitApplication: ${ex.message}`); + } + + return true; +} diff --git a/services/sync/tps/extensions/tps/resource/tps.jsm b/services/sync/tps/extensions/tps/resource/tps.jsm new file mode 100644 index 0000000000..dfecd229aa --- /dev/null +++ b/services/sync/tps/extensions/tps/resource/tps.jsm @@ -0,0 +1,1614 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This is a JavaScript module (JSM) to be imported via + * ChromeUtils.import() and acts as a singleton. Only the following + * listed symbols will exposed on import, and only when and where imported. + */ + +var EXPORTED_SYMBOLS = [ + "ACTIONS", + "Addons", + "Addresses", + "Bookmarks", + "CreditCards", + "ExtensionStorage", + "Formdata", + "History", + "Passwords", + "Prefs", + "Tabs", + "TPS", + "Windows", +]; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Addon: "resource://tps/modules/addons.sys.mjs", + AddonValidator: "resource://services-sync/engines/addons.sys.mjs", + Address: "resource://tps/modules/formautofill.sys.mjs", + Async: "resource://services-common/async.sys.mjs", + Authentication: "resource://tps/auth/fxaccounts.sys.mjs", + Bookmark: "resource://tps/modules/bookmarks.sys.mjs", + BookmarkFolder: "resource://tps/modules/bookmarks.sys.mjs", + BookmarkValidator: "resource://tps/modules/bookmarkValidator.sys.mjs", + BrowserTabs: "resource://tps/modules/tabs.sys.mjs", + BrowserWindows: "resource://tps/modules/windows.sys.mjs", + CommonUtils: "resource://services-common/utils.sys.mjs", + CreditCard: "resource://tps/modules/formautofill.sys.mjs", + DumpAddresses: "resource://tps/modules/formautofill.sys.mjs", + DumpBookmarks: "resource://tps/modules/bookmarks.sys.mjs", + DumpCreditCards: "resource://tps/modules/formautofill.sys.mjs", + DumpHistory: "resource://tps/modules/history.sys.mjs", + DumpPasswords: "resource://tps/modules/passwords.sys.mjs", + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", + FormData: "resource://tps/modules/forms.sys.mjs", + FormValidator: "resource://services-sync/engines/forms.sys.mjs", + HistoryEntry: "resource://tps/modules/history.sys.mjs", + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + Livemark: "resource://tps/modules/bookmarks.sys.mjs", + Log: "resource://gre/modules/Log.sys.mjs", + Logger: "resource://tps/logger.sys.mjs", + Password: "resource://tps/modules/passwords.sys.mjs", + PasswordValidator: "resource://services-sync/engines/passwords.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Preference: "resource://tps/modules/prefs.sys.mjs", + STATUS_OK: "resource://services-sync/constants.sys.mjs", + Separator: "resource://tps/modules/bookmarks.sys.mjs", + SessionStore: "resource:///modules/sessionstore/SessionStore.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + SyncTelemetry: "resource://services-sync/telemetry.sys.mjs", + WEAVE_VERSION: "resource://services-sync/constants.sys.mjs", + Weave: "resource://services-sync/main.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "fileProtocolHandler", () => { + let fileHandler = Services.io.getProtocolHandler("file"); + return fileHandler.QueryInterface(Ci.nsIFileProtocolHandler); +}); + +XPCOMUtils.defineLazyGetter(lazy, "gTextDecoder", () => { + return new TextDecoder(); +}); + +ChromeUtils.defineModuleGetter( + lazy, + "NetUtil", + "resource://gre/modules/NetUtil.jsm" +); + +// Options for wiping data during a sync +const SYNC_RESET_CLIENT = "resetClient"; +const SYNC_WIPE_CLIENT = "wipeClient"; +const SYNC_WIPE_REMOTE = "wipeRemote"; + +// Actions a test can perform +const ACTION_ADD = "add"; +const ACTION_DELETE = "delete"; +const ACTION_MODIFY = "modify"; +const ACTION_SET_ENABLED = "set-enabled"; +const ACTION_SYNC = "sync"; +const ACTION_SYNC_RESET_CLIENT = SYNC_RESET_CLIENT; +const ACTION_SYNC_WIPE_CLIENT = SYNC_WIPE_CLIENT; +const ACTION_SYNC_WIPE_REMOTE = SYNC_WIPE_REMOTE; +const ACTION_VERIFY = "verify"; +const ACTION_VERIFY_NOT = "verify-not"; + +const ACTIONS = [ + ACTION_ADD, + ACTION_DELETE, + ACTION_MODIFY, + ACTION_SET_ENABLED, + ACTION_SYNC, + ACTION_SYNC_RESET_CLIENT, + ACTION_SYNC_WIPE_CLIENT, + ACTION_SYNC_WIPE_REMOTE, + ACTION_VERIFY, + ACTION_VERIFY_NOT, +]; + +const OBSERVER_TOPICS = [ + "fxaccounts:onlogin", + "fxaccounts:onlogout", + "profile-before-change", + "weave:service:tracking-started", + "weave:service:tracking-stopped", + "weave:service:login:error", + "weave:service:setup-complete", + "weave:service:sync:finish", + "weave:service:sync:delayed", + "weave:service:sync:error", + "weave:service:sync:start", + "weave:service:resyncs-finished", + "places-browser-init-complete", +]; + +var TPS = { + _currentAction: -1, + _currentPhase: -1, + _enabledEngines: null, + _errors: 0, + _isTracking: false, + _phaseFinished: false, + _phaselist: {}, + _setupComplete: false, + _syncActive: false, + _syncCount: 0, + _syncsReportedViaTelemetry: 0, + _syncErrors: 0, + _syncWipeAction: null, + _tabsAdded: 0, + _tabsFinished: 0, + _test: null, + _triggeredSync: false, + _msSinceEpoch: 0, + _requestedQuit: false, + shouldValidateAddons: false, + shouldValidateBookmarks: false, + shouldValidatePasswords: false, + shouldValidateForms: false, + _placesInitDeferred: PromiseUtils.defer(), + + _init: function TPS__init() { + this.delayAutoSync(); + + OBSERVER_TOPICS.forEach(function (aTopic) { + Services.obs.addObserver(this, aTopic, true); + }, this); + + // Some engines bump their score during their sync, which then causes + // another sync immediately (notably, prefs and addons). We don't want + // this to happen, and there's no obvious preference to kill it - so + // we do this nasty hack to ensure the global score is always zero. + Services.prefs.addObserver("services.sync.globalScore", () => { + if (lazy.Weave.Service.scheduler.globalScore != 0) { + lazy.Weave.Service.scheduler.globalScore = 0; + } + }); + }, + + DumpError(msg, exc = null) { + this._errors++; + let errInfo; + if (exc) { + errInfo = lazy.Log.exceptionStr(exc); // includes details and stack-trace. + } else { + // always write a stack even if no error passed. + errInfo = lazy.Log.stackTrace(new Error()); + } + lazy.Logger.logError(`[phase ${this._currentPhase}] ${msg} - ${errInfo}`); + this.quit(); + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + observe: function TPS__observe(subject, topic, data) { + try { + lazy.Logger.logInfo("----------event observed: " + topic); + + switch (topic) { + case "profile-before-change": + OBSERVER_TOPICS.forEach(function (topic) { + Services.obs.removeObserver(this, topic); + }, this); + + lazy.Logger.close(); + + break; + + case "places-browser-init-complete": + this._placesInitDeferred.resolve(); + break; + + case "weave:service:setup-complete": + this._setupComplete = true; + + if (this._syncWipeAction) { + lazy.Weave.Svc.Prefs.set("firstSync", this._syncWipeAction); + this._syncWipeAction = null; + } + + break; + + case "weave:service:sync:error": + this._syncActive = false; + + this.delayAutoSync(); + + // If this is the first sync error, retry... + if (this._syncErrors === 0) { + lazy.Logger.logInfo("Sync error; retrying..."); + this._syncErrors++; + lazy.CommonUtils.nextTick(() => { + this.RunNextTestAction().catch(err => { + this.DumpError("RunNextTestActionFailed", err); + }); + }); + } else { + this._triggeredSync = false; + this.DumpError("Sync error; aborting test"); + return; + } + + break; + + case "weave:service:resyncs-finished": + this._syncActive = false; + this._syncErrors = 0; + this._triggeredSync = false; + + this.delayAutoSync(); + break; + + case "weave:service:sync:start": + // Ensure that the sync operation has been started by TPS + if (!this._triggeredSync) { + this.DumpError( + "Automatic sync got triggered, which is not allowed." + ); + } + + this._syncActive = true; + break; + + case "weave:service:tracking-started": + this._isTracking = true; + break; + + case "weave:service:tracking-stopped": + this._isTracking = false; + break; + + case "fxaccounts:onlogin": + // A user signed in - for TPS that always means sync - so configure + // that. + lazy.Weave.Service.configure().catch(e => { + this.DumpError("Configuring sync failed.", e); + }); + break; + + default: + lazy.Logger.logInfo(`unhandled event: ${topic}`); + } + } catch (e) { + this.DumpError("Observer failed", e); + } + }, + + /** + * Given that we cannot completely disable the automatic sync operations, we + * massively delay the next sync. Sync operations have to only happen when + * directly called via TPS.Sync()! + */ + delayAutoSync: function TPS_delayAutoSync() { + lazy.Weave.Svc.Prefs.set("scheduler.immediateInterval", 7200); + lazy.Weave.Svc.Prefs.set("scheduler.idleInterval", 7200); + lazy.Weave.Svc.Prefs.set("scheduler.activeInterval", 7200); + lazy.Weave.Svc.Prefs.set("syncThreshold", 10000000); + }, + + quit: function TPS__quit() { + lazy.Logger.logInfo("quitting"); + this._requestedQuit = true; + this.goQuitApplication(); + }, + + async HandleWindows(aWindow, action) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on window " + + JSON.stringify(aWindow) + ); + switch (action) { + case ACTION_ADD: + await lazy.BrowserWindows.Add(aWindow.private); + break; + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on windows" + ); + }, + + async HandleTabs(tabs, action) { + for (let tab of tabs) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on tab " + + JSON.stringify(tab) + ); + switch (action) { + case ACTION_ADD: + await lazy.BrowserTabs.Add(tab.uri); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + typeof tab.profile != "undefined", + "profile must be defined when verifying tabs" + ); + lazy.Logger.AssertTrue( + await lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile), + "error locating tab" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + typeof tab.profile != "undefined", + "profile must be defined when verifying tabs" + ); + lazy.Logger.AssertTrue( + await !lazy.BrowserTabs.Find(tab.uri, tab.title, tab.profile), + "tab found which was expected to be absent" + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on tabs" + ); + }, + + async HandlePrefs(prefs, action) { + for (let pref of prefs) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on pref " + + JSON.stringify(pref) + ); + let preference = new lazy.Preference(pref); + switch (action) { + case ACTION_MODIFY: + preference.Modify(); + break; + case ACTION_VERIFY: + preference.Find(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on pref" + ); + }, + + async HandleForms(data, action) { + this.shouldValidateForms = true; + for (let datum of data) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on form entry " + + JSON.stringify(datum) + ); + let formdata = new lazy.FormData(datum, this._msSinceEpoch); + switch (action) { + case ACTION_ADD: + await formdata.Create(); + break; + case ACTION_DELETE: + await formdata.Remove(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue(await formdata.Find(), "form data not found"); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await formdata.Find()), + "form data found, but it shouldn't be present" + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on formdata" + ); + }, + + async HandleHistory(entries, action) { + try { + for (let entry of entries) { + const entryString = JSON.stringify(entry); + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on history entry " + + entryString + ); + switch (action) { + case ACTION_ADD: + await lazy.HistoryEntry.Add(entry, this._msSinceEpoch); + break; + case ACTION_DELETE: + await lazy.HistoryEntry.Delete(entry, this._msSinceEpoch); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await lazy.HistoryEntry.Find(entry, this._msSinceEpoch), + "Uri visits not found in history database: " + entryString + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await lazy.HistoryEntry.Find(entry, this._msSinceEpoch)), + "Uri visits found in history database, but they shouldn't be: " + + entryString + ); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on history" + ); + } catch (e) { + await lazy.DumpHistory(); + throw e; + } + }, + + async HandlePasswords(passwords, action) { + this.shouldValidatePasswords = true; + try { + for (let password of passwords) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on password " + + JSON.stringify(password) + ); + let passwordOb = new lazy.Password(password); + switch (action) { + case ACTION_ADD: + lazy.Logger.AssertTrue( + (await passwordOb.Create()) > -1, + "error adding password" + ); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + passwordOb.Find() != -1, + "password not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + passwordOb.Find() == -1, + "password found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue( + passwordOb.Find() != -1, + "password not found" + ); + passwordOb.Remove(); + break; + case ACTION_MODIFY: + if (passwordOb.updateProps != null) { + lazy.Logger.AssertTrue( + passwordOb.Find() != -1, + "password not found" + ); + passwordOb.Update(); + } + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on passwords" + ); + } catch (e) { + lazy.DumpPasswords(); + throw e; + } + }, + + async HandleAddons(addons, action, state) { + this.shouldValidateAddons = true; + for (let entry of addons) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on addon " + + JSON.stringify(entry) + ); + let addon = new lazy.Addon(this, entry); + switch (action) { + case ACTION_ADD: + await addon.install(); + break; + case ACTION_DELETE: + await addon.uninstall(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await addon.find(state), + "addon " + addon.id + " not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertFalse( + await addon.find(state), + "addon " + addon.id + " is present, but it shouldn't be" + ); + break; + case ACTION_SET_ENABLED: + lazy.Logger.AssertTrue( + await addon.setEnabled(state), + "addon " + addon.id + " not found" + ); + break; + default: + throw new Error("Unknown action for add-on: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on addons" + ); + }, + + async HandleBookmarks(bookmarks, action) { + // wait for default bookmarks to be created. + await this._placesInitDeferred.promise; + this.shouldValidateBookmarks = true; + try { + let items = []; + for (let folder in bookmarks) { + let last_item_pos = -1; + for (let bookmark of bookmarks[folder]) { + lazy.Logger.clearPotentialError(); + let placesItem; + bookmark.location = folder; + + if (last_item_pos != -1) { + bookmark.last_item_pos = last_item_pos; + } + let itemGuid = null; + + if (action != ACTION_MODIFY && action != ACTION_DELETE) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on bookmark " + + JSON.stringify(bookmark) + ); + } + + if ("uri" in bookmark) { + placesItem = new lazy.Bookmark(bookmark); + } else if ("folder" in bookmark) { + placesItem = new lazy.BookmarkFolder(bookmark); + } else if ("livemark" in bookmark) { + placesItem = new lazy.Livemark(bookmark); + } else if ("separator" in bookmark) { + placesItem = new lazy.Separator(bookmark); + } + + if (action == ACTION_ADD) { + itemGuid = await placesItem.Create(); + } else { + itemGuid = await placesItem.Find(); + if (action == ACTION_VERIFY_NOT) { + lazy.Logger.AssertTrue( + itemGuid == null, + "places item exists but it shouldn't: " + + JSON.stringify(bookmark) + ); + } else { + lazy.Logger.AssertTrue(itemGuid, "places item not found", true); + } + } + + last_item_pos = await placesItem.GetItemIndex(); + items.push(placesItem); + } + } + + if (action == ACTION_DELETE || action == ACTION_MODIFY) { + for (let item of items) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on bookmark " + + JSON.stringify(item) + ); + switch (action) { + case ACTION_DELETE: + await item.Remove(); + break; + case ACTION_MODIFY: + if (item.updateProps != null) { + await item.Update(); + } + break; + } + } + } + + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on bookmarks" + ); + } catch (e) { + await lazy.DumpBookmarks(); + throw e; + } + }, + + async HandleAddresses(addresses, action) { + try { + for (let address of addresses) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on address " + + JSON.stringify(address) + ); + let addressOb = new lazy.Address(address); + switch (action) { + case ACTION_ADD: + await addressOb.Create(); + break; + case ACTION_MODIFY: + await addressOb.Update(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue(await addressOb.Find(), "address not found"); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await addressOb.Find()), + "address found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue(await addressOb.Find(), "address not found"); + await addressOb.Remove(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on addresses" + ); + } catch (e) { + await lazy.DumpAddresses(); + throw e; + } + }, + + async HandleCreditCards(creditCards, action) { + try { + for (let creditCard of creditCards) { + lazy.Logger.logInfo( + "executing action " + + action.toUpperCase() + + " on creditCard " + + JSON.stringify(creditCard) + ); + let creditCardOb = new lazy.CreditCard(creditCard); + switch (action) { + case ACTION_ADD: + await creditCardOb.Create(); + break; + case ACTION_MODIFY: + await creditCardOb.Update(); + break; + case ACTION_VERIFY: + lazy.Logger.AssertTrue( + await creditCardOb.Find(), + "creditCard not found" + ); + break; + case ACTION_VERIFY_NOT: + lazy.Logger.AssertTrue( + !(await creditCardOb.Find()), + "creditCard found, but it shouldn't exist" + ); + break; + case ACTION_DELETE: + lazy.Logger.AssertTrue( + await creditCardOb.Find(), + "creditCard not found" + ); + await creditCardOb.Remove(); + break; + default: + lazy.Logger.AssertTrue(false, "invalid action: " + action); + } + } + lazy.Logger.logPass( + "executing action " + action.toUpperCase() + " on creditCards" + ); + } catch (e) { + await lazy.DumpCreditCards(); + throw e; + } + }, + + async Cleanup() { + try { + await this.WipeServer(); + } catch (ex) { + lazy.Logger.logError( + "Failed to wipe server: " + lazy.Log.exceptionStr(ex) + ); + } + try { + if (await lazy.Authentication.isLoggedIn()) { + // signout and wait for Sync to completely reset itself. + lazy.Logger.logInfo("signing out"); + let waiter = this.promiseObserver("weave:service:start-over:finish"); + await lazy.Authentication.signOut(); + await waiter; + lazy.Logger.logInfo("signout complete"); + } + await lazy.Authentication.deleteEmail(this.config.fx_account.username); + } catch (e) { + lazy.Logger.logError("Failed to sign out: " + lazy.Log.exceptionStr(e)); + } + }, + + /** + * Use Sync's bookmark validation code to see if we've corrupted the tree. + */ + async ValidateBookmarks() { + let getServerBookmarkState = async () => { + let bookmarkEngine = lazy.Weave.Service.engineManager.get("bookmarks"); + let collection = bookmarkEngine.itemSource(); + let collectionKey = + bookmarkEngine.service.collectionKeys.keyForCollection( + bookmarkEngine.name + ); + collection.full = true; + let items = []; + let resp = await collection.get(); + for (let json of resp.obj) { + let record = new collection._recordObj(); + record.deserialize(json); + await record.decrypt(collectionKey); + items.push(record.cleartext); + } + return items; + }; + let serverRecordDumpStr; + try { + lazy.Logger.logInfo("About to perform bookmark validation"); + let clientTree = await lazy.PlacesUtils.promiseBookmarksTree("", { + includeItemIds: true, + }); + let serverRecords = await getServerBookmarkState(); + // We can't wait until catch to stringify this, since at that point it will have cycles. + serverRecordDumpStr = JSON.stringify(serverRecords); + + let validator = new lazy.BookmarkValidator(); + let { problemData } = await validator.compareServerWithClient( + serverRecords, + clientTree + ); + + for (let { name, count } of problemData.getSummary()) { + // Exclude mobile showing up on the server hackily so that we don't + // report it every time, see bug 1273234 and 1274394 for more information. + if ( + name === "serverUnexpected" && + problemData.serverUnexpected.includes("mobile") + ) { + --count; + } + if (count) { + // Log this out before we assert. This is useful in the context of TPS logs, since we + // can see the IDs in the test files. + lazy.Logger.logInfo( + `Validation problem: "${name}": ${JSON.stringify( + problemData[name] + )}` + ); + } + lazy.Logger.AssertEqual( + count, + 0, + `Bookmark validation error of type ${name}` + ); + } + } catch (e) { + // Dump the client records (should always be doable) + lazy.DumpBookmarks(); + // Dump the server records if gotten them already. + if (serverRecordDumpStr) { + lazy.Logger.logInfo( + "Server bookmark records:\n" + serverRecordDumpStr + "\n" + ); + } + this.DumpError("Bookmark validation failed", e); + } + lazy.Logger.logInfo("Bookmark validation finished"); + }, + + async ValidateCollection(engineName, ValidatorType) { + let serverRecordDumpStr; + let clientRecordDumpStr; + try { + lazy.Logger.logInfo(`About to perform validation for "${engineName}"`); + let engine = lazy.Weave.Service.engineManager.get(engineName); + let validator = new ValidatorType(engine); + let serverRecords = await validator.getServerItems(engine); + let clientRecords = await validator.getClientItems(); + try { + // This substantially improves the logs for addons while not making a + // substantial difference for the other two + clientRecordDumpStr = JSON.stringify( + clientRecords.map(r => { + let res = validator.normalizeClientItem(r); + delete res.original; // Try and prevent cyclic references + return res; + }) + ); + } catch (e) { + // ignore the error, the dump string is just here to make debugging easier. + clientRecordDumpStr = "<Cyclic value>"; + } + try { + serverRecordDumpStr = JSON.stringify(serverRecords); + } catch (e) { + // as above + serverRecordDumpStr = "<Cyclic value>"; + } + let { problemData } = await validator.compareClientWithServer( + clientRecords, + serverRecords + ); + for (let { name, count } of problemData.getSummary()) { + if (count) { + lazy.Logger.logInfo( + `Validation problem: "${name}": ${JSON.stringify( + problemData[name] + )}` + ); + } + lazy.Logger.AssertEqual( + count, + 0, + `Validation error for "${engineName}" of type "${name}"` + ); + } + } catch (e) { + // Dump the client records if possible + if (clientRecordDumpStr) { + lazy.Logger.logInfo( + `Client state for ${engineName}:\n${clientRecordDumpStr}\n` + ); + } + // Dump the server records if gotten them already. + if (serverRecordDumpStr) { + lazy.Logger.logInfo( + `Server state for ${engineName}:\n${serverRecordDumpStr}\n` + ); + } + this.DumpError(`Validation failed for ${engineName}`, e); + } + lazy.Logger.logInfo(`Validation finished for ${engineName}`); + }, + + ValidatePasswords() { + return this.ValidateCollection("passwords", lazy.PasswordValidator); + }, + + ValidateForms() { + return this.ValidateCollection("forms", lazy.FormValidator); + }, + + ValidateAddons() { + return this.ValidateCollection("addons", lazy.AddonValidator); + }, + + async RunNextTestAction() { + lazy.Logger.logInfo("Running next test action"); + try { + if (this._currentAction >= this._phaselist[this._currentPhase].length) { + // Run necessary validations and then finish up + lazy.Logger.logInfo("No more actions - running validations..."); + if (this.shouldValidateBookmarks) { + await this.ValidateBookmarks(); + } + if (this.shouldValidatePasswords) { + await this.ValidatePasswords(); + } + if (this.shouldValidateForms) { + await this.ValidateForms(); + } + if (this.shouldValidateAddons) { + await this.ValidateAddons(); + } + // Force this early so that we run the validation and detect missing pings + // *before* we start shutting down, since if we do it after, the python + // code won't notice the failure. + lazy.SyncTelemetry.shutdown(); + // we're all done + lazy.Logger.logInfo( + "test phase " + + this._currentPhase + + ": " + + (this._errors ? "FAIL" : "PASS") + ); + this._phaseFinished = true; + this.quit(); + return; + } + this.seconds_since_epoch = Services.prefs.getIntPref( + "tps.seconds_since_epoch" + ); + if (this.seconds_since_epoch) { + // Places dislikes it if we add visits in the future. We pretend the + // real time is 1 minute ago to avoid issues caused by places using a + // different clock than the one that set the seconds_since_epoch pref. + this._msSinceEpoch = (this.seconds_since_epoch - 60) * 1000; + } else { + this.DumpError("seconds-since-epoch not set"); + return; + } + + let phase = this._phaselist[this._currentPhase]; + let action = phase[this._currentAction]; + lazy.Logger.logInfo("starting action: " + action[0].name); + await action[0].apply(this, action.slice(1)); + + this._currentAction++; + } catch (e) { + if (lazy.Async.isShutdownException(e)) { + if (this._requestedQuit) { + lazy.Logger.logInfo("Sync aborted due to requested shutdown"); + } else { + this.DumpError( + "Sync aborted due to shutdown, but we didn't request it" + ); + } + } else { + this.DumpError("RunNextTestAction failed", e); + } + return; + } + await this.RunNextTestAction(); + }, + + _getFileRelativeToSourceRoot(testFileURL, relativePath) { + let file = lazy.fileProtocolHandler.getFileFromURLSpec(testFileURL); + let root = file.parent.parent.parent.parent.parent; // <root>/services/sync/tests/tps/test_foo.js // <root>/services/sync/tests/tps // <root>/services/sync/tests // <root>/services/sync // <root>/services // <root> + root.appendRelativePath(relativePath); + root.normalize(); + return root; + }, + + _pingValidator: null, + + // Default ping validator that always says the ping passes. This should be + // overridden unless the `testing.tps.skipPingValidation` pref is true. + get pingValidator() { + return this._pingValidator + ? this._pingValidator + : { + validate() { + lazy.Logger.logInfo( + "Not validating ping -- disabled by pref or failure to load schema" + ); + return { valid: true, errors: [] }; + }, + }; + }, + + // Attempt to load the sync_ping_schema.json and initialize `this.pingValidator` + // based on the source of the tps file. Assumes that it's at "../unit/sync_ping_schema.json" + // relative to the directory the tps test file (testFile) is contained in. + _tryLoadPingSchema(testFile) { + if (Services.prefs.getBoolPref("testing.tps.skipPingValidation", false)) { + return; + } + try { + let schemaFile = this._getFileRelativeToSourceRoot( + testFile, + "services/sync/tests/unit/sync_ping_schema.json" + ); + + let stream = Cc[ + "@mozilla.org/network/file-input-stream;1" + ].createInstance(Ci.nsIFileInputStream); + + stream.init( + schemaFile, + lazy.FileUtils.MODE_RDONLY, + lazy.FileUtils.PERMS_FILE, + 0 + ); + + let bytes = lazy.NetUtil.readInputStream(stream, stream.available()); + let schema = JSON.parse(lazy.gTextDecoder.decode(bytes)); + lazy.Logger.logInfo("Successfully loaded schema"); + + this._pingValidator = new lazy.JsonSchema.Validator(schema); + } catch (e) { + this.DumpError( + `Failed to load ping schema relative to "${testFile}".`, + e + ); + } + }, + + /** + * Runs a single test phase. + * + * This is the main entry point for each phase of a test. The TPS command + * line driver loads this module and calls into the function with the + * arguments from the command line. + * + * When a phase is executed, the file is loaded as JavaScript into the + * current object. + * + * The following keys in the options argument have meaning: + * + * - ignoreUnusedEngines If true, unused engines will be unloaded from + * Sync. This makes output easier to parse and is + * useful for debugging test failures. + * + * @param file + * String URI of the file to open. + * @param phase + * String name of the phase to run. + * @param logpath + * String path of the log file to write to. + * @param options + * Object defining addition run-time options. + */ + async RunTestPhase(file, phase, logpath, options) { + try { + let settings = options || {}; + + lazy.Logger.init(logpath); + lazy.Logger.logInfo("Sync version: " + lazy.WEAVE_VERSION); + lazy.Logger.logInfo("Firefox buildid: " + Services.appinfo.appBuildID); + lazy.Logger.logInfo("Firefox version: " + Services.appinfo.version); + lazy.Logger.logInfo( + "Firefox source revision: " + + (AppConstants.SOURCE_REVISION_URL || "unknown") + ); + lazy.Logger.logInfo("Firefox platform: " + AppConstants.platform); + + // do some sync housekeeping + if (lazy.Weave.Service.isLoggedIn) { + this.DumpError("Sync logged in on startup...profile may be dirty"); + return; + } + + // Wait for Sync service to become ready. + if (!lazy.Weave.Status.ready) { + this.waitForEvent("weave:service:ready"); + } + + await lazy.Weave.Service.promiseInitialized; + + // We only want to do this if we modified the bookmarks this phase. + this.shouldValidateBookmarks = false; + + // Always give Sync an extra tick to initialize. If we waited for the + // service:ready event, this is required to ensure all handlers have + // executed. + await lazy.Async.promiseYield(); + await this._executeTestPhase(file, phase, settings); + } catch (e) { + this.DumpError("RunTestPhase failed", e); + } + }, + + /** + * Executes a single test phase. + * + * This is called by RunTestPhase() after the environment is validated. + */ + async _executeTestPhase(file, phase, settings) { + try { + this.config = JSON.parse(Services.prefs.getCharPref("tps.config")); + // parse the test file + Services.scriptloader.loadSubScript(file, this); + this._currentPhase = phase; + // cleanup phases are in the format `cleanup-${profileName}`. + if (this._currentPhase.startsWith("cleanup-")) { + let profileToClean = this._currentPhase.slice("cleanup-".length); + this.phases[this._currentPhase] = profileToClean; + this.Phase(this._currentPhase, [[this.Cleanup]]); + } else { + // Don't bother doing this for cleanup phases. + this._tryLoadPingSchema(file); + } + let this_phase = this._phaselist[this._currentPhase]; + + if (this_phase == undefined) { + this.DumpError("invalid phase " + this._currentPhase); + return; + } + + if (this.phases[this._currentPhase] == undefined) { + this.DumpError("no profile defined for phase " + this._currentPhase); + return; + } + + // If we have restricted the active engines, unregister engines we don't + // care about. + if (settings.ignoreUnusedEngines && Array.isArray(this._enabledEngines)) { + let names = {}; + for (let name of this._enabledEngines) { + names[name] = true; + } + for (let engine of lazy.Weave.Service.engineManager.getEnabled()) { + if (!(engine.name in names)) { + lazy.Logger.logInfo("Unregistering unused engine: " + engine.name); + await lazy.Weave.Service.engineManager.unregister(engine); + } + } + } + lazy.Logger.logInfo("Starting phase " + this._currentPhase); + + lazy.Logger.logInfo( + "setting client.name to " + this.phases[this._currentPhase] + ); + lazy.Weave.Svc.Prefs.set("client.name", this.phases[this._currentPhase]); + + this._interceptSyncTelemetry(); + + // start processing the test actions + this._currentAction = 0; + await lazy.SessionStore.promiseAllWindowsRestored; + await this.RunNextTestAction(); + } catch (e) { + this.DumpError("_executeTestPhase failed", e); + } + }, + + /** + * Override sync telemetry functions so that we can detect errors generating + * the sync ping, and count how many pings we report. + */ + _interceptSyncTelemetry() { + let originalObserve = lazy.SyncTelemetry.observe; + let self = this; + lazy.SyncTelemetry.observe = function () { + try { + originalObserve.apply(this, arguments); + } catch (e) { + self.DumpError("Error when generating sync telemetry", e); + } + }; + lazy.SyncTelemetry.submit = record => { + lazy.Logger.logInfo( + "Intercepted sync telemetry submission: " + JSON.stringify(record) + ); + this._syncsReportedViaTelemetry += + record.syncs.length + (record.discarded || 0); + if (record.discarded) { + if (record.syncs.length != lazy.SyncTelemetry.maxPayloadCount) { + this.DumpError( + "Syncs discarded from ping before maximum payload count reached" + ); + } + } + // If this is the shutdown ping, check and see that the telemetry saw all the syncs. + if (record.why === "shutdown") { + // If we happen to sync outside of tps manually causing it, its not an + // error in the telemetry, so we only complain if we didn't see all of them. + if (this._syncsReportedViaTelemetry < this._syncCount) { + this.DumpError( + `Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.` + ); + } + } + if (!record.syncs.length) { + // Note: we're overwriting submit, so this is called even for pings that + // may have no data (which wouldn't be submitted to telemetry and would + // fail validation). + return; + } + // Our ping may have some undefined values, which we rely on JSON stripping + // out as part of the ping submission - but our validator fails with them, + // so round-trip via JSON here to avoid that. + record = JSON.parse(JSON.stringify(record)); + const result = this.pingValidator.validate(record); + if (!result.valid) { + // Note that we already logged the record. + this.DumpError( + "Sync ping validation failed with errors: " + + JSON.stringify(result.errors) + ); + } + }; + }, + + /** + * Register a single phase with the test harness. + * + * This is called when loading individual test files. + * + * @param phasename + * String name of the phase being loaded. + * @param fnlist + * Array of functions/actions to perform. + */ + Phase: function Test__Phase(phasename, fnlist) { + if (Object.keys(this._phaselist).length === 0) { + // This is the first phase we should force a log in + fnlist.unshift([this.Login]); + } + this._phaselist[phasename] = fnlist; + }, + + /** + * Restrict enabled Sync engines to a specified set. + * + * This can be called by a test to limit what engines are enabled. It is + * recommended to call it to reduce the overhead and log clutter for the + * test. + * + * The "clients" engine is special and is always enabled, so there is no + * need to specify it. + * + * @param names + * Array of Strings for engines to make active during the test. + */ + EnableEngines: function EnableEngines(names) { + if (!Array.isArray(names)) { + throw new Error( + "Argument to RestrictEngines() is not an array: " + typeof names + ); + } + + this._enabledEngines = names; + }, + + /** + * Returns a promise that resolves when a specific observer notification is + * resolved. This is similar to the various waitFor* functions, although is + * typically safer if you need to do some other work that may make the event + * fire. + * + * eg: + * doSomething(); // causes the event to be fired. + * await promiseObserver("something"); + * is risky as the call to doSomething may trigger the event before the + * promiseObserver call is made. Contrast with: + * + * let waiter = promiseObserver("something"); + * doSomething(); // causes the event to be fired. + * await waiter; // will return as soon as the event fires, even if it fires + * // before this function is called. + * + * @param aEventName + * String event to wait for. + */ + promiseObserver(aEventName) { + return new Promise(resolve => { + lazy.Logger.logInfo("Setting up wait for " + aEventName + "..."); + let handler = () => { + lazy.Logger.logInfo("Observed " + aEventName); + lazy.Svc.Obs.remove(aEventName, handler); + resolve(); + }; + lazy.Svc.Obs.add(aEventName, handler); + }); + }, + + /** + * Wait for the named event to be observed. + * + * Note that in general, you should probably use promiseObserver unless you + * are 100% sure that the event being waited on can only be sent after this + * call adds the listener. + * + * @param aEventName + * String event to wait for. + */ + async waitForEvent(aEventName) { + await this.promiseObserver(aEventName); + }, + + /** + * Waits for Sync to logged in before returning + */ + async waitForSetupComplete() { + if (!this._setupComplete) { + await this.waitForEvent("weave:service:setup-complete"); + } + }, + + /** + * Waits for Sync to be finished before returning + */ + async waitForSyncFinished() { + if (lazy.Weave.Service.locked) { + await this.waitForEvent("weave:service:resyncs-finished"); + } + }, + + /** + * Waits for Sync to start tracking before returning. + */ + async waitForTracking() { + if (!this._isTracking) { + await this.waitForEvent("weave:service:tracking-started"); + } + }, + + /** + * Login on the server + */ + async Login() { + if (await lazy.Authentication.isReady()) { + return; + } + + lazy.Logger.logInfo("Setting client credentials and login."); + await lazy.Authentication.signIn(this.config.fx_account); + await this.waitForSetupComplete(); + lazy.Logger.AssertEqual( + lazy.Weave.Status.service, + lazy.STATUS_OK, + "Weave status OK" + ); + await this.waitForTracking(); + }, + + /** + * Triggers a sync operation + * + * @param {String} [wipeAction] + * Type of wipe to perform (resetClient, wipeClient, wipeRemote) + * + */ + async Sync(wipeAction) { + if (this._syncActive) { + this.DumpError("Sync currently active which should be impossible"); + return; + } + lazy.Logger.logInfo( + "Executing Sync" + (wipeAction ? ": " + wipeAction : "") + ); + + // Force a wipe action if requested. In case of an initial sync the pref + // will be overwritten by Sync itself (see bug 992198), so ensure that we + // also handle it via the "weave:service:setup-complete" notification. + if (wipeAction) { + this._syncWipeAction = wipeAction; + lazy.Weave.Svc.Prefs.set("firstSync", wipeAction); + } else { + lazy.Weave.Svc.Prefs.reset("firstSync"); + } + if (!(await lazy.Weave.Service.login())) { + // We need to complete verification. + lazy.Logger.logInfo("Logging in before performing sync"); + await this.Login(); + } + ++this._syncCount; + + lazy.Logger.logInfo( + "Executing Sync" + (wipeAction ? ": " + wipeAction : "") + ); + this._triggeredSync = true; + await lazy.Weave.Service.sync(); + lazy.Logger.logInfo("Sync is complete"); + // wait a second for things to settle... + await new Promise(resolve => { + lazy.CommonUtils.namedTimer(resolve, 1000, this, "postsync"); + }); + }, + + async WipeServer() { + lazy.Logger.logInfo("Wiping data from server."); + + await this.Login(); + await lazy.Weave.Service.login(); + await lazy.Weave.Service.wipeServer(); + }, + + /** + * Action which ensures changes are being tracked before returning. + */ + async EnsureTracking() { + await this.Login(); + await this.waitForTracking(); + }, +}; + +var Addons = { + async install(addons) { + await TPS.HandleAddons(addons, ACTION_ADD); + }, + async setEnabled(addons, state) { + await TPS.HandleAddons(addons, ACTION_SET_ENABLED, state); + }, + async uninstall(addons) { + await TPS.HandleAddons(addons, ACTION_DELETE); + }, + async verify(addons, state) { + await TPS.HandleAddons(addons, ACTION_VERIFY, state); + }, + async verifyNot(addons) { + await TPS.HandleAddons(addons, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidateAddons = false; + }, +}; + +var Addresses = { + async add(addresses) { + await this.HandleAddresses(addresses, ACTION_ADD); + }, + async modify(addresses) { + await this.HandleAddresses(addresses, ACTION_MODIFY); + }, + async delete(addresses) { + await this.HandleAddresses(addresses, ACTION_DELETE); + }, + async verify(addresses) { + await this.HandleAddresses(addresses, ACTION_VERIFY); + }, + async verifyNot(addresses) { + await this.HandleAddresses(addresses, ACTION_VERIFY_NOT); + }, +}; + +var Bookmarks = { + async add(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_ADD); + }, + async modify(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_MODIFY); + }, + async delete(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_DELETE); + }, + async verify(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY); + }, + async verifyNot(bookmarks) { + await TPS.HandleBookmarks(bookmarks, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidateBookmarks = false; + }, +}; + +var CreditCards = { + async add(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_ADD); + }, + async modify(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_MODIFY); + }, + async delete(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_DELETE); + }, + async verify(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_VERIFY); + }, + async verifyNot(creditCards) { + await this.HandleCreditCards(creditCards, ACTION_VERIFY_NOT); + }, +}; + +var Formdata = { + async add(formdata) { + await this.HandleForms(formdata, ACTION_ADD); + }, + async delete(formdata) { + await this.HandleForms(formdata, ACTION_DELETE); + }, + async verify(formdata) { + await this.HandleForms(formdata, ACTION_VERIFY); + }, + async verifyNot(formdata) { + await this.HandleForms(formdata, ACTION_VERIFY_NOT); + }, +}; + +var History = { + async add(history) { + await this.HandleHistory(history, ACTION_ADD); + }, + async delete(history) { + await this.HandleHistory(history, ACTION_DELETE); + }, + async verify(history) { + await this.HandleHistory(history, ACTION_VERIFY); + }, + async verifyNot(history) { + await this.HandleHistory(history, ACTION_VERIFY_NOT); + }, +}; + +var Passwords = { + async add(passwords) { + await this.HandlePasswords(passwords, ACTION_ADD); + }, + async modify(passwords) { + await this.HandlePasswords(passwords, ACTION_MODIFY); + }, + async delete(passwords) { + await this.HandlePasswords(passwords, ACTION_DELETE); + }, + async verify(passwords) { + await this.HandlePasswords(passwords, ACTION_VERIFY); + }, + async verifyNot(passwords) { + await this.HandlePasswords(passwords, ACTION_VERIFY_NOT); + }, + skipValidation() { + TPS.shouldValidatePasswords = false; + }, +}; + +var Prefs = { + async modify(prefs) { + await TPS.HandlePrefs(prefs, ACTION_MODIFY); + }, + async verify(prefs) { + await TPS.HandlePrefs(prefs, ACTION_VERIFY); + }, +}; + +var Tabs = { + async add(tabs) { + await TPS.HandleTabs(tabs, ACTION_ADD); + }, + async verify(tabs) { + await TPS.HandleTabs(tabs, ACTION_VERIFY); + }, + async verifyNot(tabs) { + await TPS.HandleTabs(tabs, ACTION_VERIFY_NOT); + }, +}; + +var Windows = { + async add(aWindow) { + await TPS.HandleWindows(aWindow, ACTION_ADD); + }, +}; + +// Jumping through loads of hoops via calling back into a "HandleXXX" method +// and adding an ACTION_XXX indirection adds no value - let's KISS! +// eslint-disable-next-line no-unused-vars +var ExtStorage = { + async set(id, data) { + lazy.Logger.logInfo(`setting data for '${id}': ${data}`); + await lazy.extensionStorageSync.set({ id }, data); + }, + async verify(id, keys, data) { + let got = await lazy.extensionStorageSync.get({ id }, keys); + lazy.Logger.AssertEqual(got, data, `data for '${id}'/${keys}`); + }, +}; + +// Initialize TPS +TPS._init(); diff --git a/services/sync/tps/extensions/tps/schema.json b/services/sync/tps/extensions/tps/schema.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/services/sync/tps/extensions/tps/schema.json @@ -0,0 +1 @@ +[] |