1 files changed, 670 insertions, 0 deletions
diff --git a/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js b/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js
new file mode 100644
index 0000000000..16d8ed746c
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js
@@ -0,0 +1,670 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+async function promiseAllURLFrecencies() {
+  let frecencies = new Map();
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT url, frecency, recalc_frecency
+    FROM moz_places
+    WHERE url_hash BETWEEN hash('http', 'prefix_lo') AND
+                           hash('http', 'prefix_hi')`);
+  for (let row of rows) {
+    frecencies.set(row.getResultByName("url"), {
+      frecency: row.getResultByName("frecency"),
+      recalc: row.getResultByName("recalc_frecency"),
+    });
+  }
+  return frecencies;
+}
+
+function mapFilterIterator(iter, fn) {
+  let results = [];
+  for (let value of iter) {
+    let newValue = fn(value);
+    if (newValue) {
+      results.push(newValue);
+    }
+  }
+  return results;
+}
+
+add_task(async function test_update_frecencies() {
+  let buf = await openMirror("update_frecencies");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Not modified in mirror; shouldn't recalculate frecency.
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        // URL changed to B1 in mirror; should recalculate frecency for B
+        // and B1, using existing frecency to determine order.
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+      {
+        // URL changed to new URL in mirror, should recalculate frecency
+        // for new URL first, before B1.
+        guid: "bookmarkBBB1",
+        title: "B1",
+        url: "http://example.com/b1",
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkBBB1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B1",
+        bmkUri: "http://example.com/b1",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Query; shouldn't recalculate frecency.
+        guid: "queryCCCCCCC",
+        title: "C",
+        url: "place:type=6&sort=14&maxResults=10",
+      },
+    ],
+  });
+
+  info("Calculate frecencies for all local URLs");
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
+    },
+    {
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: [
+        "bookmarkBBB2",
+        "bookmarkDDDD",
+        "bookmarkEEEE",
+        "queryFFFFFFF",
+      ],
+    },
+    {
+      // Existing bookmark changed to existing URL.
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b1",
+    },
+    {
+      // Existing bookmark with new URL; should recalculate frecency first.
+      id: "bookmarkBBB1",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B1",
+      bmkUri: "http://example.com/b11",
+    },
+    {
+      id: "bookmarkBBB2",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "B2",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      // New bookmark with new URL; should recalculate frecency first.
+      id: "bookmarkDDDD",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: null,
+      bmkUri: "http://example.com/d",
+    },
+    {
+      // New bookmark with new URL.
+      id: "bookmarkEEEE",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+    },
+    {
+      // New query; shouldn't count against limit.
+      id: "queryFFFFFFF",
+      parentid: "unfiled",
+      type: "query",
+      title: "F",
+      bmkUri: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
+    },
+  ]);
+
+  info("Apply new items and recalculate 3 frecencies");
+  await buf.apply();
+  await PlacesFrecencyRecalculator.recalculateSomeFrecencies({ chunkSize: 3 });
+
+  {
+    let frecencies = await promiseAllURLFrecencies();
+    let urlsWithFrecency = mapFilterIterator(
+      frecencies.entries(),
+      ([href, { frecency, recalc }]) => (recalc == 0 ? href : null)
+    );
+
+    // A is unchanged, and we should recalculate frecency for three more
+    // random URLs.
+    equal(
+      urlsWithFrecency.length,
+      4,
+      "Should keep unchanged frecency and recalculate 3"
+    );
+    let unexpectedURLs = CommonUtils.difference(
+      urlsWithFrecency,
+      new Set([
+        // A is unchanged.
+        "http://example.com/a",
+
+        // B11, D, and E are new URLs.
+        "http://example.com/b11",
+        "http://example.com/d",
+        "http://example.com/e",
+
+        // B and B1 are existing, changed URLs.
+        "http://example.com/b",
+        "http://example.com/b1",
+      ])
+    );
+    ok(
+      !unexpectedURLs.size,
+      "Should recalculate frecency for new and changed URLs only"
+    );
+  }
+
+  info("Change non-URL property of D");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkDDDD",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "D (remote)",
+      bmkUri: "http://example.com/d",
+    },
+  ]);
+
+  info("Apply new item and recalculate remaining frecencies");
+  await buf.apply();
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  {
+    let frecencies = await promiseAllURLFrecencies();
+    let urlsWithoutFrecency = mapFilterIterator(
+      frecencies.entries(),
+      ([href, { frecency, recalc }]) => (recalc == 1 ? href : null)
+    );
+    deepEqual(
+      urlsWithoutFrecency,
+      [],
+      "Should finish calculating remaining frecencies"
+    );
+  }
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+async function setupLocalTree(localTimeSeconds) {
+  let dateAdded = new Date(localTimeSeconds * 1000);
+  let lastModified = new Date(localTimeSeconds * 1000);
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        dateAdded,
+        lastModified,
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            title: "B",
+            url: "http://example.com/b",
+            dateAdded,
+            lastModified,
+          },
+          {
+            guid: "bookmarkCCCC",
+            title: "C",
+            url: "http://example.com/c",
+            dateAdded,
+            lastModified,
+          },
+        ],
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: null,
+        url: "http://example.com/d",
+        dateAdded,
+        lastModified,
+      },
+    ],
+  });
+}
+
+// This test ensures we clean up the temp tables between merges, and don't throw
+// constraint errors recording observer notifications.
+add_task(async function test_apply_then_revert() {
+  let buf = await openMirror("apply_then_revert");
+
+  let now = Date.now() / 1000;
+  let localTimeSeconds = now - 180;
+
+  info("Set up initial local tree and mirror");
+  await setupLocalTree(localTimeSeconds);
+  let recordsToUpload = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+  });
+  await storeChangesInMirror(buf, recordsToUpload);
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEE1",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "E",
+    url: "http://example.com/e",
+    dateAdded: new Date(localTimeSeconds * 1000),
+    lastModified: new Date(localTimeSeconds * 1000),
+  });
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkEEEE", "bookmarkFFFF"],
+      modified: now,
+    },
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: ["folderAAAAAA"],
+      modified: now,
+    },
+    {
+      id: "folderAAAAAA",
+      parentid: "toolbar",
+      type: "folder",
+      title: "A (remote)",
+      children: ["bookmarkCCCC", "bookmarkBBBB"],
+      modified: now,
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b-remote",
+      modified: now,
+    },
+    {
+      id: "bookmarkDDDD",
+      deleted: true,
+      modified: now,
+    },
+    {
+      id: "bookmarkEEEE",
+      parentid: "menu",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+      modified: now,
+    },
+    {
+      id: "bookmarkFFFF",
+      parentid: "menu",
+      type: "bookmark",
+      title: "F",
+      bmkUri: "http://example.com/f",
+      modified: now,
+    },
+  ]);
+
+  info("Apply remote changes, first time");
+  let firstTimeRecords = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged after first time"
+  );
+
+  info("Revert local tree");
+  let dateAdded = new Date(localTimeSeconds * 1000);
+  await PlacesSyncUtils.bookmarks.wipe();
+  await setupLocalTree(localTimeSeconds);
+  await PlacesTestUtils.markBookmarksAsSynced();
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEE1",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "E",
+    url: "http://example.com/e",
+    dateAdded,
+    lastModified: new Date(localTimeSeconds * 1000),
+  });
+  let localIdForD = await PlacesTestUtils.promiseItemId("bookmarkDDDD");
+
+  info("Apply remote changes, second time");
+  await buf.db.execute(
+    `
+    UPDATE items SET
+      needsMerge = 1
+    WHERE guid <> :rootGuid`,
+    { rootGuid: PlacesUtils.bookmarks.rootGuid }
+  );
+  let observer = expectBookmarkChangeNotifications();
+  let secondTimeRecords = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged after second time"
+  );
+  deepEqual(
+    secondTimeRecords,
+    firstTimeRecords,
+    "Should stage identical records to upload, first and second time"
+  );
+
+  let localItemIds = await PlacesTestUtils.promiseManyItemIds([
+    "bookmarkFFFF",
+    "bookmarkEEEE",
+    "folderAAAAAA",
+    "bookmarkCCCC",
+    "bookmarkBBBB",
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+  observer.check([
+    {
+      name: "bookmark-removed",
+      params: {
+        itemId: localIdForD,
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/d",
+        title: "", // null titles get turned into empty strings.
+        guid: "bookmarkDDDD",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-guid-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "",
+        guid: "bookmarkEEEE",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkFFFF"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/f",
+        title: "F",
+        guid: "bookmarkFFFF",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        tags: "",
+        frecency: 1,
+        hidden: false,
+        visitCount: 0,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        oldIndex: 2,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkEEEE",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/e",
+        isTagging: false,
+        title: "E",
+        tags: "",
+        frecency: 0,
+        hidden: false,
+        visitCount: 0,
+        dateAdded: dateAdded.getTime(),
+        lastVisitDate: null,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("folderAAAAAA"),
+        oldIndex: 0,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "folderAAAAAA",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "",
+        isTagging: false,
+        title: "A (remote)",
+        tags: "",
+        frecency: 0,
+        hidden: false,
+        visitCount: 0,
+        dateAdded: dateAdded.getTime(),
+        lastVisitDate: null,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkCCCC",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/c",
+        isTagging: false,
+        title: "C",
+        tags: "",
+        frecency: 1,
+        hidden: false,
+        visitCount: 0,
+        dateAdded: dateAdded.getTime(),
+        lastVisitDate: null,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        oldIndex: 0,
+        newIndex: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkBBBB",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/b-remote",
+        isTagging: false,
+        title: "B",
+        tags: "",
+        frecency: -1,
+        hidden: false,
+        visitCount: 0,
+        dateAdded: dateAdded.getTime(),
+        lastVisitDate: null,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("folderAAAAAA"),
+        title: "A (remote)",
+        guid: "folderAAAAAA",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      },
+    },
+    {
+      name: "bookmark-url-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/b-remote",
+        guid: "bookmarkBBBB",
+        parentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        isTagging: false,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "E",
+              url: "http://example.com/e",
+            },
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "F",
+              url: "http://example.com/f",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A (remote)",
+              children: [
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "C",
+                  url: "http://example.com/c",
+                },
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "B",
+                  url: "http://example.com/b-remote",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should apply new structure, second time"
+  );
+
+  await storeChangesInMirror(buf, secondTimeRecords);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});