path: root/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js

/* Any copyright is dedicated to the Public Domain.
 * http://creativecommons.org/publicdomain/zero/1.0/ */

async function promiseAllURLFrecencies() {
  let frecencies = new Map();
  let db = await PlacesUtils.promiseDBConnection();
  let rows = await db.execute(`
    SELECT url, frecency, recalc_frecency
    FROM moz_places
    WHERE url_hash BETWEEN hash('http', 'prefix_lo') AND
                           hash('http', 'prefix_hi')`);
  for (let row of rows) {
    frecencies.set(row.getResultByName("url"), {
      frecency: row.getResultByName("frecency"),
      recalc: row.getResultByName("recalc_frecency"),
    });
  }
  return frecencies;
}

function mapFilterIterator(iter, fn) {
  let results = [];
  for (let value of iter) {
    let newValue = fn(value);
    if (newValue) {
      results.push(newValue);
    }
  }
  return results;
}

add_task(async function test_update_frecencies() {
  let buf = await openMirror("update_frecencies");

  info("Set up mirror");
  await PlacesUtils.bookmarks.insertTree({
    guid: PlacesUtils.bookmarks.menuGuid,
    children: [
      {
        // Not modified in mirror; shouldn't recalculate frecency.
        guid: "bookmarkAAAA",
        title: "A",
        url: "http://example.com/a",
      },
      {
        // URL changed to B1 in mirror; should recalculate frecency for B
        // and B1, using existing frecency to determine order.
        guid: "bookmarkBBBB",
        title: "B",
        url: "http://example.com/b",
      },
      {
        // URL changed to new URL in mirror, should recalculate frecency
        // for new URL first, before B1.
        guid: "bookmarkBBB1",
        title: "B1",
        url: "http://example.com/b1",
      },
    ],
  });
  await storeRecords(
    buf,
    [
      {
        id: "menu",
        parentid: "places",
        type: "folder",
        children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
      },
      {
        id: "bookmarkAAAA",
        parentid: "menu",
        type: "bookmark",
        title: "A",
        bmkUri: "http://example.com/a",
      },
      {
        id: "bookmarkBBBB",
        parentid: "menu",
        type: "bookmark",
        title: "B",
        bmkUri: "http://example.com/b",
      },
      {
        id: "bookmarkBBB1",
        parentid: "menu",
        type: "bookmark",
        title: "B1",
        bmkUri: "http://example.com/b1",
      },
    ],
    { needsMerge: false }
  );
  await PlacesTestUtils.markBookmarksAsSynced();

  info("Make local changes");
  await PlacesUtils.bookmarks.insertTree({
    guid: PlacesUtils.bookmarks.menuGuid,
    children: [
      {
        // Query; shouldn't recalculate frecency.
        guid: "queryCCCCCCC",
        title: "C",
        url: "place:type=6&sort=14&maxResults=10",
      },
    ],
  });

  info("Calculate frecencies for all local URLs");
  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();

  info("Make remote changes");
  await storeRecords(buf, [
    {
      id: "menu",
      parentid: "places",
      type: "folder",
      children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
    },
    {
      id: "unfiled",
      parentid: "places",
      type: "folder",
      children: [
        "bookmarkBBB2",
        "bookmarkDDDD",
        "bookmarkEEEE",
        "queryFFFFFFF",
      ],
    },
    {
      // Existing bookmark changed to existing URL.
      id: "bookmarkBBBB",
      parentid: "menu",
      type: "bookmark",
      title: "B",
      bmkUri: "http://example.com/b1",
    },
    {
      // Existing bookmark with new URL; should recalculate frecency first.
      id: "bookmarkBBB1",
      parentid: "menu",
      type: "bookmark",
      title: "B1",
      bmkUri: "http://example.com/b11",
    },
    {
      id: "bookmarkBBB2",
      parentid: "unfiled",
      type: "bookmark",
      title: "B2",
      bmkUri: "http://example.com/b",
    },
    {
      // New bookmark with new URL; should recalculate frecency first.
      id: "bookmarkDDDD",
      parentid: "unfiled",
      type: "bookmark",
      title: null,
      bmkUri: "http://example.com/d",
    },
    {
      // New bookmark with new URL.
      id: "bookmarkEEEE",
      parentid: "unfiled",
      type: "bookmark",
      title: "E",
      bmkUri: "http://example.com/e",
    },
    {
      // New query; shouldn't count against limit.
      id: "queryFFFFFFF",
      parentid: "unfiled",
      type: "query",
      title: "F",
      bmkUri: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
    },
  ]);

  info("Apply new items and recalculate 3 frecencies");
  await buf.apply();
  await PlacesFrecencyRecalculator.recalculateSomeFrecencies({ chunkSize: 3 });

  {
    let frecencies = await promiseAllURLFrecencies();
    let urlsWithFrecency = mapFilterIterator(
      frecencies.entries(),
      ([href, { recalc }]) => (recalc == 0 ? href : null)
    );

    // A is unchanged, and we should recalculate frecency for three more
    // random URLs.
    equal(
      urlsWithFrecency.length,
      4,
      "Should keep unchanged frecency and recalculate 3"
    );
    let unexpectedURLs = CommonUtils.difference(
      urlsWithFrecency,
      new Set([
        // A is unchanged.
        "http://example.com/a",

        // B11, D, and E are new URLs.
        "http://example.com/b11",
        "http://example.com/d",
        "http://example.com/e",

        // B and B1 are existing, changed URLs.
        "http://example.com/b",
        "http://example.com/b1",
      ])
    );
    ok(
      !unexpectedURLs.size,
      "Should recalculate frecency for new and changed URLs only"
    );
  }

  info("Change non-URL property of D");
  await storeRecords(buf, [
    {
      id: "bookmarkDDDD",
      parentid: "unfiled",
      type: "bookmark",
      title: "D (remote)",
      bmkUri: "http://example.com/d",
    },
  ]);

  info("Apply new item and recalculate remaining frecencies");
  await buf.apply();
  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();

  {
    let frecencies = await promiseAllURLFrecencies();
    let urlsWithoutFrecency = mapFilterIterator(
      frecencies.entries(),
      ([href, { recalc }]) => (recalc == 1 ? href : null)
    );
    deepEqual(
      urlsWithoutFrecency,
      [],
      "Should finish calculating remaining frecencies"
    );
  }

  await buf.finalize();
  await PlacesUtils.bookmarks.eraseEverything();
  await PlacesSyncUtils.bookmarks.reset();
});

async function setupLocalTree(localTimeSeconds) {
  let dateAdded = new Date(localTimeSeconds * 1000);
  let lastModified = new Date(localTimeSeconds * 1000);
  await PlacesUtils.bookmarks.insertTree({
    guid: PlacesUtils.bookmarks.menuGuid,
    children: [
      {
        guid: "folderAAAAAA",
        type: PlacesUtils.bookmarks.TYPE_FOLDER,
        title: "A",
        dateAdded,
        lastModified,
        children: [
          {
            guid: "bookmarkBBBB",
            title: "B",
            url: "http://example.com/b",
            dateAdded,
            lastModified,
          },
          {
            guid: "bookmarkCCCC",
            title: "C",
            url: "http://example.com/c",
            dateAdded,
            lastModified,
          },
        ],
      },
      {
        guid: "bookmarkDDDD",
        title: null,
        url: "http://example.com/d",
        dateAdded,
        lastModified,
      },
    ],
  });
}

// This test ensures we clean up the temp tables between merges, and don't throw
// constraint errors recording observer notifications.
add_task(async function test_apply_then_revert() {
  let buf = await openMirror("apply_then_revert");

  let now = Date.now() / 1000;
  let localTimeSeconds = now - 180;

  info("Set up initial local tree and mirror");
  await setupLocalTree(localTimeSeconds);
  let recordsToUpload = await buf.apply({
    localTimeSeconds,
    remoteTimeSeconds: now,
  });
  await storeChangesInMirror(buf, recordsToUpload);
  await PlacesUtils.bookmarks.insert({
    guid: "bookmarkEEE1",
    parentGuid: PlacesUtils.bookmarks.menuGuid,
    title: "E",
    url: "http://example.com/e",
    dateAdded: new Date(localTimeSeconds * 1000),
    lastModified: new Date(localTimeSeconds * 1000),
  });

  info("Make remote changes");
  await storeRecords(buf, [
    {
      id: "menu",
      parentid: "places",
      type: "folder",
      children: ["bookmarkEEEE", "bookmarkFFFF"],
      modified: now,
    },
    {
      id: "toolbar",
      parentid: "places",
      type: "folder",
      children: ["folderAAAAAA"],
      modified: now,
    },
    {
      id: "folderAAAAAA",
      parentid: "toolbar",
      type: "folder",
      title: "A (remote)",
      children: ["bookmarkCCCC", "bookmarkBBBB"],
      modified: now,
    },
    {
      id: "bookmarkBBBB",
      parentid: "folderAAAAAA",
      type: "bookmark",
      title: "B",
      bmkUri: "http://example.com/b-remote",
      modified: now,
    },
    {
      id: "bookmarkDDDD",
      deleted: true,
      modified: now,
    },
    {
      id: "bookmarkEEEE",
      parentid: "menu",
      type: "bookmark",
      title: "E",
      bmkUri: "http://example.com/e",
      modified: now,
    },
    {
      id: "bookmarkFFFF",
      parentid: "menu",
      type: "bookmark",
      title: "F",
      bmkUri: "http://example.com/f",
      modified: now,
    },
  ]);

  info("Apply remote changes, first time");
  let firstTimeRecords = await buf.apply({
    localTimeSeconds,
    remoteTimeSeconds: now,
  });
  deepEqual(
    await buf.fetchUnmergedGuids(),
    [PlacesUtils.bookmarks.menuGuid],
    "Should leave menu with new remote structure unmerged after first time"
  );

  info("Revert local tree");
  let dateAdded = new Date(localTimeSeconds * 1000);
  await PlacesSyncUtils.bookmarks.wipe();
  await setupLocalTree(localTimeSeconds);
  await PlacesTestUtils.markBookmarksAsSynced();
  await PlacesUtils.bookmarks.insert({
    guid: "bookmarkEEE1",
    parentGuid: PlacesUtils.bookmarks.menuGuid,
    title: "E",
    url: "http://example.com/e",
    dateAdded,
    lastModified: new Date(localTimeSeconds * 1000),
  });
  let localIdForD = await PlacesTestUtils.promiseItemId("bookmarkDDDD");

  info("Apply remote changes, second time");
  await buf.db.execute(
    `
    UPDATE items SET
      needsMerge = 1
    WHERE guid <> :rootGuid`,
    { rootGuid: PlacesUtils.bookmarks.rootGuid }
  );
  let observer = expectBookmarkChangeNotifications();
  let secondTimeRecords = await buf.apply({
    localTimeSeconds,
    remoteTimeSeconds: now,
    notifyInStableOrder: true,
  });
  deepEqual(
    await buf.fetchUnmergedGuids(),
    [PlacesUtils.bookmarks.menuGuid],
    "Should leave menu with new remote structure unmerged after second time"
  );
  deepEqual(
    secondTimeRecords,
    firstTimeRecords,
    "Should stage identical records to upload, first and second time"
  );

  let localItemIds = await PlacesTestUtils.promiseManyItemIds([
    "bookmarkFFFF",
    "bookmarkEEEE",
    "folderAAAAAA",
    "bookmarkCCCC",
    "bookmarkBBBB",
    PlacesUtils.bookmarks.menuGuid,
  ]);
  observer.check([
    {
      name: "bookmark-removed",
      params: {
        itemId: localIdForD,
        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
        index: 1,
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        urlHref: "http://example.com/d",
        title: "", // null titles get turned into empty strings.
        guid: "bookmarkDDDD",
        parentGuid: PlacesUtils.bookmarks.menuGuid,
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
      },
    },
    {
      name: "bookmark-guid-changed",
      params: {
        itemId: localItemIds.get("bookmarkEEEE"),
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        urlHref: "",
        guid: "bookmarkEEEE",
        parentGuid: PlacesUtils.bookmarks.menuGuid,
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        isTagging: false,
      },
    },
    {
      name: "bookmark-added",
      params: {
        itemId: localItemIds.get("bookmarkFFFF"),
        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
        index: 1,
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        urlHref: "http://example.com/f",
        title: "F",
        guid: "bookmarkFFFF",
        parentGuid: PlacesUtils.bookmarks.menuGuid,
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        tags: "",
        frecency: 1,
        hidden: false,
        visitCount: 0,
      },
    },
    {
      name: "bookmark-moved",
      params: {
        itemId: localItemIds.get("bookmarkEEEE"),
        oldIndex: 2,
        newIndex: 0,
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        guid: "bookmarkEEEE",
        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
        newParentGuid: PlacesUtils.bookmarks.menuGuid,
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        urlHref: "http://example.com/e",
        isTagging: false,
        title: "E",
        tags: "",
        frecency: 0,
        hidden: false,
        visitCount: 0,
        dateAdded: dateAdded.getTime(),
        lastVisitDate: null,
      },
    },
    {
      name: "bookmark-moved",
      params: {
        itemId: localItemIds.get("folderAAAAAA"),
        oldIndex: 0,
        newIndex: 0,
        type: PlacesUtils.bookmarks.TYPE_FOLDER,
        guid: "folderAAAAAA",
        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        urlHref: "",
        isTagging: false,
        title: "A (remote)",
        tags: "",
        frecency: 0,
        hidden: false,
        visitCount: 0,
        dateAdded: dateAdded.getTime(),
        lastVisitDate: null,
      },
    },
    {
      name: "bookmark-moved",
      params: {
        itemId: localItemIds.get("bookmarkCCCC"),
        oldIndex: 1,
        newIndex: 0,
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        guid: "bookmarkCCCC",
        oldParentGuid: "folderAAAAAA",
        newParentGuid: "folderAAAAAA",
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        urlHref: "http://example.com/c",
        isTagging: false,
        title: "C",
        tags: "",
        frecency: 1,
        hidden: false,
        visitCount: 0,
        dateAdded: dateAdded.getTime(),
        lastVisitDate: null,
      },
    },
    {
      name: "bookmark-moved",
      params: {
        itemId: localItemIds.get("bookmarkBBBB"),
        oldIndex: 0,
        newIndex: 1,
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        guid: "bookmarkBBBB",
        oldParentGuid: "folderAAAAAA",
        newParentGuid: "folderAAAAAA",
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        urlHref: "http://example.com/b-remote",
        isTagging: false,
        title: "B",
        tags: "",
        frecency: -1,
        hidden: false,
        visitCount: 0,
        dateAdded: dateAdded.getTime(),
        lastVisitDate: null,
      },
    },
    {
      name: "bookmark-title-changed",
      params: {
        itemId: localItemIds.get("folderAAAAAA"),
        title: "A (remote)",
        guid: "folderAAAAAA",
        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
      },
    },
    {
      name: "bookmark-url-changed",
      params: {
        itemId: localItemIds.get("bookmarkBBBB"),
        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
        urlHref: "http://example.com/b-remote",
        guid: "bookmarkBBBB",
        parentGuid: "folderAAAAAA",
        source: PlacesUtils.bookmarks.SOURCES.SYNC,
        isTagging: false,
      },
    },
  ]);

  await assertLocalTree(
    PlacesUtils.bookmarks.rootGuid,
    {
      guid: PlacesUtils.bookmarks.rootGuid,
      type: PlacesUtils.bookmarks.TYPE_FOLDER,
      index: 0,
      title: "",
      children: [
        {
          guid: PlacesUtils.bookmarks.menuGuid,
          type: PlacesUtils.bookmarks.TYPE_FOLDER,
          index: 0,
          title: BookmarksMenuTitle,
          children: [
            {
              guid: "bookmarkEEEE",
              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
              index: 0,
              title: "E",
              url: "http://example.com/e",
            },
            {
              guid: "bookmarkFFFF",
              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
              index: 1,
              title: "F",
              url: "http://example.com/f",
            },
          ],
        },
        {
          guid: PlacesUtils.bookmarks.toolbarGuid,
          type: PlacesUtils.bookmarks.TYPE_FOLDER,
          index: 1,
          title: BookmarksToolbarTitle,
          children: [
            {
              guid: "folderAAAAAA",
              type: PlacesUtils.bookmarks.TYPE_FOLDER,
              index: 0,
              title: "A (remote)",
              children: [
                {
                  guid: "bookmarkCCCC",
                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
                  index: 0,
                  title: "C",
                  url: "http://example.com/c",
                },
                {
                  guid: "bookmarkBBBB",
                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
                  index: 1,
                  title: "B",
                  url: "http://example.com/b-remote",
                },
              ],
            },
          ],
        },
        {
          guid: PlacesUtils.bookmarks.unfiledGuid,
          type: PlacesUtils.bookmarks.TYPE_FOLDER,
          index: 3,
          title: UnfiledBookmarksTitle,
        },
        {
          guid: PlacesUtils.bookmarks.mobileGuid,
          type: PlacesUtils.bookmarks.TYPE_FOLDER,
          index: 4,
          title: MobileBookmarksTitle,
        },
      ],
    },
    "Should apply new structure, second time"
  );

  await storeChangesInMirror(buf, secondTimeRecords);
  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");

  await buf.finalize();
  await PlacesUtils.bookmarks.eraseEverything();
  await PlacesSyncUtils.bookmarks.reset();
});