From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- .../components/places/BookmarkHTMLUtils.sys.mjs | 1166 ++++++ .../components/places/BookmarkJSONUtils.sys.mjs | 583 +++ toolkit/components/places/Bookmarks.sys.mjs | 3264 +++++++++++++++ toolkit/components/places/Database.cpp | 2902 +++++++++++++ toolkit/components/places/Database.h | 389 ++ .../places/ExtensionSearchHandler.sys.mjs | 336 ++ toolkit/components/places/FaviconHelpers.cpp | 1261 ++++++ toolkit/components/places/FaviconHelpers.h | 327 ++ toolkit/components/places/Helpers.cpp | 353 ++ toolkit/components/places/Helpers.h | 281 ++ toolkit/components/places/History.cpp | 2371 +++++++++++ toolkit/components/places/History.h | 203 + toolkit/components/places/History.sys.mjs | 1738 ++++++++ .../components/places/INativePlacesEventCallback.h | 32 + toolkit/components/places/NotifyRankingChanged.h | 41 + .../components/places/PageIconProtocolHandler.cpp | 396 ++ .../components/places/PageIconProtocolHandler.h | 94 + toolkit/components/places/PlaceInfo.cpp | 120 + toolkit/components/places/PlaceInfo.h | 48 + toolkit/components/places/PlacesBackups.sys.mjs | 519 +++ toolkit/components/places/PlacesDBUtils.sys.mjs | 1443 +++++++ toolkit/components/places/PlacesExpiration.sys.mjs | 957 +++++ .../places/PlacesFrecencyRecalculator.sys.mjs | 541 +++ toolkit/components/places/PlacesPreviews.sys.mjs | 451 ++ toolkit/components/places/PlacesQuery.sys.mjs | 210 + toolkit/components/places/PlacesSyncUtils.sys.mjs | 1996 +++++++++ .../components/places/PlacesTransactions.sys.mjs | 1725 ++++++++ toolkit/components/places/PlacesUtils.sys.mjs | 3083 ++++++++++++++ toolkit/components/places/SQLFunctions.cpp | 1481 +++++++ toolkit/components/places/SQLFunctions.h | 663 +++ toolkit/components/places/Shutdown.cpp | 217 + toolkit/components/places/Shutdown.h | 155 + toolkit/components/places/SyncedBookmarksMirror.h | 30 + .../places/SyncedBookmarksMirror.sys.mjs | 2473 +++++++++++ toolkit/components/places/TaggingService.sys.mjs | 565 +++ toolkit/components/places/VisitInfo.cpp | 57 + toolkit/components/places/VisitInfo.h | 36 + toolkit/components/places/bookmark_sync/Cargo.toml | 20 + .../components/places/bookmark_sync/src/driver.rs | 259 ++ .../components/places/bookmark_sync/src/error.rs | 106 + toolkit/components/places/bookmark_sync/src/lib.rs | 27 + .../components/places/bookmark_sync/src/merger.rs | 237 ++ .../components/places/bookmark_sync/src/store.rs | 1328 ++++++ toolkit/components/places/components.conf | 128 + toolkit/components/places/moz.build | 88 + toolkit/components/places/mozIAsyncHistory.idl | 186 + .../components/places/mozIPlacesAutoComplete.idl | 113 + .../places/mozIPlacesPendingOperation.idl | 14 + .../places/mozISyncedBookmarksMirror.idl | 100 + .../components/places/nsAnnoProtocolHandler.cpp | 359 ++ toolkit/components/places/nsAnnoProtocolHandler.h | 56 + toolkit/components/places/nsFaviconService.cpp | 837 ++++ toolkit/components/places/nsFaviconService.h | 147 + toolkit/components/places/nsIFaviconService.idl | 339 ++ .../components/places/nsINavBookmarksService.idl | 211 + toolkit/components/places/nsINavHistoryService.idl | 1197 ++++++ .../places/nsIPlacesPreviewsHelperService.idl | 20 + toolkit/components/places/nsITaggingService.idl | 66 + toolkit/components/places/nsNavBookmarks.cpp | 1717 ++++++++ toolkit/components/places/nsNavBookmarks.h | 312 ++ toolkit/components/places/nsNavHistory.cpp | 3033 ++++++++++++++ toolkit/components/places/nsNavHistory.h | 465 +++ toolkit/components/places/nsNavHistoryQuery.cpp | 1241 ++++++ toolkit/components/places/nsNavHistoryQuery.h | 143 + toolkit/components/places/nsNavHistoryResult.cpp | 4409 ++++++++++++++++++++ toolkit/components/places/nsNavHistoryResult.h | 855 ++++ toolkit/components/places/nsPlacesIndexes.h | 118 + toolkit/components/places/nsPlacesMacros.h | 23 + toolkit/components/places/nsPlacesTables.h | 323 ++ toolkit/components/places/nsPlacesTriggers.h | 415 ++ .../places/tests/PlacesTestUtils.sys.mjs | 616 +++ .../places/tests/bookmarks/bookmarks_long_tag.json | 55 + .../places/tests/bookmarks/head_bookmarks.js | 150 + .../test_1016953-renaming-uncompressed.js | 119 + .../test_1017502-bookmarks_foreign_count.js | 117 + .../places/tests/bookmarks/test_1129529.js | 24 + .../places/tests/bookmarks/test_384228.js | 93 + .../places/tests/bookmarks/test_385829.js | 180 + .../places/tests/bookmarks/test_388695.js | 45 + .../places/tests/bookmarks/test_393498.js | 161 + .../tests/bookmarks/test_405938_restore_queries.js | 253 ++ .../bookmarks/test_424958-json-quoted-folders.js | 48 + .../places/tests/bookmarks/test_448584.js | 90 + .../places/tests/bookmarks/test_458683.js | 111 + .../bookmarks/test_466303-json-remove-backups.js | 86 + .../bookmarks/test_477583_json-backup-in-future.js | 56 + .../test_818584-discard-duplicate-backups.js | 66 + .../test_818587_compress-bookmarks-backups.js | 61 + .../bookmarks/test_818593-store-backup-metadata.js | 53 + .../test_992901-backup-unsorted-hierarchy.js | 65 + .../bookmarks/test_997030-bookmarks-html-encode.js | 37 + .../places/tests/bookmarks/test_async_observers.js | 71 + .../places/tests/bookmarks/test_bmindex.js | 133 + .../tests/bookmarks/test_bookmark_observer.js | 937 +++++ .../bookmarks/test_bookmarks_eraseEverything.js | 207 + .../places/tests/bookmarks/test_bookmarks_fetch.js | 614 +++ .../tests/bookmarks/test_bookmarks_getRecent.js | 117 + .../tests/bookmarks/test_bookmarks_insert.js | 432 ++ .../tests/bookmarks/test_bookmarks_insertTree.js | 590 +++ .../tests/bookmarks/test_bookmarks_moveToFolder.js | 723 ++++ .../bookmarks/test_bookmarks_notifications.js | 1135 +++++ .../tests/bookmarks/test_bookmarks_remove.js | 465 +++ .../tests/bookmarks/test_bookmarks_remove_batch.js | 129 + .../tests/bookmarks/test_bookmarks_reorder.js | 310 ++ .../tests/bookmarks/test_bookmarks_search.js | 339 ++ .../tests/bookmarks/test_bookmarks_update.js | 587 +++ .../tests/bookmarks/test_bookmarkstree_cache.js | 23 + .../test_insertTree_fixupOrSkipInvalidEntries.js | 114 + .../places/tests/bookmarks/test_keywords.js | 691 +++ .../test_removeFolderTransaction_reinsert.js | 115 + .../places/tests/bookmarks/test_savedsearches.js | 224 + .../places/tests/bookmarks/test_sync_fields.js | 438 ++ .../components/places/tests/bookmarks/test_tags.js | 128 + .../places/tests/bookmarks/test_untitled.js | 114 + .../components/places/tests/bookmarks/xpcshell.ini | 48 + .../components/places/tests/browser/1601563-1.html | 20 + .../components/places/tests/browser/1601563-2.html | 3 + .../places/tests/browser/399606-history.go-0.html | 13 + .../places/tests/browser/399606-httprefresh.html | 8 + .../tests/browser/399606-location.reload.html | 13 + .../tests/browser/399606-location.replace.html | 13 + .../tests/browser/399606-window.location.href.html | 14 + .../tests/browser/399606-window.location.html | 14 + .../places/tests/browser/461710_link_page-2.html | 13 + .../places/tests/browser/461710_link_page-3.html | 13 + .../places/tests/browser/461710_link_page.html | 13 + .../places/tests/browser/461710_visited_page.html | 9 + toolkit/components/places/tests/browser/begin.html | 10 + .../components/places/tests/browser/browser.ini | 88 + .../places/tests/browser/browser_bug1601563.js | 40 + .../places/tests/browser/browser_bug399606.js | 50 + .../places/tests/browser/browser_bug461710.js | 89 + .../places/tests/browser/browser_bug646422.js | 44 + .../places/tests/browser/browser_bug680727.js | 127 + .../tests/browser/browser_double_redirect.js | 83 + .../browser_favicon_privatebrowsing_perwindowpb.js | 43 + .../places/tests/browser/browser_history_post.js | 35 + .../browser/browser_multi_redirect_frecency.js | 177 + .../places/tests/browser/browser_notfound.js | 38 + .../browser_onvisit_title_null_for_navigation.js | 41 + .../places/tests/browser/browser_redirect.js | 149 + .../places/tests/browser/browser_redirect_self.js | 51 + .../places/tests/browser/browser_settitle.js | 48 + .../tests/browser/browser_visited_notfound.js | 60 + .../places/tests/browser/browser_visituri.js | 100 + .../tests/browser/browser_visituri_nohistory.js | 44 + ...browser_visituri_privatebrowsing_perwindowpb.js | 63 + .../places/tests/browser/empty_page.html | 8 + .../places/tests/browser/favicon-normal16.png | Bin 0 -> 286 bytes .../places/tests/browser/favicon-normal32.png | Bin 0 -> 344 bytes .../components/places/tests/browser/favicon.html | 13 + toolkit/components/places/tests/browser/final.html | 10 + toolkit/components/places/tests/browser/head.js | 74 + .../places/tests/browser/history_post.html | 12 + .../places/tests/browser/history_post.sjs | 5 + .../places/tests/browser/previews/browser.ini | 11 + .../tests/browser/previews/browser_thumbnails.js | 174 + .../places/tests/browser/redirect-target.html | 1 + .../components/places/tests/browser/redirect.sjs | 13 + .../places/tests/browser/redirect_once.sjs | 13 + .../places/tests/browser/redirect_self.sjs | 27 + .../places/tests/browser/redirect_thrice.sjs | 9 + .../places/tests/browser/redirect_twice.sjs | 9 + .../places/tests/browser/redirect_twice_perma.sjs | 9 + .../components/places/tests/browser/title1.html | 12 + .../components/places/tests/browser/title2.html | 13 + .../components/places/tests/chrome/bad_links.atom | 74 + .../chrome/browser_disableglobalhistory.xhtml | 40 + toolkit/components/places/tests/chrome/chrome.ini | 7 + toolkit/components/places/tests/chrome/head.js | 8 + .../tests/chrome/link-less-items-no-site-uri.rss | 18 + .../places/tests/chrome/link-less-items.rss | 19 + .../components/places/tests/chrome/rss_as_html.rss | 27 + .../places/tests/chrome/rss_as_html.rss^headers^ | 2 + .../places/tests/chrome/sample_feed.atom | 23 + .../places/tests/chrome/test_371798.xhtml | 76 + .../chrome/test_browser_disableglobalhistory.xhtml | 25 + .../tests/chrome/test_favicon_annotations.xhtml | 135 + .../places/tests/expiration/head_expiration.js | 112 + .../tests/expiration/test_annos_expire_never.js | 72 + .../places/tests/expiration/test_clearHistory.js | 57 + .../tests/expiration/test_debug_expiration.js | 469 +++ .../places/tests/expiration/test_idle_daily.js | 22 + .../expiration/test_interactions_expiration.js | 102 + .../places/tests/expiration/test_notifications.js | 36 + .../test_notifications_pageRemoved_allVisits.js | 156 + .../test_notifications_pageRemoved_fromStore.js | 110 + .../places/tests/expiration/test_pref_interval.js | 62 + .../places/tests/expiration/test_pref_maxpages.js | 116 + .../places/tests/expiration/xpcshell.ini | 14 + .../favicons/expected-favicon-animated16.png.png | Bin 0 -> 360 bytes .../tests/favicons/expected-favicon-big16.ico.png | Bin 0 -> 520 bytes .../tests/favicons/expected-favicon-big32.jpg.png | Bin 0 -> 3026 bytes .../tests/favicons/expected-favicon-big4.jpg.png | Bin 0 -> 87 bytes .../tests/favicons/expected-favicon-big48.ico.png | Bin 0 -> 2973 bytes .../tests/favicons/expected-favicon-big64.png.png | Bin 0 -> 10698 bytes .../favicons/expected-favicon-scale160x3.jpg.png | Bin 0 -> 887 bytes .../favicons/expected-favicon-scale3x160.jpg.png | Bin 0 -> 1057 bytes .../places/tests/favicons/favicon-animated16.png | Bin 0 -> 1791 bytes .../places/tests/favicons/favicon-big16.ico | Bin 0 -> 1406 bytes .../places/tests/favicons/favicon-big32.jpg | Bin 0 -> 3494 bytes .../places/tests/favicons/favicon-big4.jpg | Bin 0 -> 4751 bytes .../places/tests/favicons/favicon-big48.ico | Bin 0 -> 56646 bytes .../places/tests/favicons/favicon-big64.png | Bin 0 -> 10698 bytes .../tests/favicons/favicon-multi-frame16.png | Bin 0 -> 412 bytes .../tests/favicons/favicon-multi-frame32.png | Bin 0 -> 935 bytes .../tests/favicons/favicon-multi-frame64.png | Bin 0 -> 2125 bytes .../places/tests/favicons/favicon-multi.ico | Bin 0 -> 3860 bytes .../places/tests/favicons/favicon-normal16.png | Bin 0 -> 286 bytes .../places/tests/favicons/favicon-normal32.png | Bin 0 -> 344 bytes .../places/tests/favicons/favicon-scale160x3.jpg | Bin 0 -> 5095 bytes .../places/tests/favicons/favicon-scale3x160.jpg | Bin 0 -> 5059 bytes .../places/tests/favicons/head_favicons.js | 81 + toolkit/components/places/tests/favicons/noise.png | Bin 0 -> 159476 bytes .../places/tests/favicons/test_copyFavicons.js | 166 + .../tests/favicons/test_expireAllFavicons.js | 38 + .../tests/favicons/test_expire_migrated_icons.js | 30 + .../tests/favicons/test_expire_on_new_icons.js | 151 + .../tests/favicons/test_favicons_conversions.js | 192 + .../tests/favicons/test_favicons_protocols_ref.js | 114 + .../tests/favicons/test_getFaviconDataForPage.js | 131 + .../tests/favicons/test_getFaviconURLForPage.js | 101 + .../places/tests/favicons/test_heavy_favicon.js | 36 + .../tests/favicons/test_incremental_vacuum.js | 48 + .../favicons/test_moz-anno_favicon_mime_type.js | 88 + .../places/tests/favicons/test_multiple_frames.js | 46 + .../tests/favicons/test_page-icon_protocol.js | 243 ++ .../test_query_result_favicon_changed_on_child.js | 153 + .../tests/favicons/test_replaceFaviconData.js | 395 ++ .../favicons/test_replaceFaviconDataFromDataURL.js | 537 +++ .../places/tests/favicons/test_root_icons.js | 246 ++ .../favicons/test_setAndFetchFaviconForPage.js | 123 + .../test_setAndFetchFaviconForPage_failures.js | 156 + .../test_setAndFetchFaviconForPage_redirects.js | 89 + .../places/tests/favicons/test_svg_favicon.js | 34 + .../components/places/tests/favicons/xpcshell.ini | 49 + toolkit/components/places/tests/gtest/mock_Link.h | 72 + toolkit/components/places/tests/gtest/moz.build | 12 + .../places/tests/gtest/places_test_harness.h | 353 ++ .../places/tests/gtest/places_test_harness_tail.h | 89 + .../places/tests/gtest/test_IHistory.cpp | 445 ++ .../components/places/tests/gtest/test_casing.cpp | 29 + toolkit/components/places/tests/head_common.js | 928 ++++ .../places/tests/history/head_history.js | 13 + .../places/tests/history/test_async_history_api.js | 1343 ++++++ .../places/tests/history/test_bookmark_unhide.js | 26 + .../components/places/tests/history/test_fetch.js | 270 ++ .../tests/history/test_fetchAnnotatedPages.js | 146 + .../places/tests/history/test_fetchMany.js | 96 + .../places/tests/history/test_hasVisits.js | 60 + .../components/places/tests/history/test_insert.js | 196 + .../places/tests/history/test_insertMany.js | 247 ++ .../places/tests/history/test_insert_null_title.js | 78 + .../components/places/tests/history/test_remove.js | 354 ++ .../places/tests/history/test_removeByFilter.js | 497 +++ .../places/tests/history/test_removeMany.js | 206 + .../places/tests/history/test_removeVisits.js | 376 ++ .../tests/history/test_removeVisitsByFilter.js | 409 ++ .../tests/history/test_sameUri_titleChanged.js | 54 + .../components/places/tests/history/test_update.js | 700 ++++ .../tests/history/test_updatePlaces_embed.js | 81 + .../components/places/tests/history/xpcshell.ini | 20 + .../components/places/tests/legacy/head_legacy.js | 14 + .../places/tests/legacy/test_bookmarks.js | 519 +++ .../tests/legacy/test_bookmarks_setNullTitle.js | 50 + .../places/tests/legacy/test_protectRoots.js | 21 + .../components/places/tests/legacy/xpcshell.ini | 10 + .../places/tests/maintenance/corruptDB.sqlite | Bin 0 -> 32772 bytes .../places/tests/maintenance/corruptPayload.sqlite | Bin 0 -> 1146880 bytes .../components/places/tests/maintenance/head.js | 119 + .../tests/maintenance/test_corrupt_favicons.js | 16 + .../maintenance/test_corrupt_favicons_schema.js | 23 + .../maintenance/test_corrupt_places_schema.js | 22 + .../tests/maintenance/test_corrupt_telemetry.js | 24 + .../maintenance/test_favicons_replaceOnStartup.js | 14 + .../test_favicons_replaceOnStartup_clone.js | 16 + .../maintenance/test_integrity_replacement.js | 17 + .../tests/maintenance/test_places_purge_caches.js | 31 + .../maintenance/test_places_replaceOnStartup.js | 14 + .../test_places_replaceOnStartup_clone.js | 14 + .../maintenance/test_preventive_maintenance.js | 2823 +++++++++++++ ...t_preventive_maintenance_checkAndFixDatabase.js | 36 + .../test_preventive_maintenance_runTasks.js | 31 + .../places/tests/maintenance/xpcshell.ini | 20 + .../places/tests/migration/favicons_v41.sqlite | Bin 0 -> 229376 bytes .../places/tests/migration/head_migration.js | 47 + .../places/tests/migration/places_outdated.sqlite | Bin 0 -> 155648 bytes .../places/tests/migration/places_v43.sqlite | Bin 0 -> 1146880 bytes .../places/tests/migration/places_v54.sqlite | Bin 0 -> 1212416 bytes .../places/tests/migration/places_v66.sqlite | Bin 0 -> 1703936 bytes .../places/tests/migration/places_v68.sqlite | Bin 0 -> 1703936 bytes .../places/tests/migration/places_v69.sqlite | Bin 0 -> 1703936 bytes .../places/tests/migration/places_v70.sqlite | Bin 0 -> 1703936 bytes .../places/tests/migration/places_v72.sqlite | Bin 0 -> 1409024 bytes .../places/tests/migration/places_v74.sqlite | Bin 0 -> 1441792 bytes .../migration/test_current_from_downgraded.js | 29 + .../tests/migration/test_current_from_outdated.js | 47 + .../tests/migration/test_current_from_v43.js | 253 ++ .../tests/migration/test_current_from_v45.js | 100 + .../tests/migration/test_current_from_v46.js | 52 + .../tests/migration/test_current_from_v47.js | 128 + .../tests/migration/test_current_from_v48.js | 190 + .../tests/migration/test_current_from_v50.js | 209 + .../tests/migration/test_current_from_v53.js | 23 + .../tests/migration/test_current_from_v54.js | 58 + .../tests/migration/test_current_from_v66.js | 53 + .../tests/migration/test_current_from_v68.js | 35 + .../tests/migration/test_current_from_v69.js | 84 + .../tests/migration/test_current_from_v70.js | 96 + .../tests/migration/test_current_from_v72.js | 29 + .../components/places/tests/migration/xpcshell.ini | 33 + toolkit/components/places/tests/moz.build | 77 + .../places/tests/queries/head_queries.js | 354 ++ toolkit/components/places/tests/queries/readme.txt | 16 + .../components/places/tests/queries/test_415716.js | 109 + .../queries/test_abstime-annotation-domain.js | 321 ++ .../tests/queries/test_abstime-annotation-uri.js | 226 + .../components/places/tests/queries/test_async.js | 379 ++ .../places/tests/queries/test_bookmarks.js | 105 + .../queries/test_containersQueries_sorting.js | 492 +++ .../queries/test_downloadHistory_liveUpdate.js | 121 + .../places/tests/queries/test_excludeQueries.js | 118 + .../test_history_queries_tags_liveUpdate.js | 189 + .../test_history_queries_titles_liveUpdate.js | 217 + .../places/tests/queries/test_onlyBookmarked.js | 110 + .../places/tests/queries/test_options_inherit.js | 118 + .../tests/queries/test_queryMultipleFolder.js | 106 + .../tests/queries/test_querySerialization.js | 746 ++++ .../tests/queries/test_query_uri_liveupdate.js | 45 + .../places/tests/queries/test_redirects.js | 351 ++ .../queries/test_result_observeHistoryDetails.js | 155 + .../tests/queries/test_results-as-left-pane.js | 83 + .../places/tests/queries/test_results-as-roots.js | 114 + .../tests/queries/test_results-as-tag-query.js | 63 + .../places/tests/queries/test_results-as-visit.js | 158 + .../queries/test_searchTerms_includeHidden.js | 74 + .../places/tests/queries/test_search_tags.js | 73 + .../tests/queries/test_searchterms-bookmarklets.js | 63 + .../tests/queries/test_searchterms-domain.js | 197 + .../places/tests/queries/test_searchterms-uri.js | 125 + .../tests/queries/test_sort-date-site-grouping.js | 223 + .../places/tests/queries/test_sorting.js | 968 +++++ .../components/places/tests/queries/test_tags.js | 626 +++ .../places/tests/queries/test_transitions.js | 175 + .../components/places/tests/queries/xpcshell.ini | 34 + toolkit/components/places/tests/sync/head_sync.js | 449 ++ .../places/tests/sync/mirror_corrupt.sqlite | 1 + .../components/places/tests/sync/mirror_v1.sqlite | Bin 0 -> 294912 bytes .../components/places/tests/sync/mirror_v5.sqlite | Bin 0 -> 262144 bytes .../places/tests/sync/sync_utils_bookmarks.html | 18 + .../places/tests/sync/sync_utils_bookmarks.json | 94 + .../tests/sync/test_bookmark_abort_merging.js | 220 + .../places/tests/sync/test_bookmark_chunking.js | 165 + .../places/tests/sync/test_bookmark_corruption.js | 3290 +++++++++++++++ .../places/tests/sync/test_bookmark_deduping.js | 1290 ++++++ .../places/tests/sync/test_bookmark_deletion.js | 1602 +++++++ .../places/tests/sync/test_bookmark_haschanges.js | 228 + .../places/tests/sync/test_bookmark_kinds.js | 312 ++ .../places/tests/sync/test_bookmark_mirror_meta.js | 193 + .../tests/sync/test_bookmark_mirror_migration.js | 228 + .../tests/sync/test_bookmark_observer_recorder.js | 637 +++ .../places/tests/sync/test_bookmark_reconcile.js | 191 + .../tests/sync/test_bookmark_structure_changes.js | 2870 +++++++++++++ .../tests/sync/test_bookmark_value_changes.js | 2580 ++++++++++++ .../places/tests/sync/test_sync_utils.js | 3074 ++++++++++++++ toolkit/components/places/tests/sync/xpcshell.ini | 23 + .../places/tests/unit/bookmarks.corrupt.html | 36 + .../components/places/tests/unit/bookmarks.json | 307 ++ .../places/tests/unit/bookmarks.preplaces.html | 36 + .../places/tests/unit/bookmarks_corrupt.json | 72 + .../tests/unit/bookmarks_html_localized.html | 21 + .../tests/unit/bookmarks_html_singleframe.html | 10 + .../places/tests/unit/bookmarks_iconuri.json | 307 ++ .../components/places/tests/unit/head_bookmarks.js | 30 + .../places/tests/unit/missingBuiltIn.sqlite | Bin 0 -> 1179648 bytes .../tests/unit/mobile_bookmarks_folder_import.json | 135 + .../tests/unit/mobile_bookmarks_folder_merge.json | 101 + .../unit/mobile_bookmarks_multiple_folders.json | 159 + .../tests/unit/mobile_bookmarks_root_import.json | 89 + .../tests/unit/mobile_bookmarks_root_merge.json | 89 + toolkit/components/places/tests/unit/noRoot.sqlite | Bin 0 -> 1179648 bytes .../places/tests/unit/places.sparse.sqlite | Bin 0 -> 221184 bytes .../components/places/tests/unit/test_1085291.js | 48 + .../components/places/tests/unit/test_1105208.js | 25 + .../components/places/tests/unit/test_1105866.js | 77 + .../components/places/tests/unit/test_1606731.js | 21 + .../components/places/tests/unit/test_331487.js | 113 + .../components/places/tests/unit/test_384370.js | 188 + .../components/places/tests/unit/test_385397.js | 152 + .../components/places/tests/unit/test_399266.js | 82 + .../components/places/tests/unit/test_402799.js | 60 + .../components/places/tests/unit/test_412132.js | 181 + .../components/places/tests/unit/test_415460.js | 37 + .../components/places/tests/unit/test_415757.js | 92 + .../tests/unit/test_419792_node_tags_property.js | 52 + .../components/places/tests/unit/test_425563.js | 76 + .../tests/unit/test_429505_remove_shortcuts.js | 45 + .../tests/unit/test_433317_query_title_update.js | 43 + .../tests/unit/test_433525_hasChildren_crash.js | 52 + .../components/places/tests/unit/test_454977.js | 121 + .../components/places/tests/unit/test_463863.js | 56 + ...st_485442_crash_bug_nsNavHistoryQuery_GetUri.js | 19 + .../tests/unit/test_486978_sort_by_date_queries.js | 130 + .../components/places/tests/unit/test_536081.js | 35 + .../unit/test_PlacesDBUtils_removeOldCorruptDBs.js | 42 + .../places/tests/unit/test_PlacesQuery_history.js | 100 + .../test_PlacesUtils_invalidateCachedGuidFor.js | 32 + .../unit/test_PlacesUtils_invalidateCachedGuids.js | 86 + .../tests/unit/test_PlacesUtils_isRootItem.js | 21 + .../unit/test_PlacesUtils_unwrapNodes_place.js | 34 + .../tests/unit/test_asyncExecuteLegacyQueries.js | 84 + .../places/tests/unit/test_async_transactions.js | 2214 ++++++++++ .../unit/test_autocomplete_match_fallbackTitle.js | 27 + .../unit/test_bookmark-tags-changed_frequency.js | 56 + .../places/tests/unit/test_bookmarks_html.js | 417 ++ .../tests/unit/test_bookmarks_html_corrupt.js | 126 + .../unit/test_bookmarks_html_escape_entities.js | 98 + .../tests/unit/test_bookmarks_html_import_tags.js | 64 + .../tests/unit/test_bookmarks_html_localized.js | 51 + .../tests/unit/test_bookmarks_html_singleframe.js | 31 + .../places/tests/unit/test_bookmarks_json.js | 368 ++ .../tests/unit/test_bookmarks_json_corrupt.js | 65 + .../unit/test_bookmarks_restore_notification.js | 319 ++ .../unit/test_broken_folderShortcut_result.js | 78 + .../places/tests/unit/test_browserhistory.js | 122 + .../places/tests/unit/test_childlessTags.js | 140 + .../places/tests/unit/test_frecency_decay.js | 82 + .../places/tests/unit/test_frecency_observers.js | 99 + .../unit/test_frecency_origins_alternative.js | 301 ++ .../tests/unit/test_frecency_origins_recalc.js | 62 + .../tests/unit/test_frecency_pages_alternative.js | 352 ++ .../tests/unit/test_frecency_pages_recalc_alt.js | 76 + .../tests/unit/test_frecency_recalc_triggers.js | 281 ++ .../tests/unit/test_frecency_recalculator.js | 176 + .../tests/unit/test_frecency_unvisited_bookmark.js | 54 + .../tests/unit/test_frecency_zero_updated.js | 38 + .../places/tests/unit/test_getChildIndex.js | 73 + .../unit/test_get_query_param_sql_function.js | 21 + toolkit/components/places/tests/unit/test_hash.js | 50 + .../components/places/tests/unit/test_history.js | 178 + .../places/tests/unit/test_history_clear.js | 146 + .../tests/unit/test_history_notifications.js | 50 + .../places/tests/unit/test_history_observer.js | 186 + .../places/tests/unit/test_history_sidebar.js | 418 ++ .../tests/unit/test_import_mobile_bookmarks.js | 326 ++ .../places/tests/unit/test_isPageInDB.js | 10 + .../places/tests/unit/test_isURIVisited.js | 73 + .../components/places/tests/unit/test_isvisited.js | 69 + .../components/places/tests/unit/test_keywords.js | 733 ++++ .../places/tests/unit/test_lastModified.js | 78 + .../places/tests/unit/test_markpageas.js | 48 + .../components/places/tests/unit/test_metadata.js | 253 ++ .../tests/unit/test_missing_builtin_folders.js | 123 + .../places/tests/unit/test_missing_root_folder.js | 109 + .../places/tests/unit/test_multi_observation.js | 384 ++ .../places/tests/unit/test_multi_word_tags.js | 147 + .../places/tests/unit/test_nested_notifications.js | 178 + .../places/tests/unit/test_nsINavHistoryViewer.js | 285 ++ .../places/tests/unit/test_null_interfaces.js | 105 + .../components/places/tests/unit/test_origins.js | 1122 +++++ .../places/tests/unit/test_origins_parsing.js | 104 + .../tests/unit/test_pageGuid_bookmarkGuid.js | 256 ++ .../components/places/tests/unit/test_placeURIs.js | 18 + .../places/tests/unit/test_promiseBookmarksTree.js | 282 ++ .../tests/unit/test_resolveNullBookmarkTitles.js | 39 + .../places/tests/unit/test_result_sort.js | 112 + .../tests/unit/test_resultsAsVisit_details.js | 106 + .../places/tests/unit/test_sql_function_origin.js | 79 + .../places/tests/unit/test_sql_guid_functions.js | 94 + .../tests/unit/test_tag_autocomplete_search.js | 119 + .../components/places/tests/unit/test_tagging.js | 188 + .../components/places/tests/unit/test_telemetry.js | 151 + .../unit/test_update_frecency_after_delete.js | 211 + .../places/tests/unit/test_utils_backups_create.js | 152 + .../tests/unit/test_utils_backups_hasRecent.js | 43 + .../unit/test_utils_getURLsForContainerNode.js | 266 ++ .../places/tests/unit/test_utils_timeConversion.js | 51 + .../places/tests/unit/test_visitsInDB.js | 12 + toolkit/components/places/tests/unit/xpcshell.ini | 120 + 479 files changed, 123494 insertions(+) create mode 100644 toolkit/components/places/BookmarkHTMLUtils.sys.mjs create mode 100644 toolkit/components/places/BookmarkJSONUtils.sys.mjs create mode 100644 toolkit/components/places/Bookmarks.sys.mjs create mode 100644 toolkit/components/places/Database.cpp create mode 100644 toolkit/components/places/Database.h create mode 100644 toolkit/components/places/ExtensionSearchHandler.sys.mjs create mode 100644 toolkit/components/places/FaviconHelpers.cpp create mode 100644 toolkit/components/places/FaviconHelpers.h create mode 100644 toolkit/components/places/Helpers.cpp create mode 100644 toolkit/components/places/Helpers.h create mode 100644 toolkit/components/places/History.cpp create mode 100644 toolkit/components/places/History.h create mode 100644 toolkit/components/places/History.sys.mjs create mode 100644 toolkit/components/places/INativePlacesEventCallback.h create mode 100644 toolkit/components/places/NotifyRankingChanged.h create mode 100644 toolkit/components/places/PageIconProtocolHandler.cpp create mode 100644 toolkit/components/places/PageIconProtocolHandler.h create mode 100644 toolkit/components/places/PlaceInfo.cpp create mode 100644 toolkit/components/places/PlaceInfo.h create mode 100644 toolkit/components/places/PlacesBackups.sys.mjs create mode 100644 toolkit/components/places/PlacesDBUtils.sys.mjs create mode 100644 toolkit/components/places/PlacesExpiration.sys.mjs create mode 100644 toolkit/components/places/PlacesFrecencyRecalculator.sys.mjs create mode 100644 toolkit/components/places/PlacesPreviews.sys.mjs create mode 100644 toolkit/components/places/PlacesQuery.sys.mjs create mode 100644 toolkit/components/places/PlacesSyncUtils.sys.mjs create mode 100644 toolkit/components/places/PlacesTransactions.sys.mjs create mode 100644 toolkit/components/places/PlacesUtils.sys.mjs create mode 100644 toolkit/components/places/SQLFunctions.cpp create mode 100644 toolkit/components/places/SQLFunctions.h create mode 100644 toolkit/components/places/Shutdown.cpp create mode 100644 toolkit/components/places/Shutdown.h create mode 100644 toolkit/components/places/SyncedBookmarksMirror.h create mode 100644 toolkit/components/places/SyncedBookmarksMirror.sys.mjs create mode 100644 toolkit/components/places/TaggingService.sys.mjs create mode 100644 toolkit/components/places/VisitInfo.cpp create mode 100644 toolkit/components/places/VisitInfo.h create mode 100644 toolkit/components/places/bookmark_sync/Cargo.toml create mode 100644 toolkit/components/places/bookmark_sync/src/driver.rs create mode 100644 toolkit/components/places/bookmark_sync/src/error.rs create mode 100644 toolkit/components/places/bookmark_sync/src/lib.rs create mode 100644 toolkit/components/places/bookmark_sync/src/merger.rs create mode 100644 toolkit/components/places/bookmark_sync/src/store.rs create mode 100644 toolkit/components/places/components.conf create mode 100644 toolkit/components/places/moz.build create mode 100644 toolkit/components/places/mozIAsyncHistory.idl create mode 100644 toolkit/components/places/mozIPlacesAutoComplete.idl create mode 100644 toolkit/components/places/mozIPlacesPendingOperation.idl create mode 100644 toolkit/components/places/mozISyncedBookmarksMirror.idl create mode 100644 toolkit/components/places/nsAnnoProtocolHandler.cpp create mode 100644 toolkit/components/places/nsAnnoProtocolHandler.h create mode 100644 toolkit/components/places/nsFaviconService.cpp create mode 100644 toolkit/components/places/nsFaviconService.h create mode 100644 toolkit/components/places/nsIFaviconService.idl create mode 100644 toolkit/components/places/nsINavBookmarksService.idl create mode 100644 toolkit/components/places/nsINavHistoryService.idl create mode 100644 toolkit/components/places/nsIPlacesPreviewsHelperService.idl create mode 100644 toolkit/components/places/nsITaggingService.idl create mode 100644 toolkit/components/places/nsNavBookmarks.cpp create mode 100644 toolkit/components/places/nsNavBookmarks.h create mode 100644 toolkit/components/places/nsNavHistory.cpp create mode 100644 toolkit/components/places/nsNavHistory.h create mode 100644 toolkit/components/places/nsNavHistoryQuery.cpp create mode 100644 toolkit/components/places/nsNavHistoryQuery.h create mode 100644 toolkit/components/places/nsNavHistoryResult.cpp create mode 100644 toolkit/components/places/nsNavHistoryResult.h create mode 100644 toolkit/components/places/nsPlacesIndexes.h create mode 100644 toolkit/components/places/nsPlacesMacros.h create mode 100644 toolkit/components/places/nsPlacesTables.h create mode 100644 toolkit/components/places/nsPlacesTriggers.h create mode 100644 toolkit/components/places/tests/PlacesTestUtils.sys.mjs create mode 100644 toolkit/components/places/tests/bookmarks/bookmarks_long_tag.json create mode 100644 toolkit/components/places/tests/bookmarks/head_bookmarks.js create mode 100644 toolkit/components/places/tests/bookmarks/test_1016953-renaming-uncompressed.js create mode 100644 toolkit/components/places/tests/bookmarks/test_1017502-bookmarks_foreign_count.js create mode 100644 toolkit/components/places/tests/bookmarks/test_1129529.js create mode 100644 toolkit/components/places/tests/bookmarks/test_384228.js create mode 100644 toolkit/components/places/tests/bookmarks/test_385829.js create mode 100644 toolkit/components/places/tests/bookmarks/test_388695.js create mode 100644 toolkit/components/places/tests/bookmarks/test_393498.js create mode 100644 toolkit/components/places/tests/bookmarks/test_405938_restore_queries.js create mode 100644 toolkit/components/places/tests/bookmarks/test_424958-json-quoted-folders.js create mode 100644 toolkit/components/places/tests/bookmarks/test_448584.js create mode 100644 toolkit/components/places/tests/bookmarks/test_458683.js create mode 100644 toolkit/components/places/tests/bookmarks/test_466303-json-remove-backups.js create mode 100644 toolkit/components/places/tests/bookmarks/test_477583_json-backup-in-future.js create mode 100644 toolkit/components/places/tests/bookmarks/test_818584-discard-duplicate-backups.js create mode 100644 toolkit/components/places/tests/bookmarks/test_818587_compress-bookmarks-backups.js create mode 100644 toolkit/components/places/tests/bookmarks/test_818593-store-backup-metadata.js create mode 100644 toolkit/components/places/tests/bookmarks/test_992901-backup-unsorted-hierarchy.js create mode 100644 toolkit/components/places/tests/bookmarks/test_997030-bookmarks-html-encode.js create mode 100644 toolkit/components/places/tests/bookmarks/test_async_observers.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bmindex.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmark_observer.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_eraseEverything.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_fetch.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_getRecent.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_insert.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_insertTree.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_moveToFolder.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_notifications.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_remove.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_remove_batch.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_reorder.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_search.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarks_update.js create mode 100644 toolkit/components/places/tests/bookmarks/test_bookmarkstree_cache.js create mode 100644 toolkit/components/places/tests/bookmarks/test_insertTree_fixupOrSkipInvalidEntries.js create mode 100644 toolkit/components/places/tests/bookmarks/test_keywords.js create mode 100644 toolkit/components/places/tests/bookmarks/test_removeFolderTransaction_reinsert.js create mode 100644 toolkit/components/places/tests/bookmarks/test_savedsearches.js create mode 100644 toolkit/components/places/tests/bookmarks/test_sync_fields.js create mode 100644 toolkit/components/places/tests/bookmarks/test_tags.js create mode 100644 toolkit/components/places/tests/bookmarks/test_untitled.js create mode 100644 toolkit/components/places/tests/bookmarks/xpcshell.ini create mode 100644 toolkit/components/places/tests/browser/1601563-1.html create mode 100644 toolkit/components/places/tests/browser/1601563-2.html create mode 100644 toolkit/components/places/tests/browser/399606-history.go-0.html create mode 100644 toolkit/components/places/tests/browser/399606-httprefresh.html create mode 100644 toolkit/components/places/tests/browser/399606-location.reload.html create mode 100644 toolkit/components/places/tests/browser/399606-location.replace.html create mode 100644 toolkit/components/places/tests/browser/399606-window.location.href.html create mode 100644 toolkit/components/places/tests/browser/399606-window.location.html create mode 100644 toolkit/components/places/tests/browser/461710_link_page-2.html create mode 100644 toolkit/components/places/tests/browser/461710_link_page-3.html create mode 100644 toolkit/components/places/tests/browser/461710_link_page.html create mode 100644 toolkit/components/places/tests/browser/461710_visited_page.html create mode 100644 toolkit/components/places/tests/browser/begin.html create mode 100644 toolkit/components/places/tests/browser/browser.ini create mode 100644 toolkit/components/places/tests/browser/browser_bug1601563.js create mode 100644 toolkit/components/places/tests/browser/browser_bug399606.js create mode 100644 toolkit/components/places/tests/browser/browser_bug461710.js create mode 100644 toolkit/components/places/tests/browser/browser_bug646422.js create mode 100644 toolkit/components/places/tests/browser/browser_bug680727.js create mode 100644 toolkit/components/places/tests/browser/browser_double_redirect.js create mode 100644 toolkit/components/places/tests/browser/browser_favicon_privatebrowsing_perwindowpb.js create mode 100644 toolkit/components/places/tests/browser/browser_history_post.js create mode 100644 toolkit/components/places/tests/browser/browser_multi_redirect_frecency.js create mode 100644 toolkit/components/places/tests/browser/browser_notfound.js create mode 100644 toolkit/components/places/tests/browser/browser_onvisit_title_null_for_navigation.js create mode 100644 toolkit/components/places/tests/browser/browser_redirect.js create mode 100644 toolkit/components/places/tests/browser/browser_redirect_self.js create mode 100644 toolkit/components/places/tests/browser/browser_settitle.js create mode 100644 toolkit/components/places/tests/browser/browser_visited_notfound.js create mode 100644 toolkit/components/places/tests/browser/browser_visituri.js create mode 100644 toolkit/components/places/tests/browser/browser_visituri_nohistory.js create mode 100644 toolkit/components/places/tests/browser/browser_visituri_privatebrowsing_perwindowpb.js create mode 100644 toolkit/components/places/tests/browser/empty_page.html create mode 100644 toolkit/components/places/tests/browser/favicon-normal16.png create mode 100644 toolkit/components/places/tests/browser/favicon-normal32.png create mode 100644 toolkit/components/places/tests/browser/favicon.html create mode 100644 toolkit/components/places/tests/browser/final.html create mode 100644 toolkit/components/places/tests/browser/head.js create mode 100644 toolkit/components/places/tests/browser/history_post.html create mode 100644 toolkit/components/places/tests/browser/history_post.sjs create mode 100644 toolkit/components/places/tests/browser/previews/browser.ini create mode 100644 toolkit/components/places/tests/browser/previews/browser_thumbnails.js create mode 100644 toolkit/components/places/tests/browser/redirect-target.html create mode 100644 toolkit/components/places/tests/browser/redirect.sjs create mode 100644 toolkit/components/places/tests/browser/redirect_once.sjs create mode 100644 toolkit/components/places/tests/browser/redirect_self.sjs create mode 100644 toolkit/components/places/tests/browser/redirect_thrice.sjs create mode 100644 toolkit/components/places/tests/browser/redirect_twice.sjs create mode 100644 toolkit/components/places/tests/browser/redirect_twice_perma.sjs create mode 100644 toolkit/components/places/tests/browser/title1.html create mode 100644 toolkit/components/places/tests/browser/title2.html create mode 100644 toolkit/components/places/tests/chrome/bad_links.atom create mode 100644 toolkit/components/places/tests/chrome/browser_disableglobalhistory.xhtml create mode 100644 toolkit/components/places/tests/chrome/chrome.ini create mode 100644 toolkit/components/places/tests/chrome/head.js create mode 100644 toolkit/components/places/tests/chrome/link-less-items-no-site-uri.rss create mode 100644 toolkit/components/places/tests/chrome/link-less-items.rss create mode 100644 toolkit/components/places/tests/chrome/rss_as_html.rss create mode 100644 toolkit/components/places/tests/chrome/rss_as_html.rss^headers^ create mode 100644 toolkit/components/places/tests/chrome/sample_feed.atom create mode 100644 toolkit/components/places/tests/chrome/test_371798.xhtml create mode 100644 toolkit/components/places/tests/chrome/test_browser_disableglobalhistory.xhtml create mode 100644 toolkit/components/places/tests/chrome/test_favicon_annotations.xhtml create mode 100644 toolkit/components/places/tests/expiration/head_expiration.js create mode 100644 toolkit/components/places/tests/expiration/test_annos_expire_never.js create mode 100644 toolkit/components/places/tests/expiration/test_clearHistory.js create mode 100644 toolkit/components/places/tests/expiration/test_debug_expiration.js create mode 100644 toolkit/components/places/tests/expiration/test_idle_daily.js create mode 100644 toolkit/components/places/tests/expiration/test_interactions_expiration.js create mode 100644 toolkit/components/places/tests/expiration/test_notifications.js create mode 100644 toolkit/components/places/tests/expiration/test_notifications_pageRemoved_allVisits.js create mode 100644 toolkit/components/places/tests/expiration/test_notifications_pageRemoved_fromStore.js create mode 100644 toolkit/components/places/tests/expiration/test_pref_interval.js create mode 100644 toolkit/components/places/tests/expiration/test_pref_maxpages.js create mode 100644 toolkit/components/places/tests/expiration/xpcshell.ini create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-animated16.png.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-big16.ico.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-big32.jpg.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-big4.jpg.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-big48.ico.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-big64.png.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-scale160x3.jpg.png create mode 100644 toolkit/components/places/tests/favicons/expected-favicon-scale3x160.jpg.png create mode 100644 toolkit/components/places/tests/favicons/favicon-animated16.png create mode 100644 toolkit/components/places/tests/favicons/favicon-big16.ico create mode 100644 toolkit/components/places/tests/favicons/favicon-big32.jpg create mode 100644 toolkit/components/places/tests/favicons/favicon-big4.jpg create mode 100644 toolkit/components/places/tests/favicons/favicon-big48.ico create mode 100644 toolkit/components/places/tests/favicons/favicon-big64.png create mode 100644 toolkit/components/places/tests/favicons/favicon-multi-frame16.png create mode 100644 toolkit/components/places/tests/favicons/favicon-multi-frame32.png create mode 100644 toolkit/components/places/tests/favicons/favicon-multi-frame64.png create mode 100644 toolkit/components/places/tests/favicons/favicon-multi.ico create mode 100644 toolkit/components/places/tests/favicons/favicon-normal16.png create mode 100644 toolkit/components/places/tests/favicons/favicon-normal32.png create mode 100644 toolkit/components/places/tests/favicons/favicon-scale160x3.jpg create mode 100644 toolkit/components/places/tests/favicons/favicon-scale3x160.jpg create mode 100644 toolkit/components/places/tests/favicons/head_favicons.js create mode 100644 toolkit/components/places/tests/favicons/noise.png create mode 100644 toolkit/components/places/tests/favicons/test_copyFavicons.js create mode 100644 toolkit/components/places/tests/favicons/test_expireAllFavicons.js create mode 100644 toolkit/components/places/tests/favicons/test_expire_migrated_icons.js create mode 100644 toolkit/components/places/tests/favicons/test_expire_on_new_icons.js create mode 100644 toolkit/components/places/tests/favicons/test_favicons_conversions.js create mode 100644 toolkit/components/places/tests/favicons/test_favicons_protocols_ref.js create mode 100644 toolkit/components/places/tests/favicons/test_getFaviconDataForPage.js create mode 100644 toolkit/components/places/tests/favicons/test_getFaviconURLForPage.js create mode 100644 toolkit/components/places/tests/favicons/test_heavy_favicon.js create mode 100644 toolkit/components/places/tests/favicons/test_incremental_vacuum.js create mode 100644 toolkit/components/places/tests/favicons/test_moz-anno_favicon_mime_type.js create mode 100644 toolkit/components/places/tests/favicons/test_multiple_frames.js create mode 100644 toolkit/components/places/tests/favicons/test_page-icon_protocol.js create mode 100644 toolkit/components/places/tests/favicons/test_query_result_favicon_changed_on_child.js create mode 100644 toolkit/components/places/tests/favicons/test_replaceFaviconData.js create mode 100644 toolkit/components/places/tests/favicons/test_replaceFaviconDataFromDataURL.js create mode 100644 toolkit/components/places/tests/favicons/test_root_icons.js create mode 100644 toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage.js create mode 100644 toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_failures.js create mode 100644 toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_redirects.js create mode 100644 toolkit/components/places/tests/favicons/test_svg_favicon.js create mode 100644 toolkit/components/places/tests/favicons/xpcshell.ini create mode 100644 toolkit/components/places/tests/gtest/mock_Link.h create mode 100644 toolkit/components/places/tests/gtest/moz.build create mode 100644 toolkit/components/places/tests/gtest/places_test_harness.h create mode 100644 toolkit/components/places/tests/gtest/places_test_harness_tail.h create mode 100644 toolkit/components/places/tests/gtest/test_IHistory.cpp create mode 100644 toolkit/components/places/tests/gtest/test_casing.cpp create mode 100644 toolkit/components/places/tests/head_common.js create mode 100644 toolkit/components/places/tests/history/head_history.js create mode 100644 toolkit/components/places/tests/history/test_async_history_api.js create mode 100644 toolkit/components/places/tests/history/test_bookmark_unhide.js create mode 100644 toolkit/components/places/tests/history/test_fetch.js create mode 100644 toolkit/components/places/tests/history/test_fetchAnnotatedPages.js create mode 100644 toolkit/components/places/tests/history/test_fetchMany.js create mode 100644 toolkit/components/places/tests/history/test_hasVisits.js create mode 100644 toolkit/components/places/tests/history/test_insert.js create mode 100644 toolkit/components/places/tests/history/test_insertMany.js create mode 100644 toolkit/components/places/tests/history/test_insert_null_title.js create mode 100644 toolkit/components/places/tests/history/test_remove.js create mode 100644 toolkit/components/places/tests/history/test_removeByFilter.js create mode 100644 toolkit/components/places/tests/history/test_removeMany.js create mode 100644 toolkit/components/places/tests/history/test_removeVisits.js create mode 100644 toolkit/components/places/tests/history/test_removeVisitsByFilter.js create mode 100644 toolkit/components/places/tests/history/test_sameUri_titleChanged.js create mode 100644 toolkit/components/places/tests/history/test_update.js create mode 100644 toolkit/components/places/tests/history/test_updatePlaces_embed.js create mode 100644 toolkit/components/places/tests/history/xpcshell.ini create mode 100644 toolkit/components/places/tests/legacy/head_legacy.js create mode 100644 toolkit/components/places/tests/legacy/test_bookmarks.js create mode 100644 toolkit/components/places/tests/legacy/test_bookmarks_setNullTitle.js create mode 100644 toolkit/components/places/tests/legacy/test_protectRoots.js create mode 100644 toolkit/components/places/tests/legacy/xpcshell.ini create mode 100644 toolkit/components/places/tests/maintenance/corruptDB.sqlite create mode 100644 toolkit/components/places/tests/maintenance/corruptPayload.sqlite create mode 100644 toolkit/components/places/tests/maintenance/head.js create mode 100644 toolkit/components/places/tests/maintenance/test_corrupt_favicons.js create mode 100644 toolkit/components/places/tests/maintenance/test_corrupt_favicons_schema.js create mode 100644 toolkit/components/places/tests/maintenance/test_corrupt_places_schema.js create mode 100644 toolkit/components/places/tests/maintenance/test_corrupt_telemetry.js create mode 100644 toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup.js create mode 100644 toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup_clone.js create mode 100644 toolkit/components/places/tests/maintenance/test_integrity_replacement.js create mode 100644 toolkit/components/places/tests/maintenance/test_places_purge_caches.js create mode 100644 toolkit/components/places/tests/maintenance/test_places_replaceOnStartup.js create mode 100644 toolkit/components/places/tests/maintenance/test_places_replaceOnStartup_clone.js create mode 100644 toolkit/components/places/tests/maintenance/test_preventive_maintenance.js create mode 100644 toolkit/components/places/tests/maintenance/test_preventive_maintenance_checkAndFixDatabase.js create mode 100644 toolkit/components/places/tests/maintenance/test_preventive_maintenance_runTasks.js create mode 100644 toolkit/components/places/tests/maintenance/xpcshell.ini create mode 100644 toolkit/components/places/tests/migration/favicons_v41.sqlite create mode 100644 toolkit/components/places/tests/migration/head_migration.js create mode 100644 toolkit/components/places/tests/migration/places_outdated.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v43.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v54.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v66.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v68.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v69.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v70.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v72.sqlite create mode 100644 toolkit/components/places/tests/migration/places_v74.sqlite create mode 100644 toolkit/components/places/tests/migration/test_current_from_downgraded.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_outdated.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v43.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v45.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v46.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v47.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v48.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v50.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v53.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v54.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v66.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v68.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v69.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v70.js create mode 100644 toolkit/components/places/tests/migration/test_current_from_v72.js create mode 100644 toolkit/components/places/tests/migration/xpcshell.ini create mode 100644 toolkit/components/places/tests/moz.build create mode 100644 toolkit/components/places/tests/queries/head_queries.js create mode 100644 toolkit/components/places/tests/queries/readme.txt create mode 100644 toolkit/components/places/tests/queries/test_415716.js create mode 100644 toolkit/components/places/tests/queries/test_abstime-annotation-domain.js create mode 100644 toolkit/components/places/tests/queries/test_abstime-annotation-uri.js create mode 100644 toolkit/components/places/tests/queries/test_async.js create mode 100644 toolkit/components/places/tests/queries/test_bookmarks.js create mode 100644 toolkit/components/places/tests/queries/test_containersQueries_sorting.js create mode 100644 toolkit/components/places/tests/queries/test_downloadHistory_liveUpdate.js create mode 100644 toolkit/components/places/tests/queries/test_excludeQueries.js create mode 100644 toolkit/components/places/tests/queries/test_history_queries_tags_liveUpdate.js create mode 100644 toolkit/components/places/tests/queries/test_history_queries_titles_liveUpdate.js create mode 100644 toolkit/components/places/tests/queries/test_onlyBookmarked.js create mode 100644 toolkit/components/places/tests/queries/test_options_inherit.js create mode 100644 toolkit/components/places/tests/queries/test_queryMultipleFolder.js create mode 100644 toolkit/components/places/tests/queries/test_querySerialization.js create mode 100644 toolkit/components/places/tests/queries/test_query_uri_liveupdate.js create mode 100644 toolkit/components/places/tests/queries/test_redirects.js create mode 100644 toolkit/components/places/tests/queries/test_result_observeHistoryDetails.js create mode 100644 toolkit/components/places/tests/queries/test_results-as-left-pane.js create mode 100644 toolkit/components/places/tests/queries/test_results-as-roots.js create mode 100644 toolkit/components/places/tests/queries/test_results-as-tag-query.js create mode 100644 toolkit/components/places/tests/queries/test_results-as-visit.js create mode 100644 toolkit/components/places/tests/queries/test_searchTerms_includeHidden.js create mode 100644 toolkit/components/places/tests/queries/test_search_tags.js create mode 100644 toolkit/components/places/tests/queries/test_searchterms-bookmarklets.js create mode 100644 toolkit/components/places/tests/queries/test_searchterms-domain.js create mode 100644 toolkit/components/places/tests/queries/test_searchterms-uri.js create mode 100644 toolkit/components/places/tests/queries/test_sort-date-site-grouping.js create mode 100644 toolkit/components/places/tests/queries/test_sorting.js create mode 100644 toolkit/components/places/tests/queries/test_tags.js create mode 100644 toolkit/components/places/tests/queries/test_transitions.js create mode 100644 toolkit/components/places/tests/queries/xpcshell.ini create mode 100644 toolkit/components/places/tests/sync/head_sync.js create mode 100644 toolkit/components/places/tests/sync/mirror_corrupt.sqlite create mode 100644 toolkit/components/places/tests/sync/mirror_v1.sqlite create mode 100644 toolkit/components/places/tests/sync/mirror_v5.sqlite create mode 100644 toolkit/components/places/tests/sync/sync_utils_bookmarks.html create mode 100644 toolkit/components/places/tests/sync/sync_utils_bookmarks.json create mode 100644 toolkit/components/places/tests/sync/test_bookmark_abort_merging.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_chunking.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_corruption.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_deduping.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_deletion.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_haschanges.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_kinds.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_mirror_meta.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_mirror_migration.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_reconcile.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_structure_changes.js create mode 100644 toolkit/components/places/tests/sync/test_bookmark_value_changes.js create mode 100644 toolkit/components/places/tests/sync/test_sync_utils.js create mode 100644 toolkit/components/places/tests/sync/xpcshell.ini create mode 100644 toolkit/components/places/tests/unit/bookmarks.corrupt.html create mode 100644 toolkit/components/places/tests/unit/bookmarks.json create mode 100644 toolkit/components/places/tests/unit/bookmarks.preplaces.html create mode 100644 toolkit/components/places/tests/unit/bookmarks_corrupt.json create mode 100644 toolkit/components/places/tests/unit/bookmarks_html_localized.html create mode 100644 toolkit/components/places/tests/unit/bookmarks_html_singleframe.html create mode 100644 toolkit/components/places/tests/unit/bookmarks_iconuri.json create mode 100644 toolkit/components/places/tests/unit/head_bookmarks.js create mode 100644 toolkit/components/places/tests/unit/missingBuiltIn.sqlite create mode 100644 toolkit/components/places/tests/unit/mobile_bookmarks_folder_import.json create mode 100644 toolkit/components/places/tests/unit/mobile_bookmarks_folder_merge.json create mode 100644 toolkit/components/places/tests/unit/mobile_bookmarks_multiple_folders.json create mode 100644 toolkit/components/places/tests/unit/mobile_bookmarks_root_import.json create mode 100644 toolkit/components/places/tests/unit/mobile_bookmarks_root_merge.json create mode 100644 toolkit/components/places/tests/unit/noRoot.sqlite create mode 100644 toolkit/components/places/tests/unit/places.sparse.sqlite create mode 100644 toolkit/components/places/tests/unit/test_1085291.js create mode 100644 toolkit/components/places/tests/unit/test_1105208.js create mode 100644 toolkit/components/places/tests/unit/test_1105866.js create mode 100644 toolkit/components/places/tests/unit/test_1606731.js create mode 100644 toolkit/components/places/tests/unit/test_331487.js create mode 100644 toolkit/components/places/tests/unit/test_384370.js create mode 100644 toolkit/components/places/tests/unit/test_385397.js create mode 100644 toolkit/components/places/tests/unit/test_399266.js create mode 100644 toolkit/components/places/tests/unit/test_402799.js create mode 100644 toolkit/components/places/tests/unit/test_412132.js create mode 100644 toolkit/components/places/tests/unit/test_415460.js create mode 100644 toolkit/components/places/tests/unit/test_415757.js create mode 100644 toolkit/components/places/tests/unit/test_419792_node_tags_property.js create mode 100644 toolkit/components/places/tests/unit/test_425563.js create mode 100644 toolkit/components/places/tests/unit/test_429505_remove_shortcuts.js create mode 100644 toolkit/components/places/tests/unit/test_433317_query_title_update.js create mode 100644 toolkit/components/places/tests/unit/test_433525_hasChildren_crash.js create mode 100644 toolkit/components/places/tests/unit/test_454977.js create mode 100644 toolkit/components/places/tests/unit/test_463863.js create mode 100644 toolkit/components/places/tests/unit/test_485442_crash_bug_nsNavHistoryQuery_GetUri.js create mode 100644 toolkit/components/places/tests/unit/test_486978_sort_by_date_queries.js create mode 100644 toolkit/components/places/tests/unit/test_536081.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesDBUtils_removeOldCorruptDBs.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesQuery_history.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuidFor.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuids.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesUtils_isRootItem.js create mode 100644 toolkit/components/places/tests/unit/test_PlacesUtils_unwrapNodes_place.js create mode 100644 toolkit/components/places/tests/unit/test_asyncExecuteLegacyQueries.js create mode 100644 toolkit/components/places/tests/unit/test_async_transactions.js create mode 100644 toolkit/components/places/tests/unit/test_autocomplete_match_fallbackTitle.js create mode 100644 toolkit/components/places/tests/unit/test_bookmark-tags-changed_frequency.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html_corrupt.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html_escape_entities.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html_import_tags.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html_localized.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_html_singleframe.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_json.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_json_corrupt.js create mode 100644 toolkit/components/places/tests/unit/test_bookmarks_restore_notification.js create mode 100644 toolkit/components/places/tests/unit/test_broken_folderShortcut_result.js create mode 100644 toolkit/components/places/tests/unit/test_browserhistory.js create mode 100644 toolkit/components/places/tests/unit/test_childlessTags.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_decay.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_observers.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_origins_alternative.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_origins_recalc.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_pages_alternative.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_pages_recalc_alt.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_recalc_triggers.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_recalculator.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_unvisited_bookmark.js create mode 100644 toolkit/components/places/tests/unit/test_frecency_zero_updated.js create mode 100644 toolkit/components/places/tests/unit/test_getChildIndex.js create mode 100644 toolkit/components/places/tests/unit/test_get_query_param_sql_function.js create mode 100644 toolkit/components/places/tests/unit/test_hash.js create mode 100644 toolkit/components/places/tests/unit/test_history.js create mode 100644 toolkit/components/places/tests/unit/test_history_clear.js create mode 100644 toolkit/components/places/tests/unit/test_history_notifications.js create mode 100644 toolkit/components/places/tests/unit/test_history_observer.js create mode 100644 toolkit/components/places/tests/unit/test_history_sidebar.js create mode 100644 toolkit/components/places/tests/unit/test_import_mobile_bookmarks.js create mode 100644 toolkit/components/places/tests/unit/test_isPageInDB.js create mode 100644 toolkit/components/places/tests/unit/test_isURIVisited.js create mode 100644 toolkit/components/places/tests/unit/test_isvisited.js create mode 100644 toolkit/components/places/tests/unit/test_keywords.js create mode 100644 toolkit/components/places/tests/unit/test_lastModified.js create mode 100644 toolkit/components/places/tests/unit/test_markpageas.js create mode 100644 toolkit/components/places/tests/unit/test_metadata.js create mode 100644 toolkit/components/places/tests/unit/test_missing_builtin_folders.js create mode 100644 toolkit/components/places/tests/unit/test_missing_root_folder.js create mode 100644 toolkit/components/places/tests/unit/test_multi_observation.js create mode 100644 toolkit/components/places/tests/unit/test_multi_word_tags.js create mode 100644 toolkit/components/places/tests/unit/test_nested_notifications.js create mode 100644 toolkit/components/places/tests/unit/test_nsINavHistoryViewer.js create mode 100644 toolkit/components/places/tests/unit/test_null_interfaces.js create mode 100644 toolkit/components/places/tests/unit/test_origins.js create mode 100644 toolkit/components/places/tests/unit/test_origins_parsing.js create mode 100644 toolkit/components/places/tests/unit/test_pageGuid_bookmarkGuid.js create mode 100644 toolkit/components/places/tests/unit/test_placeURIs.js create mode 100644 toolkit/components/places/tests/unit/test_promiseBookmarksTree.js create mode 100644 toolkit/components/places/tests/unit/test_resolveNullBookmarkTitles.js create mode 100644 toolkit/components/places/tests/unit/test_result_sort.js create mode 100644 toolkit/components/places/tests/unit/test_resultsAsVisit_details.js create mode 100644 toolkit/components/places/tests/unit/test_sql_function_origin.js create mode 100644 toolkit/components/places/tests/unit/test_sql_guid_functions.js create mode 100644 toolkit/components/places/tests/unit/test_tag_autocomplete_search.js create mode 100644 toolkit/components/places/tests/unit/test_tagging.js create mode 100644 toolkit/components/places/tests/unit/test_telemetry.js create mode 100644 toolkit/components/places/tests/unit/test_update_frecency_after_delete.js create mode 100644 toolkit/components/places/tests/unit/test_utils_backups_create.js create mode 100644 toolkit/components/places/tests/unit/test_utils_backups_hasRecent.js create mode 100644 toolkit/components/places/tests/unit/test_utils_getURLsForContainerNode.js create mode 100644 toolkit/components/places/tests/unit/test_utils_timeConversion.js create mode 100644 toolkit/components/places/tests/unit/test_visitsInDB.js create mode 100644 toolkit/components/places/tests/unit/xpcshell.ini (limited to 'toolkit/components/places') diff --git a/toolkit/components/places/BookmarkHTMLUtils.sys.mjs b/toolkit/components/places/BookmarkHTMLUtils.sys.mjs new file mode 100644 index 0000000000..6cd1099c73 --- /dev/null +++ b/toolkit/components/places/BookmarkHTMLUtils.sys.mjs @@ -0,0 +1,1166 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file works on the old-style "bookmarks.html" file. It includes + * functions to import and export existing bookmarks to this file format. + * + * Format + * ------ + * + * Primary heading := h1 + * Old version used this to set attributes on the bookmarks RDF root, such + * as the last modified date. We only use H1 to check for the attribute + * PLACES_ROOT, which tells us that this hierarchy root is the places root. + * For backwards compatibility, if we don't find this, we assume that the + * hierarchy is rooted at the bookmarks menu. + * Heading := any heading other than h1 + * Old version used this to set attributes on the current container. We only + * care about the content of the heading container, which contains the title + * of the bookmark container. + * Bookmark := a + * HREF is the destination of the bookmark + * FEEDURL is the URI of the RSS feed. This is deprecated and no more + * supported, but some old files may still contain it. + * LAST_CHARSET is stored as an annotation so that the next time we go to + * that page we remember the user's preference. + * ICON will be stored in the favicon service + * ICON_URI is new for places bookmarks.html, it refers to the original + * URI of the favicon so we don't have to make up favicon URLs. + * Text of the container is the name of the bookmark + * Ignored: LAST_VISIT, ID (writing out non-RDF IDs can confuse Firefox 2) + * Bookmark comment := dd + * This affects the previosly added bookmark + * Separator := hr + * Insert a separator into the current container + * The folder hierarchy is defined by
/
    / (the old importing code + * handles all these cases, when we write, use
    ). + * + * Overall design + * -------------- + * + * We need to emulate a recursive parser. A "Bookmark import frame" is created + * corresponding to each folder we encounter. These are arranged in a stack, + * and contain all the state we need to keep track of. + * + * A frame is created when we find a heading, which defines a new container. + * The frame also keeps track of the nesting of
    s, (in well-formed + * bookmarks files, these will have a 1-1 correspondence with frames, but we + * try to be a little more flexible here). When the nesting count decreases + * to 0, then we know a frame is complete and to pop back to the previous + * frame. + * + * Note that a lot of things happen when tags are CLOSED because we need to + * get the text from the content of the tag. For example, link and heading tags + * both require the content (= title) before actually creating it. + */ + +const { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm"); +import { FileUtils } from "resource://gre/modules/FileUtils.sys.mjs"; +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs", +}); + +const Container_Normal = 0; +const Container_Toolbar = 1; +const Container_Menu = 2; +const Container_Unfiled = 3; +const Container_Places = 4; + +const MICROSEC_PER_SEC = 1000000; + +const EXPORT_INDENT = " "; // four spaces + +function base64EncodeString(aString) { + let stream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance( + Ci.nsIStringInputStream + ); + stream.setData(aString, aString.length); + let encoder = Cc["@mozilla.org/scriptablebase64encoder;1"].createInstance( + Ci.nsIScriptableBase64Encoder + ); + return encoder.encodeToString(stream, aString.length); +} + +/** + * Provides HTML escaping for use in HTML attributes and body of the bookmarks + * file, compatible with the old bookmarks system. + */ +function escapeHtmlEntities(aText) { + return (aText || "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +/** + * Provides URL escaping for use in HTML attributes of the bookmarks file, + * compatible with the old bookmarks system. + */ +function escapeUrl(aText) { + return (aText || "").replace(/"/g, "%22"); +} + +function notifyObservers(aTopic, aInitialImport) { + Services.obs.notifyObservers( + null, + aTopic, + aInitialImport ? "html-initial" : "html" + ); +} + +export var BookmarkHTMLUtils = Object.freeze({ + /** + * Loads the current bookmarks hierarchy from a "bookmarks.html" file. + * + * @param aSpec + * String containing the "file:" URI for the existing "bookmarks.html" + * file to be loaded. + * @param [options.replace] + * Whether we should erase existing bookmarks before loading. + * Defaults to `false`. + * @param [options.source] + * The bookmark change source, used to determine the sync status for + * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or + * `IMPORT` otherwise. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromURL( + aSpec, + { + replace: aInitialImport = false, + source: aSource = aInitialImport + ? PlacesUtils.bookmarks.SOURCES.RESTORE + : PlacesUtils.bookmarks.SOURCES.IMPORT, + } = {} + ) { + let bookmarkCount; + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aInitialImport); + try { + let importer = new BookmarkImporter(aInitialImport, aSource); + bookmarkCount = await importer.importFromURL(aSpec); + + notifyObservers( + PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, + aInitialImport + ); + } catch (ex) { + console.error("Failed to import bookmarks from " + aSpec + ": " + ex); + notifyObservers( + PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, + aInitialImport + ); + throw ex; + } + return bookmarkCount; + }, + + /** + * Loads the current bookmarks hierarchy from a "bookmarks.html" file. + * + * @param aFilePath + * OS.File path string of the "bookmarks.html" file to be loaded. + * @param [options.replace] + * Whether we should erase existing bookmarks before loading. + * Defaults to `false`. + * @param [options.source] + * The bookmark change source, used to determine the sync status for + * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or + * `IMPORT` otherwise. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromFile( + aFilePath, + { + replace: aInitialImport = false, + source: aSource = aInitialImport + ? PlacesUtils.bookmarks.SOURCES.RESTORE + : PlacesUtils.bookmarks.SOURCES.IMPORT, + } = {} + ) { + let bookmarkCount; + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aInitialImport); + try { + if (!(await IOUtils.exists(aFilePath))) { + throw new Error( + "Cannot import from nonexisting html file: " + aFilePath + ); + } + let importer = new BookmarkImporter(aInitialImport, aSource); + bookmarkCount = await importer.importFromURL( + PathUtils.toFileURI(aFilePath) + ); + + notifyObservers( + PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, + aInitialImport + ); + } catch (ex) { + console.error("Failed to import bookmarks from " + aFilePath + ": " + ex); + notifyObservers( + PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, + aInitialImport + ); + throw ex; + } + return bookmarkCount; + }, + + /** + * Saves the current bookmarks hierarchy to a "bookmarks.html" file. + * + * @param aFilePath + * OS.File path string for the "bookmarks.html" file to be created. + * + * @return {Promise} + * @resolves To the exported bookmarks count when the file has been created. + * @rejects JavaScript exception. + */ + async exportToFile(aFilePath) { + let [bookmarks, count] = await lazy.PlacesBackups.getBookmarksTree(); + let startTime = Date.now(); + + // Report the time taken to convert the tree to HTML. + let exporter = new BookmarkExporter(bookmarks); + await exporter.exportToFile(aFilePath); + + try { + Services.telemetry + .getHistogramById("PLACES_EXPORT_TOHTML_MS") + .add(Date.now() - startTime); + } catch (ex) { + console.error("Unable to report telemetry."); + } + + return count; + }, + + get defaultPath() { + try { + return Services.prefs.getCharPref("browser.bookmarks.file"); + } catch (ex) {} + return PathUtils.join(PathUtils.profileDir, "bookmarks.html"); + }, +}); + +function Frame(aFolder) { + this.folder = aFolder; + + /** + * How many
    s have been nested. Each frame/container should start + * with a heading, and is then followed by a
    ,
      , or . When + * that list is complete, then it is the end of this container and we need + * to pop back up one level for new items. If we never get an open tag for + * one of these things, we should assume that the container is empty and + * that things we find should be siblings of it. Normally, these
      s won't + * be nested so this will be 0 or 1. + */ + this.containerNesting = 0; + + /** + * when we find a heading tag, it actually affects the title of the NEXT + * container in the list. This stores that heading tag and whether it was + * special. 'consumeHeading' resets this._ + */ + this.lastContainerType = Container_Normal; + + /** + * this contains the text from the last begin tag until now. It is reset + * at every begin tag. We can check it when we see a , or + * to see what the text content of that node should be. + */ + this.previousText = ""; + + /** + * true when we hit a
      , which contains the description for the preceding + * tag. We can't just check for
      like we can for or + * because if there is a sub-folder, it is actually a child of the
      + * because the tag is never explicitly closed. If this is true and we see a + * new open tag, that means to commit the description to the previous + * bookmark. + * + * Additional weirdness happens when the previous
      tag contains a

      : + * this means there is a new folder with the given description, and whose + * children are contained in the following
      list. + * + * This is handled in openContainer(), which commits previous text if + * necessary. + */ + this.inDescription = false; + + /** + * contains the URL of the previous bookmark created. This is used so that + * when we encounter a
      , we know what bookmark to associate the text with. + * This is cleared whenever we hit a

      , so that we know NOT to save this + * with a bookmark, but to keep it until + */ + this.previousLink = null; + + /** + * Contains a reference to the last created bookmark or folder object. + */ + this.previousItem = null; + + /** + * Contains the date-added and last-modified-date of an imported item. + * Used to override the values set by insertBookmark, createFolder, etc. + */ + this.previousDateAdded = null; + this.previousLastModifiedDate = null; +} + +function BookmarkImporter(aInitialImport, aSource) { + this._isImportDefaults = aInitialImport; + this._source = aSource; + + // This root is where we construct the bookmarks tree into, following the format + // of the imported file. + // If we're doing an initial import, the non-menu roots will be created as + // children of this root, so in _getBookmarkTrees we'll split them out. + // If we're not doing an initial import, everything gets imported under the + // bookmark menu folder, so there won't be any need for _getBookmarkTrees to + // do separation. + this._bookmarkTree = { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + guid: PlacesUtils.bookmarks.menuGuid, + children: [], + }; + + this._frames = []; + this._frames.push(new Frame(this._bookmarkTree)); +} + +BookmarkImporter.prototype = { + _safeTrim: function safeTrim(aStr) { + return aStr ? aStr.trim() : aStr; + }, + + get _curFrame() { + return this._frames[this._frames.length - 1]; + }, + + get _previousFrame() { + return this._frames[this._frames.length - 2]; + }, + + /** + * This is called when there is a new folder found. The folder takes the + * name from the previous frame's heading. + */ + _newFrame: function newFrame() { + let frame = this._curFrame; + let containerTitle = frame.previousText; + frame.previousText = ""; + let containerType = frame.lastContainerType; + + let folder = { + children: [], + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }; + + switch (containerType) { + case Container_Normal: + // This can only be a sub-folder so no need to set a guid here. + folder.title = containerTitle; + break; + case Container_Places: + folder.guid = PlacesUtils.bookmarks.rootGuid; + break; + case Container_Menu: + folder.guid = PlacesUtils.bookmarks.menuGuid; + break; + case Container_Unfiled: + folder.guid = PlacesUtils.bookmarks.unfiledGuid; + break; + case Container_Toolbar: + folder.guid = PlacesUtils.bookmarks.toolbarGuid; + break; + default: + // NOT REACHED + throw new Error("Unknown bookmark container type!"); + } + + frame.folder.children.push(folder); + + if (frame.previousDateAdded != null) { + folder.dateAdded = frame.previousDateAdded; + frame.previousDateAdded = null; + } + + if (frame.previousLastModifiedDate != null) { + folder.lastModified = frame.previousLastModifiedDate; + frame.previousLastModifiedDate = null; + } + + if ( + !folder.hasOwnProperty("dateAdded") && + folder.hasOwnProperty("lastModified") + ) { + folder.dateAdded = folder.lastModified; + } + + frame.previousItem = folder; + + this._frames.push(new Frame(folder)); + }, + + /** + * Handles
      as a separator. + * + * @note Separators may have a title in old html files, though Places dropped + * support for them. + * We also don't import ADD_DATE or LAST_MODIFIED for separators because + * pre-Places bookmarks did not support them. + */ + _handleSeparator: function handleSeparator(aElt) { + let frame = this._curFrame; + + let separator = { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }; + frame.folder.children.push(separator); + frame.previousItem = separator; + }, + + /** + * Called for h2,h3,h4,h5,h6. This just stores the correct information in + * the current frame; the actual new frame corresponding to the container + * associated with the heading will be created when the tag has been closed + * and we know the title (we don't know to create a new folder or to merge + * with an existing one until we have the title). + */ + _handleHeadBegin: function handleHeadBegin(aElt) { + let frame = this._curFrame; + + // after a heading, a previous bookmark is not applicable (for example, for + // the descriptions contained in a
      ). Neither is any previous head type + frame.previousLink = null; + frame.lastContainerType = Container_Normal; + + // It is syntactically possible for a heading to appear after another heading + // but before the
      that encloses that folder's contents. This should not + // happen in practice, as the file will contain "
      " sequence for + // empty containers. + // + // Just to be on the safe side, if we encounter + //

      FOO

      + //

      BAR

      + //
      ...content 1...
      + //
      ...content 2...
      + // we'll pop the stack when we find the h3 for BAR, treating that as an + // implicit ending of the FOO container. The output will be FOO and BAR as + // siblings. If there's another
      following (as in "content 2"), those + // items will be treated as further siblings of FOO and BAR + // This special frame popping business, of course, only happens when our + // frame array has more than one element so we can avoid situations where + // we don't have a frame to parse into anymore. + if (frame.containerNesting == 0 && this._frames.length > 1) { + this._frames.pop(); + } + + // We have to check for some attributes to see if this is a "special" + // folder, which will have different creation rules when the end tag is + // processed. + if (aElt.hasAttribute("personal_toolbar_folder")) { + if (this._isImportDefaults) { + frame.lastContainerType = Container_Toolbar; + } + } else if (aElt.hasAttribute("bookmarks_menu")) { + if (this._isImportDefaults) { + frame.lastContainerType = Container_Menu; + } + } else if (aElt.hasAttribute("unfiled_bookmarks_folder")) { + if (this._isImportDefaults) { + frame.lastContainerType = Container_Unfiled; + } + } else if (aElt.hasAttribute("places_root")) { + if (this._isImportDefaults) { + frame.lastContainerType = Container_Places; + } + } else { + let addDate = aElt.getAttribute("add_date"); + if (addDate) { + frame.previousDateAdded = + this._convertImportedDateToInternalDate(addDate); + } + let modDate = aElt.getAttribute("last_modified"); + if (modDate) { + frame.previousLastModifiedDate = + this._convertImportedDateToInternalDate(modDate); + } + } + this._curFrame.previousText = ""; + }, + + /* + * Handles " tags that have no href. + try { + frame.previousLink = Services.io.newURI(href).spec; + } catch (e) { + frame.previousLink = null; + return; + } + + let bookmark = {}; + + // Only set the url for bookmarks. + if (frame.previousLink) { + bookmark.url = frame.previousLink; + } + + if (dateAdded) { + bookmark.dateAdded = this._convertImportedDateToInternalDate(dateAdded); + } + // Save bookmark's last modified date. + if (lastModified) { + bookmark.lastModified = + this._convertImportedDateToInternalDate(lastModified); + } + + if (!dateAdded && lastModified) { + bookmark.dateAdded = bookmark.lastModified; + } + + if (tags) { + bookmark.tags = tags + .split(",") + .filter( + aTag => + !!aTag.length && aTag.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH + ); + + // If we end up with none, then delete the property completely. + if (!bookmark.tags.length) { + delete bookmark.tags; + } + } + + if (lastCharset) { + bookmark.charset = lastCharset; + } + + if (keyword) { + bookmark.keyword = keyword; + } + + if (postData) { + bookmark.postData = postData; + } + + if (icon) { + bookmark.icon = icon; + } + + if (iconUri) { + bookmark.iconUri = iconUri; + } + + // Add bookmark to the tree. + frame.folder.children.push(bookmark); + frame.previousItem = bookmark; + }, + + _handleContainerBegin: function handleContainerBegin() { + this._curFrame.containerNesting++; + }, + + /** + * Our "indent" count has decreased, and when we hit 0 that means that this + * container is complete and we need to pop back to the outer frame. Never + * pop the toplevel frame + */ + _handleContainerEnd: function handleContainerEnd() { + let frame = this._curFrame; + if (frame.containerNesting > 0) { + frame.containerNesting--; + } + if (this._frames.length > 1 && frame.containerNesting == 0) { + this._frames.pop(); + } + }, + + /** + * Creates the new frame for this heading now that we know the name of the + * container (tokens since the heading open tag will have been placed in + * previousText). + */ + _handleHeadEnd: function handleHeadEnd() { + this._newFrame(); + }, + + /** + * Saves the title for the given bookmark. + */ + _handleLinkEnd: function handleLinkEnd() { + let frame = this._curFrame; + frame.previousText = frame.previousText.trim(); + + if (frame.previousItem != null) { + frame.previousItem.title = frame.previousText; + } + + frame.previousText = ""; + }, + + _openContainer: function openContainer(aElt) { + if (aElt.namespaceURI != "http://www.w3.org/1999/xhtml") { + return; + } + switch (aElt.localName) { + case "h2": + case "h3": + case "h4": + case "h5": + case "h6": + this._handleHeadBegin(aElt); + break; + case "a": + this._handleLinkBegin(aElt); + break; + case "dl": + case "ul": + case "menu": + this._handleContainerBegin(); + break; + case "dd": + this._curFrame.inDescription = true; + break; + case "hr": + this._handleSeparator(aElt); + break; + } + }, + + _closeContainer: function closeContainer(aElt) { + let frame = this._curFrame; + + // Although we no longer support importing descriptions, we still need to + // clear any previous text, so that it doesn't get swallowed into other elements. + if (frame.inDescription) { + frame.previousText = ""; + frame.inDescription = false; + } + + if (aElt.namespaceURI != "http://www.w3.org/1999/xhtml") { + return; + } + switch (aElt.localName) { + case "dl": + case "ul": + case "menu": + this._handleContainerEnd(); + break; + case "dt": + break; + case "h1": + // ignore + break; + case "h2": + case "h3": + case "h4": + case "h5": + case "h6": + this._handleHeadEnd(); + break; + case "a": + this._handleLinkEnd(); + break; + default: + break; + } + }, + + _appendText: function appendText(str) { + this._curFrame.previousText += str; + }, + + /** + * Converts a string date in seconds to a date object + */ + _convertImportedDateToInternalDate: + function convertImportedDateToInternalDate(aDate) { + try { + if (aDate && !isNaN(aDate)) { + return new Date(parseInt(aDate) * 1000); // in bookmarks.html this value is in seconds + } + } catch (ex) { + // Do nothing. + } + return new Date(); + }, + + _walkTreeForImport(aDoc) { + if (!aDoc) { + return; + } + + let current = aDoc; + let next; + for (;;) { + switch (current.nodeType) { + case current.ELEMENT_NODE: + this._openContainer(current); + break; + case current.TEXT_NODE: + this._appendText(current.data); + break; + } + if ((next = current.firstChild)) { + current = next; + continue; + } + for (;;) { + if (current.nodeType == current.ELEMENT_NODE) { + this._closeContainer(current); + } + if (current == aDoc) { + return; + } + if ((next = current.nextSibling)) { + current = next; + break; + } + current = current.parentNode; + } + } + }, + + /** + * Returns the bookmark tree(s) from the importer. These are suitable for + * passing to PlacesUtils.bookmarks.insertTree(). + * + * @returns {Array} An array of bookmark trees. + */ + _getBookmarkTrees() { + // If we're not importing defaults, then everything gets imported under the + // Bookmarks menu. + if (!this._isImportDefaults) { + return [this._bookmarkTree]; + } + + // If we are importing defaults, we need to separate out the top-level + // default folders into separate items, for the caller to pass into insertTree. + let bookmarkTrees = [this._bookmarkTree]; + + // The children of this "root" element will contain normal children of the + // bookmark menu as well as the places roots. Hence, we need to filter out + // the separate roots, but keep the children that are relevant to the + // bookmark menu. + this._bookmarkTree.children = this._bookmarkTree.children.filter(child => { + if ( + child.guid && + PlacesUtils.bookmarks.userContentRoots.includes(child.guid) + ) { + bookmarkTrees.push(child); + return false; + } + return true; + }); + + return bookmarkTrees; + }, + + /** + * Imports the bookmarks from the importer into the places database. + * + * @param {BookmarkImporter} importer The importer from which to get the + * bookmark information. + * @returns {number} The number of imported bookmarks, not including + * folders and separators + */ + async _importBookmarks() { + if (this._isImportDefaults) { + await PlacesUtils.bookmarks.eraseEverything(); + } + + let bookmarksTrees = this._getBookmarkTrees(); + let bookmarkCount = 0; + for (let tree of bookmarksTrees) { + if (!tree.children.length) { + continue; + } + + // Give the tree the source. + tree.source = this._source; + let bookmarks = await PlacesUtils.bookmarks.insertTree(tree, { + fixupOrSkipInvalidEntries: true, + }); + // We want to count only bookmarks, not folders or separators + bookmarkCount += bookmarks.filter( + bookmark => bookmark.type == PlacesUtils.bookmarks.TYPE_BOOKMARK + ).length; + insertFaviconsForTree(tree); + } + return bookmarkCount; + }, + + /** + * Imports data into the places database from the supplied url. + * + * @param {String} href The url to import data from. + * @returns {number} The number of imported bookmarks, not including + * folders and separators. + */ + async importFromURL(href) { + let data = await fetchData(href); + + if (this._isImportDefaults && data) { + // Localize default bookmarks. Find rel="localization" links and manually + // localize using them. + let hrefs = []; + let links = data.head.querySelectorAll("link[rel='localization']"); + for (let link of links) { + if (link.getAttribute("href")) { + // We need the text, not the fully qualified URL, so we use `getAttribute`. + hrefs.push(link.getAttribute("href")); + } + } + + if (hrefs.length) { + let domLoc = new DOMLocalization(hrefs); + await domLoc.translateFragment(data.body); + } + } + + this._walkTreeForImport(data); + return this._importBookmarks(); + }, +}; + +function BookmarkExporter(aBookmarksTree) { + // Create a map of the roots. + let rootsMap = new Map(); + for (let child of aBookmarksTree.children) { + if (child.root) { + rootsMap.set(child.root, child); + // Also take the opportunity to get the correctly localised title for the + // root. + child.title = PlacesUtils.bookmarks.getLocalizedTitle(child); + } + } + + // For backwards compatibility reasons the bookmarks menu is the root, while + // the bookmarks toolbar and unfiled bookmarks will be child items. + this._root = rootsMap.get("bookmarksMenuFolder"); + + for (let key of ["toolbarFolder", "unfiledBookmarksFolder"]) { + let root = rootsMap.get(key); + if (root.children && root.children.length) { + if (!this._root.children) { + this._root.children = []; + } + this._root.children.push(root); + } + } +} + +BookmarkExporter.prototype = { + exportToFile: function exportToFile(aFilePath) { + return (async () => { + // Create a file that can be accessed by the current user only. + let out = FileUtils.openAtomicFileOutputStream( + new FileUtils.File(aFilePath) + ); + try { + // We need a buffered output stream for performance. See bug 202477. + let bufferedOut = Cc[ + "@mozilla.org/network/buffered-output-stream;1" + ].createInstance(Ci.nsIBufferedOutputStream); + bufferedOut.init(out, 4096); + try { + // Write bookmarks in UTF-8. + this._converterOut = Cc[ + "@mozilla.org/intl/converter-output-stream;1" + ].createInstance(Ci.nsIConverterOutputStream); + this._converterOut.init(bufferedOut, "utf-8"); + try { + this._writeHeader(); + await this._writeContainer(this._root); + // Retain the target file on success only. + bufferedOut.QueryInterface(Ci.nsISafeOutputStream).finish(); + } finally { + this._converterOut.close(); + this._converterOut = null; + } + } finally { + bufferedOut.close(); + } + } finally { + out.close(); + } + })(); + }, + + _converterOut: null, + + _write(aText) { + this._converterOut.writeString(aText || ""); + }, + + _writeAttribute(aName, aValue) { + this._write(" " + aName + '="' + aValue + '"'); + }, + + _writeLine(aText) { + this._write(aText + "\n"); + }, + + _writeHeader() { + this._writeLine(""); + this._writeLine(""); + this._writeLine( + '' + ); + this._writeLine(``); + this._writeLine("Bookmarks"); + }, + + async _writeContainer(aItem, aIndent = "") { + if (aItem == this._root) { + this._writeLine("

      " + escapeHtmlEntities(this._root.title) + "

      "); + this._writeLine(""); + } else { + this._write(aIndent + "
      " + escapeHtmlEntities(aItem.title) + "

      "); + } + + this._writeLine(aIndent + "

      "); + if (aItem.children) { + await this._writeContainerContents(aItem, aIndent); + } + if (aItem == this._root) { + this._writeLine(aIndent + "

      "); + } else { + this._writeLine(aIndent + "

      "); + } + }, + + async _writeContainerContents(aItem, aIndent) { + let localIndent = aIndent + EXPORT_INDENT; + + for (let child of aItem.children) { + if (child.type == PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER) { + await this._writeContainer(child, localIndent); + } else if (child.type == PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR) { + this._writeSeparator(child, localIndent); + } else { + await this._writeItem(child, localIndent); + } + } + }, + + _writeSeparator(aItem, aIndent) { + this._write(aIndent + ""); + }, + + async _writeItem(aItem, aIndent) { + try { + NetUtil.newURI(aItem.uri); + } catch (ex) { + // If the item URI is invalid, skip the item instead of failing later. + return; + } + + this._write(aIndent + "

      " + escapeHtmlEntities(aItem.title) + ""); + }, + + _writeDateAttributes(aItem) { + if (aItem.dateAdded) { + this._writeAttribute( + "ADD_DATE", + Math.floor(aItem.dateAdded / MICROSEC_PER_SEC) + ); + } + if (aItem.lastModified) { + this._writeAttribute( + "LAST_MODIFIED", + Math.floor(aItem.lastModified / MICROSEC_PER_SEC) + ); + } + }, + + async _writeFaviconAttribute(aItem) { + if (!aItem.iconUri) { + return; + } + let favicon; + try { + favicon = await PlacesUtils.promiseFaviconData(aItem.uri); + } catch (ex) { + console.error("Unexpected Error trying to fetch icon data"); + return; + } + + this._writeAttribute("ICON_URI", escapeUrl(favicon.uri.spec)); + + if (!favicon.uri.schemeIs("chrome") && favicon.dataLen > 0) { + let faviconContents = + "data:image/png;base64," + + base64EncodeString(String.fromCharCode.apply(String, favicon.data)); + this._writeAttribute("ICON", faviconContents); + } + }, +}; + +/** + * Handles inserting favicons into the database for a bookmark node. + * It is assumed the node has already been inserted into the bookmarks + * database. + * + * @param {Object} node The bookmark node for icons to be inserted. + */ +function insertFaviconForNode(node) { + if (node.icon) { + try { + // Create a fake faviconURI to use (FIXME: bug 523932) + let faviconURI = Services.io.newURI("fake-favicon-uri:" + node.url); + PlacesUtils.favicons.replaceFaviconDataFromDataURL( + faviconURI, + node.icon, + 0, + Services.scriptSecurityManager.getSystemPrincipal() + ); + PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(node.url), + faviconURI, + false, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } catch (ex) { + console.error("Failed to import favicon data:" + ex); + } + } + + if (!node.iconUri) { + return; + } + + try { + PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(node.url), + Services.io.newURI(node.iconUri), + false, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } catch (ex) { + console.error("Failed to import favicon URI:" + ex); + } +} + +/** + * Handles inserting favicons into the database for a bookmark tree - a node + * and its children. + * + * It is assumed the nodes have already been inserted into the bookmarks + * database. + * + * @param {Object} nodeTree The bookmark node tree for icons to be inserted. + */ +function insertFaviconsForTree(nodeTree) { + insertFaviconForNode(nodeTree); + + if (nodeTree.children) { + for (let child of nodeTree.children) { + insertFaviconsForTree(child); + } + } +} + +/** + * Handles fetching data from a URL. + * + * @param {String} href The url to fetch data from. + * @return {Promise} Returns a promise that is resolved with the data once + * the fetch is complete, or is rejected if it fails. + */ +function fetchData(href) { + return new Promise((resolve, reject) => { + let xhr = new XMLHttpRequest(); + xhr.onload = () => { + resolve(xhr.responseXML); + }; + xhr.onabort = + xhr.onerror = + xhr.ontimeout = + () => { + reject(new Error("xmlhttprequest failed")); + }; + xhr.open("GET", href); + xhr.responseType = "document"; + xhr.overrideMimeType("text/html"); + xhr.send(); + }); +} diff --git a/toolkit/components/places/BookmarkJSONUtils.sys.mjs b/toolkit/components/places/BookmarkJSONUtils.sys.mjs new file mode 100644 index 0000000000..40f670bb92 --- /dev/null +++ b/toolkit/components/places/BookmarkJSONUtils.sys.mjs @@ -0,0 +1,583 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs", +}); + +// This is used to translate old folder pseudonyms in queries with their newer +// guids. +const OLD_BOOKMARK_QUERY_TRANSLATIONS = { + PLACES_ROOT: PlacesUtils.bookmarks.rootGuid, + BOOKMARKS_MENU: PlacesUtils.bookmarks.menuGuid, + TAGS: PlacesUtils.bookmarks.tagsGuid, + UNFILED_BOOKMARKS: PlacesUtils.bookmarks.unfiledGuid, + TOOLBAR: PlacesUtils.bookmarks.toolbarGuid, + MOBILE_BOOKMARKS: PlacesUtils.bookmarks.mobileGuid, +}; + +/** + * Generates an hash for the given string. + * + * @note The generated hash is returned in base64 form. Mind the fact base64 + * is case-sensitive if you are going to reuse this code. + */ +function generateHash(aString) { + let cryptoHash = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + cryptoHash.init(Ci.nsICryptoHash.MD5); + let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance( + Ci.nsIStringInputStream + ); + stringStream.setUTF8Data(aString); + cryptoHash.updateFromStream(stringStream, -1); + // base64 allows the '/' char, but we can't use it for filenames. + return cryptoHash.finish(true).replace(/\//g, "-"); +} + +export var BookmarkJSONUtils = Object.freeze({ + /** + * Import bookmarks from a url. + * + * @param {string} aSpec + * url of the bookmark data. + * @param {boolean} [options.replace] + * Whether we should erase existing bookmarks before importing. + * @param {PlacesUtils.bookmarks.SOURCES} [options.source] + * The bookmark change source, used to determine the sync status for + * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or + * `IMPORT` otherwise. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromURL( + aSpec, + { + replace: aReplace = false, + source: aSource = aReplace + ? PlacesUtils.bookmarks.SOURCES.RESTORE + : PlacesUtils.bookmarks.SOURCES.IMPORT, + } = {} + ) { + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace); + let bookmarkCount = 0; + try { + let importer = new BookmarkImporter(aReplace, aSource); + bookmarkCount = await importer.importFromURL(aSpec); + + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace); + } catch (ex) { + console.error("Failed to restore bookmarks from " + aSpec + ": " + ex); + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace); + throw ex; + } + return bookmarkCount; + }, + + /** + * Restores bookmarks and tags from a JSON file. + * + * @param aFilePath + * OS.File path string of bookmarks in JSON or JSONlz4 format to be restored. + * @param [options.replace] + * Whether we should erase existing bookmarks before importing. + * @param [options.source] + * The bookmark change source, used to determine the sync status for + * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or + * `IMPORT` otherwise. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromFile( + aFilePath, + { + replace: aReplace = false, + source: aSource = aReplace + ? PlacesUtils.bookmarks.SOURCES.RESTORE + : PlacesUtils.bookmarks.SOURCES.IMPORT, + } = {} + ) { + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace); + let bookmarkCount = 0; + try { + if (!(await IOUtils.exists(aFilePath))) { + throw new Error("Cannot restore from nonexisting json file"); + } + + let importer = new BookmarkImporter(aReplace, aSource); + if (aFilePath.endsWith("jsonlz4")) { + bookmarkCount = await importer.importFromCompressedFile(aFilePath); + } else { + bookmarkCount = await importer.importFromURL( + PathUtils.toFileURI(aFilePath) + ); + } + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace); + } catch (ex) { + console.error( + "Failed to restore bookmarks from " + aFilePath + ": " + ex + ); + notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace); + throw ex; + } + return bookmarkCount; + }, + + /** + * Serializes bookmarks using JSON, and writes to the supplied file path. + * + * @param {path} aFilePath + * Path string for the bookmarks file to be created. + * @param {object} [aOptions] + * @param {string} [failIfHashIs] + * If the generated file would have the same hash defined here, will reject + * with ex.becauseSameHash + * @param {boolean} [compress] + * If true, writes file using lz4 compression + * @return {Promise} + * @resolves once the file has been created, to an object with the + * following properties: + * - count: number of exported bookmarks + * - hash: file hash for contents comparison + * @rejects JavaScript exception. + */ + async exportToFile(aFilePath, aOptions = {}) { + let [bookmarks, count] = await lazy.PlacesBackups.getBookmarksTree(); + let startTime = Date.now(); + let jsonString = JSON.stringify(bookmarks); + // Report the time taken to convert the tree to JSON. + try { + Services.telemetry + .getHistogramById("PLACES_BACKUPS_TOJSON_MS") + .add(Date.now() - startTime); + } catch (ex) { + console.error("Unable to report telemetry."); + } + + let hash = generateHash(jsonString); + + if (hash === aOptions.failIfHashIs) { + let e = new Error("Hash conflict"); + e.becauseSameHash = true; + throw e; + } + + // Do not write to the tmp folder, otherwise if it has a different + // filesystem writeAtomic will fail. Eventual dangling .tmp files should + // be cleaned up by the caller. + await IOUtils.writeUTF8(aFilePath, jsonString, { + compress: aOptions.compress, + tmpPath: PathUtils.join(aFilePath + ".tmp"), + }); + return { count, hash }; + }, +}); + +function BookmarkImporter(aReplace, aSource) { + this._replace = aReplace; + this._source = aSource; +} +BookmarkImporter.prototype = { + /** + * Import bookmarks from a url. + * + * @param {string} aSpec + * url of the bookmark data. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromURL(spec) { + if (!spec.startsWith("chrome://") && !spec.startsWith("file://")) { + throw new Error( + "importFromURL can only be used with chrome:// and file:// URLs" + ); + } + let nodes = await (await fetch(spec)).json(); + + if (!nodes.children || !nodes.children.length) { + return 0; + } + + return this.import(nodes); + }, + + /** + * Import bookmarks from a compressed file. + * + * @param aFilePath + * OS.File path string of the bookmark data. + * + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + importFromCompressedFile: async function BI_importFromCompressedFile( + aFilePath + ) { + // We read as UTF8 rather than JSON, as PlacesUtils.unwrapNodes expects + // a JSON string. + let result = await IOUtils.readUTF8(aFilePath, { decompress: true }); + return this.importFromJSON(result); + }, + + /** + * Import bookmarks from a JSON string. + * + * @param {String} aString JSON string of serialized bookmark data. + * @returns {Promise} The number of imported bookmarks, not including + * folders and separators. + * @resolves When the new bookmarks have been created. + * @rejects JavaScript exception. + */ + async importFromJSON(aString) { + let nodes = PlacesUtils.unwrapNodes( + aString, + PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER + ); + + if (!nodes.length || !nodes[0].children || !nodes[0].children.length) { + return 0; + } + + return this.import(nodes[0]); + }, + + async import(rootNode) { + // Change to rootNode.children as we don't import the root, and also filter + // out any obsolete "tagsFolder" sections. + let nodes = rootNode.children.filter(node => node.root !== "tagsFolder"); + + // If we're replacing, then erase existing bookmarks first. + if (this._replace) { + await PlacesUtils.bookmarks.eraseEverything({ source: this._source }); + } + + let folderIdToGuidMap = {}; + + // Now do some cleanup on the imported nodes so that the various guids + // match what we need for insertTree, and we also have mappings of folders + // so we can repair any searches after inserting the bookmarks (see bug 824502). + for (let node of nodes) { + if (!node.children || !node.children.length) { + continue; + } // Nothing to restore for this root + + // Ensure we set the source correctly. + node.source = this._source; + + // Translate the node for insertTree. + let folders = translateTreeTypes(node); + + folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders); + } + + let bookmarkCount = 0; + // Now we can add the actual nodes to the database. + for (let node of nodes) { + // Drop any nodes without children, we can't insert them. + if (!node.children || !node.children.length) { + continue; + } + + // Drop any roots whose guid we don't recognise - we don't support anything + // apart from the built-in roots. + if (!PlacesUtils.bookmarks.userContentRoots.includes(node.guid)) { + continue; + } + + fixupSearchQueries(node, folderIdToGuidMap); + + let bookmarks = await PlacesUtils.bookmarks.insertTree(node, { + fixupOrSkipInvalidEntries: true, + }); + // We want to count only bookmarks, not folders or separators + bookmarkCount += bookmarks.filter( + bookmark => bookmark.type == PlacesUtils.bookmarks.TYPE_BOOKMARK + ).length; + // Now add any favicons. + try { + insertFaviconsForTree(node); + } catch (ex) { + console.error(`Failed to insert favicons: ${ex}`); + } + } + return bookmarkCount; + }, +}; + +function notifyObservers(topic, replace) { + Services.obs.notifyObservers(null, topic, replace ? "json" : "json-append"); +} + +/** + * Iterates through a node, fixing up any place: URL queries that are found. This + * replaces any old (pre Firefox 62) queries that contain "folder=" parts with + * "parent=". + * + * @param {Object} aNode The node to search. + * @param {Array} aFolderIdMap An array mapping of old folder IDs to new folder GUIDs. + */ +function fixupSearchQueries(aNode, aFolderIdMap) { + if (aNode.url && aNode.url.startsWith("place:")) { + aNode.url = fixupQuery(aNode.url, aFolderIdMap); + } + if (aNode.children) { + for (let child of aNode.children) { + fixupSearchQueries(child, aFolderIdMap); + } + } +} + +/** + * Replaces imported folder ids with their local counterparts in a place: URI. + * + * @param {String} aQueryURL + * A place: URI with folder ids. + * @param {Object} aFolderIdMap + * An array mapping of old folder IDs to new folder GUIDs. + * @return {String} the fixed up URI if all matched. If some matched, it returns + * the URI with only the matching folders included. If none matched + * it returns the input URI unchanged. + */ +function fixupQuery(aQueryURL, aFolderIdMap) { + let invalid = false; + let convert = function (str, existingFolderId) { + let guid; + if ( + Object.keys(OLD_BOOKMARK_QUERY_TRANSLATIONS).includes(existingFolderId) + ) { + guid = OLD_BOOKMARK_QUERY_TRANSLATIONS[existingFolderId]; + } else { + guid = aFolderIdMap[existingFolderId]; + if (!guid) { + invalid = true; + return `invalidOldParentId=${existingFolderId}`; + } + } + return `parent=${guid}`; + }; + + let url = aQueryURL.replace(/folder=([A-Za-z0-9_]+)/g, convert); + if (invalid) { + // One or more of the folders don't exist, cause an empty query so that + // we don't try to display the whole database. + url += "&excludeItems=1"; + } + return url; +} + +/** + * A mapping of root folder names to Guids. To help fixupRootFolderGuid. + */ +const rootToFolderGuidMap = { + placesRoot: PlacesUtils.bookmarks.rootGuid, + bookmarksMenuFolder: PlacesUtils.bookmarks.menuGuid, + unfiledBookmarksFolder: PlacesUtils.bookmarks.unfiledGuid, + toolbarFolder: PlacesUtils.bookmarks.toolbarGuid, + mobileFolder: PlacesUtils.bookmarks.mobileGuid, +}; + +/** + * Updates a bookmark node from the json version to the places GUID. This + * will only change GUIDs for the built-in folders. Other folders will remain + * unchanged. + * + * @param {Object} A bookmark node that is updated with the new GUID if necessary. + */ +function fixupRootFolderGuid(node) { + if (!node.guid && node.root && node.root in rootToFolderGuidMap) { + node.guid = rootToFolderGuidMap[node.root]; + } +} + +/** + * Translates the JSON types for a node and its children into Places compatible + * types. Also handles updating of other parameters e.g. dateAdded and lastModified. + * + * @param {Object} node A node to be updated. If it contains children, they will + * be updated as well. + * @return {Array} An array containing two items: + * - {Object} A map of current folder ids to GUIDS + * - {Array} An array of GUIDs for nodes that contain query URIs + */ +function translateTreeTypes(node) { + let folderIdToGuidMap = {}; + + // Do the uri fixup first, so we can be consistent in this function. + if (node.uri) { + node.url = node.uri; + delete node.uri; + } + + switch (node.type) { + case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: + node.type = PlacesUtils.bookmarks.TYPE_FOLDER; + + // Older type mobile folders have a random guid with an annotation. We need + // to make sure those go into the proper mobile folder. + let isMobileFolder = + node.annos && + node.annos.some(anno => anno.name == PlacesUtils.MOBILE_ROOT_ANNO); + if (isMobileFolder) { + node.guid = PlacesUtils.bookmarks.mobileGuid; + } else { + // In case the Guid is broken, we need to fix it up. + fixupRootFolderGuid(node); + } + + // Record the current id and the guid so that we can update any search + // queries later. + folderIdToGuidMap[node.id] = node.guid; + break; + case PlacesUtils.TYPE_X_MOZ_PLACE: + node.type = PlacesUtils.bookmarks.TYPE_BOOKMARK; + break; + case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: + node.type = PlacesUtils.bookmarks.TYPE_SEPARATOR; + if ("title" in node) { + delete node.title; + } + break; + default: + // No need to throw/reject here, insertTree will remove this node automatically. + console.error(`Unexpected bookmark type ${node.type}`); + break; + } + + if (node.dateAdded) { + node.dateAdded = PlacesUtils.toDate(node.dateAdded); + } + + if (node.lastModified) { + let lastModified = PlacesUtils.toDate(node.lastModified); + // Ensure we get a last modified date that's later or equal to the dateAdded + // so that we don't upset the Bookmarks API. + if (lastModified >= node.dateAdded) { + node.lastModified = lastModified; + } else { + delete node.lastModified; + } + } + + if (node.tags) { + // Separate any tags into an array, and ignore any that are too long. + node.tags = node.tags + .split(",") + .filter( + aTag => + !!aTag.length && aTag.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH + ); + + // If we end up with none, then delete the property completely. + if (!node.tags.length) { + delete node.tags; + } + } + + // Sometimes postData can be null, so delete it to make the validators happy. + if (node.postData == null) { + delete node.postData; + } + + // Now handle any children. + if (!node.children) { + return folderIdToGuidMap; + } + + // First sort the children by index. + node.children = node.children.sort((a, b) => { + return a.index - b.index; + }); + + // Now do any adjustments required for the children. + for (let child of node.children) { + let folders = translateTreeTypes(child); + folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders); + } + + return folderIdToGuidMap; +} + +/** + * Handles inserting favicons into the database for a bookmark node. + * It is assumed the node has already been inserted into the bookmarks + * database. + * + * @param {Object} node The bookmark node for icons to be inserted. + */ +function insertFaviconForNode(node) { + if (node.icon) { + try { + // Create a fake faviconURI to use (FIXME: bug 523932) + let faviconURI = Services.io.newURI("fake-favicon-uri:" + node.url); + PlacesUtils.favicons.replaceFaviconDataFromDataURL( + faviconURI, + node.icon, + 0, + Services.scriptSecurityManager.getSystemPrincipal() + ); + PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(node.url), + faviconURI, + false, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } catch (ex) { + console.error("Failed to import favicon data:" + ex); + } + } + + if (!node.iconUri) { + return; + } + + try { + PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(node.url), + Services.io.newURI(node.iconUri), + false, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } catch (ex) { + console.error("Failed to import favicon URI:" + ex); + } +} + +/** + * Handles inserting favicons into the database for a bookmark tree - a node + * and its children. + * + * It is assumed the nodes have already been inserted into the bookmarks + * database. + * + * @param {Object} nodeTree The bookmark node tree for icons to be inserted. + */ +function insertFaviconsForTree(nodeTree) { + insertFaviconForNode(nodeTree); + + if (nodeTree.children) { + for (let child of nodeTree.children) { + insertFaviconsForTree(child); + } + } +} diff --git a/toolkit/components/places/Bookmarks.sys.mjs b/toolkit/components/places/Bookmarks.sys.mjs new file mode 100644 index 0000000000..59f69bef53 --- /dev/null +++ b/toolkit/components/places/Bookmarks.sys.mjs @@ -0,0 +1,3264 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This module provides an asynchronous API for managing bookmarks. + * + * Bookmarks are organized in a tree structure, and include URLs, folders and + * separators. Multiple bookmarks for the same URL are allowed. + * + * Note that if you are handling bookmarks operations in the UI, you should + * not use this API directly, but rather use PlacesTransactions.jsm, so that + * any operation is undo/redo-able. + * + * Each bookmark-item is represented by an object having the following + * properties: + * + * - guid (string) + * The globally unique identifier of the item. + * - parentGuid (string) + * The globally unique identifier of the folder containing the item. + * This will be an empty string for the Places root folder. + * - index (number) + * The 0-based position of the item in the parent folder. + * - dateAdded (Date) + * The time at which the item was added. + * - lastModified (Date) + * The time at which the item was last modified. + * - type (number) + * The item's type, either TYPE_BOOKMARK, TYPE_FOLDER or TYPE_SEPARATOR. + * + * The following properties are only valid for URLs or folders. + * + * - title (string) + * The item's title, if any. Empty titles and null titles are considered + * the same. Titles longer than DB_TITLE_LENGTH_MAX will be truncated. + * + * The following properties are only valid for URLs: + * + * - url (URL, href or nsIURI) + * The item's URL. Note that while input objects can contains either + * an URL object, an href string, or an nsIURI, output objects will always + * contain an URL object. + * An URL cannot be longer than DB_URL_LENGTH_MAX, methods will throw if a + * longer value is provided. + * + * Each successful operation notifies through the PlacesObservers + * interface. To listen to such notifications you must register using + * PlacesUtils.observers.addListener and PlacesUtils.observers.removeListener + * methods. + * Note that bookmark addition or order changes won't notify bookmark-moved for + * items that have their indexes changed. + * Similarly, lastModified changes not done explicitly (like changing another + * property) won't fire a bookmark-time-changed notification. + * @see PlacesObservers + */ + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "NetUtil", + "resource://gre/modules/NetUtil.jsm" +); +ChromeUtils.defineESModuleGetters(lazy, { + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +const MATCH_ANYWHERE_UNMODIFIED = + Ci.mozIPlacesAutoComplete.MATCH_ANYWHERE_UNMODIFIED; +const BEHAVIOR_BOOKMARK = Ci.mozIPlacesAutoComplete.BEHAVIOR_BOOKMARK; + +export var Bookmarks = Object.freeze({ + /** + * Item's type constants. + * These should stay consistent with nsINavBookmarksService.idl + */ + TYPE_BOOKMARK: 1, + TYPE_FOLDER: 2, + TYPE_SEPARATOR: 3, + + /** + * Sync status constants, stored for each item. + */ + SYNC_STATUS: { + UNKNOWN: Ci.nsINavBookmarksService.SYNC_STATUS_UNKNOWN, + NEW: Ci.nsINavBookmarksService.SYNC_STATUS_NEW, + NORMAL: Ci.nsINavBookmarksService.SYNC_STATUS_NORMAL, + }, + + /** + * Default index used to append a bookmark-item at the end of a folder. + * This should stay consistent with nsINavBookmarksService.idl + */ + DEFAULT_INDEX: -1, + + /** + * Maximum length of a tag. + * Any tag above this length is rejected. + */ + MAX_TAG_LENGTH: 100, + + /** + * Bookmark change source constants, passed as optional properties and + * forwarded to observers. See nsINavBookmarksService.idl for an explanation. + */ + SOURCES: { + DEFAULT: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + SYNC: Ci.nsINavBookmarksService.SOURCE_SYNC, + IMPORT: Ci.nsINavBookmarksService.SOURCE_IMPORT, + SYNC_REPARENT_REMOVED_FOLDER_CHILDREN: + Ci.nsINavBookmarksService.SOURCE_SYNC_REPARENT_REMOVED_FOLDER_CHILDREN, + RESTORE: Ci.nsINavBookmarksService.SOURCE_RESTORE, + RESTORE_ON_STARTUP: Ci.nsINavBookmarksService.SOURCE_RESTORE_ON_STARTUP, + }, + + /** + * Special GUIDs associated with bookmark roots. + * It's guaranteed that the roots will always have these guids. + */ + rootGuid: "root________", + menuGuid: "menu________", + toolbarGuid: "toolbar_____", + unfiledGuid: "unfiled_____", + mobileGuid: "mobile______", + + // With bug 424160, tags will stop being bookmarks, thus this root will + // be removed. Do not rely on this, rather use the tagging service API. + tagsGuid: "tags________", + + /** + * The GUIDs of the user content root folders that we support, for easy access + * as a set. + */ + userContentRoots: [ + "toolbar_____", + "menu________", + "unfiled_____", + "mobile______", + ], + + /** + * GUID associated with bookmarks that haven't been saved to the database yet. + */ + unsavedGuid: "new_________", + + /** + * GUIDs associated with virtual queries that are used for displaying bookmark + * folders in the left pane. + */ + virtualMenuGuid: "menu_______v", + virtualToolbarGuid: "toolbar____v", + virtualUnfiledGuid: "unfiled____v", + virtualMobileGuid: "mobile_____v", + + /** + * Checks if a guid is a virtual root. + * + * @param {String} guid The guid of the item to look for. + * @returns {Boolean} true if guid is a virtual root, false otherwise. + */ + isVirtualRootItem(guid) { + return ( + guid == lazy.PlacesUtils.bookmarks.virtualMenuGuid || + guid == lazy.PlacesUtils.bookmarks.virtualToolbarGuid || + guid == lazy.PlacesUtils.bookmarks.virtualUnfiledGuid || + guid == lazy.PlacesUtils.bookmarks.virtualMobileGuid + ); + }, + + /** + * Returns the title to use on the UI for a bookmark item. Root folders + * in the database don't store fully localised versions of the title. To + * get those this function should be called. + * + * Hence, this function should only be called if a root folder object is + * likely to be displayed to the user. + * + * @param {Object} info An object representing a bookmark-item. + * @returns {String} The correct string. + * @throws {Error} If the guid in PlacesUtils.bookmarks.userContentRoots is + * not supported. + */ + getLocalizedTitle(info) { + if (!lazy.PlacesUtils.bookmarks.userContentRoots.includes(info.guid)) { + return info.title; + } + + switch (info.guid) { + case lazy.PlacesUtils.bookmarks.toolbarGuid: + return lazy.PlacesUtils.getString("BookmarksToolbarFolderTitle"); + case lazy.PlacesUtils.bookmarks.menuGuid: + return lazy.PlacesUtils.getString("BookmarksMenuFolderTitle"); + case lazy.PlacesUtils.bookmarks.unfiledGuid: + return lazy.PlacesUtils.getString("OtherBookmarksFolderTitle"); + case lazy.PlacesUtils.bookmarks.mobileGuid: + return lazy.PlacesUtils.getString("MobileBookmarksFolderTitle"); + default: + throw new Error( + `Unsupported guid ${info.guid} passed to getLocalizedTitle!` + ); + } + }, + + /** + * Inserts a bookmark-item into the bookmarks tree. + * + * For creating a bookmark, the following set of properties is required: + * - type + * - parentGuid + * - url, only for bookmarked URLs + * + * If an index is not specified, it defaults to appending. + * It's also possible to pass a non-existent GUID to force creation of an + * item with the given GUID, but unless you have a very sound reason, such as + * an undo manager implementation or synchronization, don't do that. + * + * Note that any known properties that don't apply to the specific item type + * cause an exception. + * + * @param info + * object representing a bookmark-item. + * + * @return {Promise} resolved when the creation is complete. + * @resolves to an object representing the created bookmark. + * @rejects if it's not possible to create the requested bookmark. + * @throws if the arguments are invalid. + */ + insert(info) { + let now = new Date(); + let addedTime = (info && info.dateAdded) || now; + let modTime = addedTime; + if (addedTime > now) { + modTime = now; + } + let insertInfo = validateBookmarkObject("Bookmarks.jsm: insert", info, { + type: { defaultValue: this.TYPE_BOOKMARK }, + index: { defaultValue: this.DEFAULT_INDEX }, + url: { + requiredIf: b => b.type == this.TYPE_BOOKMARK, + validIf: b => b.type == this.TYPE_BOOKMARK, + }, + parentGuid: { + required: true, + // Inserting into the root folder is not allowed unless it's testing. + validIf: b => + lazy.PlacesUtils.isInAutomation || b.parentGuid != this.rootGuid, + }, + title: { + defaultValue: "", + validIf: b => + b.type == this.TYPE_BOOKMARK || + b.type == this.TYPE_FOLDER || + b.title === "", + }, + dateAdded: { defaultValue: addedTime }, + lastModified: { + defaultValue: modTime, + validIf: b => + b.lastModified >= now || + (b.dateAdded && b.lastModified >= b.dateAdded), + }, + source: { defaultValue: this.SOURCES.DEFAULT }, + }); + + return (async () => { + // Ensure the parent exists. + let parent = await fetchBookmark({ guid: insertInfo.parentGuid }); + if (!parent) { + throw new Error("parentGuid must be valid"); + } + + // Set index in the appending case. + if ( + insertInfo.index == this.DEFAULT_INDEX || + insertInfo.index > parent._childCount + ) { + insertInfo.index = parent._childCount; + } + + let item = await insertBookmark(insertInfo, parent); + + // We need the itemId to notify, though once the switch to guids is + // complete we may stop using it. + let itemId = await lazy.PlacesUtils.promiseItemId(item.guid); + + // Pass tagging information for the observers to skip over these notifications when needed. + let isTagging = parent._parentId == lazy.PlacesUtils.tagsFolderId; + let isTagsFolder = parent._id == lazy.PlacesUtils.tagsFolderId; + let url = ""; + if (item.type == Bookmarks.TYPE_BOOKMARK) { + url = item.url.href; + } + + const notifications = [ + new PlacesBookmarkAddition({ + id: itemId, + url, + itemType: item.type, + parentId: parent._id, + index: item.index, + title: item.title, + dateAdded: item.dateAdded, + guid: item.guid, + parentGuid: item.parentGuid, + source: item.source, + isTagging: isTagging || isTagsFolder, + }), + ]; + + // If it's a tag, notify bookmark-tags-changed event to all bookmarks for this URL. + if (isTagging) { + for (let entry of await fetchBookmarksByURL(item, { + concurrent: true, + })) { + notifications.push( + new PlacesBookmarkTags({ + id: entry._id, + itemType: entry.type, + url, + guid: entry.guid, + parentGuid: entry.parentGuid, + tags: entry._tags, + lastModified: entry.lastModified, + source: item.source, + isTagging: false, + }) + ); + } + } + + PlacesObservers.notifyListeners(notifications); + + // Remove non-enumerable properties. + delete item.source; + return Object.assign({}, item); + })(); + }, + + /** + * Inserts a bookmark-tree into the existing bookmarks tree. + * + * All the specified folders and bookmarks will be inserted as new, even + * if duplicates. There's no merge support at this time. + * + * The input should be of the form: + * { + * guid: "", + * source: "", (optional) + * children: [ + * ... valid bookmark objects. + * ] + * } + * + * Children will be appended to any existing children of the parent + * that is specified. The source specified on the root of the tree + * will be used for all the items inserted. Any indices or custom parentGuids + * set on children will be ignored and overwritten. + * + * @param {Object} tree + * object representing a tree of bookmark items to insert. + * @param {Object} options [optional] + * object with properties representing options. Current options are: + * - fixupOrSkipInvalidEntries: makes the insert more lenient to + * mistakes in the input tree. Properties of an entry that are + * fixable will be corrected, otherwise the entry will be skipped. + * This is particularly convenient for import/restore operations, + * but should not be abused for common inserts, since it may hide + * bugs in the calling code. + * + * @return {Promise} resolved when the creation is complete. + * @resolves to an array of objects representing the created bookmark(s). + * @rejects if it's not possible to create the requested bookmark. + * @throws if the arguments are invalid. + */ + insertTree(tree, options) { + if (!tree || typeof tree != "object") { + throw new Error("Should be provided a valid tree object."); + } + if (!Array.isArray(tree.children) || !tree.children.length) { + throw new Error("Should have a non-zero number of children to insert."); + } + if (!lazy.PlacesUtils.isValidGuid(tree.guid)) { + throw new Error( + `The parent guid is not valid (${tree.guid} ${tree.title}).` + ); + } + if (tree.guid == this.rootGuid) { + throw new Error("Can't insert into the root."); + } + if (tree.guid == this.tagsGuid) { + throw new Error("Can't use insertTree to insert tags."); + } + if ( + tree.hasOwnProperty("source") && + !Object.values(this.SOURCES).includes(tree.source) + ) { + throw new Error("Can't use source value " + tree.source); + } + if (options && typeof options != "object") { + throw new Error("Options should be a valid object"); + } + let fixupOrSkipInvalidEntries = + options && !!options.fixupOrSkipInvalidEntries; + + // Serialize the tree into an array of items to insert into the db. + let insertInfos = []; + let urlsThatMightNeedPlaces = []; + + // We want to use the same 'last added' time for all the entries + // we import (so they won't differ by a few ms based on where + // they are in the tree, and so we don't needlessly construct + // multiple dates). + let fallbackLastAdded = new Date(); + + const { TYPE_BOOKMARK, TYPE_FOLDER, SOURCES } = this; + + // Reuse the 'source' property for all the entries. + let source = tree.source || SOURCES.DEFAULT; + + // This is recursive. + function appendInsertionInfoForInfoArray(infos, indexToUse, parentGuid) { + // We want to keep the index of items that will be inserted into the root + // NULL, and then use a subquery to select the right index, to avoid + // races where other consumers might add items between when we determine + // the index and when we insert. However, the validator does not allow + // NULL values in in the index, so we fake it while validating and then + // correct later. Keep track of whether we're doing this: + let shouldUseNullIndices = false; + if (indexToUse === null) { + shouldUseNullIndices = true; + indexToUse = 0; + } + + // When a folder gets an item added, its last modified date is updated + // to be equal to the date we added the item (if that date is newer). + // Because we're inserting a tree, we keep track of this date for the + // loop, updating it for inserted items as well as from any subfolders + // we insert. + let lastAddedForParent = new Date(0); + for (let info of infos) { + // Ensure to use the same date for dateAdded and lastModified, even if + // dateAdded may be imposed by the caller. + let time = (info && info.dateAdded) || fallbackLastAdded; + let insertInfo = { + guid: { defaultValue: lazy.PlacesUtils.history.makeGuid() }, + type: { defaultValue: TYPE_BOOKMARK }, + url: { + requiredIf: b => b.type == TYPE_BOOKMARK, + validIf: b => b.type == TYPE_BOOKMARK, + }, + parentGuid: { replaceWith: parentGuid }, // Set the correct parent guid. + title: { + defaultValue: "", + validIf: b => + b.type == TYPE_BOOKMARK || + b.type == TYPE_FOLDER || + b.title === "", + }, + dateAdded: { + defaultValue: time, + validIf: b => !b.lastModified || b.dateAdded <= b.lastModified, + }, + lastModified: { + defaultValue: time, + validIf: b => + (!b.dateAdded && b.lastModified >= time) || + (b.dateAdded && b.lastModified >= b.dateAdded), + }, + index: { replaceWith: indexToUse++ }, + source: { replaceWith: source }, + keyword: { validIf: b => b.type == TYPE_BOOKMARK }, + charset: { validIf: b => b.type == TYPE_BOOKMARK }, + postData: { validIf: b => b.type == TYPE_BOOKMARK }, + tags: { validIf: b => b.type == TYPE_BOOKMARK }, + children: { + validIf: b => b.type == TYPE_FOLDER && Array.isArray(b.children), + }, + }; + if (fixupOrSkipInvalidEntries) { + insertInfo.guid.fixup = b => + (b.guid = lazy.PlacesUtils.history.makeGuid()); + insertInfo.dateAdded.fixup = insertInfo.lastModified.fixup = b => + (b.lastModified = b.dateAdded = fallbackLastAdded); + } + try { + insertInfo = validateBookmarkObject( + "Bookmarks.jsm: insertTree", + info, + insertInfo + ); + } catch (ex) { + if (fixupOrSkipInvalidEntries) { + indexToUse--; + continue; + } else { + throw ex; + } + } + + if (shouldUseNullIndices) { + insertInfo.index = null; + } + // Store the URL if this is a bookmark, so we can ensure we create an + // entry in moz_places for it. + if (insertInfo.type == Bookmarks.TYPE_BOOKMARK) { + urlsThatMightNeedPlaces.push(insertInfo.url); + } + + insertInfos.push(insertInfo); + // Process any children. We have to use info.children here rather than + // insertInfo.children because validateBookmarkObject doesn't copy over + // the children ref, as the default bookmark validators object doesn't + // know about children. + if (info.children) { + // start children of this item off at index 0. + let childrenLastAdded = appendInsertionInfoForInfoArray( + info.children, + 0, + insertInfo.guid + ); + if (childrenLastAdded > insertInfo.lastModified) { + insertInfo.lastModified = childrenLastAdded; + } + if (childrenLastAdded > lastAddedForParent) { + lastAddedForParent = childrenLastAdded; + } + } + + // Ensure we track what time to update the parent to. + if (insertInfo.dateAdded > lastAddedForParent) { + lastAddedForParent = insertInfo.dateAdded; + } + } + return lastAddedForParent; + } + + // We want to validate synchronously, but we can't know the index at which + // we're inserting into the parent. We just use NULL instead, + // and the SQL query with which we insert will update it as necessary. + let lastAddedForParent = appendInsertionInfoForInfoArray( + tree.children, + null, + tree.guid + ); + + // appendInsertionInfoForInfoArray will remove invalid items and may leave + // us with nothing to insert, if so, just return early. + if (!insertInfos.length) { + return []; + } + + return (async function () { + let treeParent = await fetchBookmark({ guid: tree.guid }); + if (!treeParent) { + throw new Error("The parent you specified doesn't exist."); + } + + if (treeParent._parentId == lazy.PlacesUtils.tagsFolderId) { + throw new Error("Can't use insertTree to insert tags."); + } + + await insertBookmarkTree( + insertInfos, + source, + treeParent, + urlsThatMightNeedPlaces, + lastAddedForParent + ); + + // Now update the indices of root items in the objects we return. + // These may be wrong if someone else modified the table between + // when we fetched the parent and inserted our items, but the actual + // inserts will have been correct, and we don't want to query the DB + // again if we don't have to. bug 1347230 covers improving this. + let rootIndex = treeParent._childCount; + for (let insertInfo of insertInfos) { + if (insertInfo.parentGuid == tree.guid) { + insertInfo.index += rootIndex++; + } + } + // We need the itemIds to notify, though once the switch to guids is + // complete we may stop using them. + let itemIdMap = await lazy.PlacesUtils.promiseManyItemIds( + insertInfos.map(info => info.guid) + ); + + let notifications = []; + for (let i = 0; i < insertInfos.length; i++) { + let item = insertInfos[i]; + let itemId = itemIdMap.get(item.guid); + // For sub-folders, we need to make sure their children have the correct parent ids. + let parentId; + if (item.parentGuid === treeParent.guid) { + // This is a direct child of the tree parent, so we can use the + // existing parent's id. + parentId = treeParent._id; + } else { + // This is a parent folder that's been updated, so we need to + // use the new item id. + parentId = itemIdMap.get(item.parentGuid); + } + + let url = ""; + if (item.type == Bookmarks.TYPE_BOOKMARK) { + url = URL.isInstance(item.url) ? item.url.href : item.url; + } + + notifications.push( + new PlacesBookmarkAddition({ + id: itemId, + url, + itemType: item.type, + parentId, + index: item.index, + title: item.title, + dateAdded: item.dateAdded, + guid: item.guid, + parentGuid: item.parentGuid, + source: item.source, + isTagging: false, + }) + ); + + try { + await handleBookmarkItemSpecialData(itemId, item); + } catch (ex) { + // This is not critical, regardless the bookmark has been created + // and we should continue notifying the next ones. + console.error( + `An error occured while handling special bookmark data: ${ex}` + ); + } + + // Remove non-enumerable properties. + delete item.source; + + insertInfos[i] = Object.assign({}, item); + } + + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + + return insertInfos; + })(); + }, + + /** + * Updates a bookmark-item. + * + * Only set the properties which should be changed (undefined properties + * won't be taken into account). + * Moreover, the item's type or dateAdded cannot be changed, since they are + * immutable after creation. Trying to change them will reject. + * + * Note that any known properties that don't apply to the specific item type + * cause an exception. + * + * @param info + * object representing a bookmark-item, as defined above. + * + * @return {Promise} resolved when the update is complete. + * @resolves to an object representing the updated bookmark. + * @rejects if it's not possible to update the given bookmark. + * @throws if the arguments are invalid. + */ + update(info) { + // The info object is first validated here to ensure it's consistent, then + // it's compared to the existing item to remove any properties that don't + // need to be updated. + let updateInfo = validateBookmarkObject("Bookmarks.jsm: update", info, { + guid: { required: true }, + index: { + requiredIf: b => b.hasOwnProperty("parentGuid"), + validIf: b => b.index >= 0 || b.index == this.DEFAULT_INDEX, + }, + parentGuid: { validIf: b => b.parentGuid != this.rootGuid }, + source: { defaultValue: this.SOURCES.DEFAULT }, + }); + + // There should be at last one more property in addition to guid and source. + if (Object.keys(updateInfo).length < 3) { + throw new Error("Not enough properties to update"); + } + + return (async () => { + // Ensure the item exists. + let item = await fetchBookmark(updateInfo); + if (!item) { + throw new Error("No bookmarks found for the provided GUID"); + } + if (updateInfo.hasOwnProperty("type") && updateInfo.type != item.type) { + throw new Error("The bookmark type cannot be changed"); + } + + // Remove any property that will stay the same. + removeSameValueProperties(updateInfo, item); + // Check if anything should still be updated. + if (Object.keys(updateInfo).length < 3) { + // Remove non-enumerable properties. + return Object.assign({}, item); + } + const now = new Date(); + let lastModifiedDefault = now; + // In the case where `dateAdded` is specified, but `lastModified` is not, + // we only update `lastModified` if it is older than the new `dateAdded`. + if (!("lastModified" in updateInfo) && "dateAdded" in updateInfo) { + lastModifiedDefault = new Date( + Math.max(item.lastModified, updateInfo.dateAdded) + ); + } + updateInfo = validateBookmarkObject("Bookmarks.jsm: update", updateInfo, { + url: { validIf: () => item.type == this.TYPE_BOOKMARK }, + title: { + validIf: () => + [this.TYPE_BOOKMARK, this.TYPE_FOLDER].includes(item.type), + }, + lastModified: { + defaultValue: lastModifiedDefault, + validIf: b => + b.lastModified >= now || + b.lastModified >= (b.dateAdded || item.dateAdded), + }, + dateAdded: { defaultValue: item.dateAdded }, + }); + + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: update", + async db => { + let parent; + if (updateInfo.hasOwnProperty("parentGuid")) { + if (lazy.PlacesUtils.isRootItem(item.guid)) { + throw new Error("It's not possible to move Places root folders."); + } + if (item.type == this.TYPE_FOLDER) { + // Make sure we are not moving a folder into itself or one of its + // descendants. + let rows = await db.executeCached( + `WITH RECURSIVE + descendants(did) AS ( + VALUES(:id) + UNION ALL + SELECT id FROM moz_bookmarks + JOIN descendants ON parent = did + WHERE type = :type + ) + SELECT guid FROM moz_bookmarks + WHERE id IN descendants + `, + { id: item._id, type: this.TYPE_FOLDER } + ); + if ( + rows + .map(r => r.getResultByName("guid")) + .includes(updateInfo.parentGuid) + ) { + throw new Error( + "Cannot insert a folder into itself or one of its descendants" + ); + } + } + + parent = await fetchBookmark({ guid: updateInfo.parentGuid }); + if (!parent) { + throw new Error("No bookmarks found for the provided parentGuid"); + } + } + + if (updateInfo.hasOwnProperty("index")) { + if (lazy.PlacesUtils.isRootItem(item.guid)) { + throw new Error("It's not possible to move Places root folders."); + } + // If at this point we don't have a parent yet, we are moving into + // the same container. Thus we know it exists. + if (!parent) { + parent = await fetchBookmark({ guid: item.parentGuid }); + } + + if ( + updateInfo.index >= parent._childCount || + updateInfo.index == this.DEFAULT_INDEX + ) { + updateInfo.index = parent._childCount; + + // Fix the index when moving within the same container. + if (parent.guid == item.parentGuid) { + updateInfo.index--; + } + } + } + + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta( + info.source + ); + + let updatedItem = await db.executeTransaction(async function () { + let updatedItem = await updateBookmark( + db, + updateInfo, + item, + item.index, + parent, + syncChangeDelta + ); + if (parent) { + await setAncestorsLastModified( + db, + parent.guid, + updatedItem.lastModified, + syncChangeDelta + ); + } + return updatedItem; + }); + + const notifications = []; + + // For lastModified, we only care about the original input, since we + // should not notify implicit lastModified changes. + if ( + (info.hasOwnProperty("lastModified") && + updateInfo.hasOwnProperty("lastModified") && + item.lastModified != updatedItem.lastModified) || + (info.hasOwnProperty("dateAdded") && + updateInfo.hasOwnProperty("dateAdded") && + item.dateAdded != updatedItem.dateAdded) + ) { + let isTagging = updatedItem.parentGuid == Bookmarks.tagsGuid; + if (!isTagging) { + if (!parent) { + parent = await fetchBookmark({ guid: updatedItem.parentGuid }); + } + isTagging = parent.parentGuid === Bookmarks.tagsGuid; + } + + notifications.push( + new PlacesBookmarkTime({ + id: updatedItem._id, + itemType: updatedItem.type, + url: updatedItem.url?.href, + guid: updatedItem.guid, + parentGuid: updatedItem.parentGuid, + dateAdded: updatedItem.dateAdded, + lastModified: updatedItem.lastModified, + source: updatedItem.source, + isTagging, + }) + ); + } + + if (updateInfo.hasOwnProperty("title")) { + let isTagging = updatedItem.parentGuid == Bookmarks.tagsGuid; + if (!isTagging) { + if (!parent) { + parent = await fetchBookmark({ guid: updatedItem.parentGuid }); + } + isTagging = parent.parentGuid === Bookmarks.tagsGuid; + } + + notifications.push( + new PlacesBookmarkTitle({ + id: updatedItem._id, + itemType: updatedItem.type, + url: updatedItem.url?.href, + guid: updatedItem.guid, + parentGuid: updatedItem.parentGuid, + title: updatedItem.title, + lastModified: updatedItem.lastModified, + source: updatedItem.source, + isTagging, + }) + ); + + // If we're updating a tag, we must notify all the tagged bookmarks + // about the change. + if (isTagging) { + for (let entry of await fetchBookmarksByTags( + { tags: [updatedItem.title] }, + { concurrent: true } + )) { + notifications.push( + new PlacesBookmarkTags({ + id: entry._id, + itemType: entry.type, + url: entry.url, + guid: entry.guid, + parentGuid: entry.parentGuid, + tags: entry._tags, + lastModified: entry.lastModified, + source: updatedItem.source, + isTagging: false, + }) + ); + } + } + } + if (updateInfo.hasOwnProperty("url")) { + await lazy.PlacesUtils.keywords.reassign( + item.url, + updatedItem.url, + updatedItem.source + ); + + let isTagging = updatedItem.parentGuid == Bookmarks.tagsGuid; + if (!isTagging) { + if (!parent) { + parent = await fetchBookmark({ guid: updatedItem.parentGuid }); + } + isTagging = parent.parentGuid === Bookmarks.tagsGuid; + } + + notifications.push( + new PlacesBookmarkUrl({ + id: updatedItem._id, + itemType: updatedItem.type, + url: updatedItem.url.href, + guid: updatedItem.guid, + parentGuid: updatedItem.parentGuid, + source: updatedItem.source, + isTagging, + lastModified: updatedItem.lastModified, + }) + ); + } + // If the item was moved, notify bookmark-moved. + if ( + item.parentGuid != updatedItem.parentGuid || + item.index != updatedItem.index + ) { + notifications.push( + new PlacesBookmarkMoved({ + id: updatedItem._id, + itemType: updatedItem.type, + url: updatedItem.url && updatedItem.url.href, + guid: updatedItem.guid, + parentGuid: updatedItem.parentGuid, + source: updatedItem.source, + index: updatedItem.index, + oldParentGuid: item.parentGuid, + oldIndex: item.index, + isTagging: + updatedItem.parentGuid === Bookmarks.tagsGuid || + parent.parentGuid === Bookmarks.tagsGuid, + }) + ); + } + + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + + // Remove non-enumerable properties. + delete updatedItem.source; + return Object.assign({}, updatedItem); + } + ); + })(); + }, + + /** + * Moves multiple bookmark-items to a specific folder. + * + * If you are only updating/moving a single bookmark, use update() instead. + * + * @param {Array} guids + * An array of GUIDs representing the bookmarks to move. + * @param {String} parentGuid + * Optional, the parent GUID to move the bookmarks to. + * @param {Integer} index + * The index to move the bookmarks to. If this is -1, the bookmarks + * will be appended to the folder. + * @param {Integer} source + * One of the Bookmarks.SOURCES.* options, representing the source of + * this change. + * + * @return {Promise} resolved when the move is complete. + * @resolves to an array of objects representing the moved bookmarks. + * @rejects if it's not possible to move the given bookmark(s). + * @throws if the arguments are invalid. + */ + moveToFolder(guids, parentGuid, index, source) { + if (!Array.isArray(guids) || guids.length < 1) { + throw new Error("guids should be an array of at least one item"); + } + if (!guids.every(guid => lazy.PlacesUtils.isValidGuid(guid))) { + throw new Error("Expected only valid GUIDs to be passed."); + } + if (parentGuid && !lazy.PlacesUtils.isValidGuid(parentGuid)) { + throw new Error("parentGuid should be a valid GUID"); + } + if (parentGuid == lazy.PlacesUtils.bookmarks.rootGuid) { + throw new Error("Cannot move bookmarks into root."); + } + if (typeof index != "number" || index < this.DEFAULT_INDEX) { + throw new Error( + `index should be a number greater than ${this.DEFAULT_INDEX}` + ); + } + + if (!source) { + source = this.SOURCES.DEFAULT; + } + + return (async () => { + let updateInfos = []; + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta(source); + + await lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: moveToFolder", + async db => { + const lastModified = new Date(); + + let targetParentGuid = parentGuid || undefined; + + for (let guid of guids) { + // Ensure the item exists. + let existingItem = await fetchBookmark({ guid }, { db }); + if (!existingItem) { + throw new Error("No bookmarks found for the provided GUID"); + } + + if (parentGuid) { + // We're moving to a different folder. + if (existingItem.type == this.TYPE_FOLDER) { + // Make sure we are not moving a folder into itself or one of its + // descendants. + let rows = await db.executeCached( + `WITH RECURSIVE + descendants(did) AS ( + VALUES(:id) + UNION ALL + SELECT id FROM moz_bookmarks + JOIN descendants ON parent = did + WHERE type = :type + ) + SELECT guid FROM moz_bookmarks + WHERE id IN descendants + `, + { id: existingItem._id, type: this.TYPE_FOLDER } + ); + if ( + rows.map(r => r.getResultByName("guid")).includes(parentGuid) + ) { + throw new Error( + "Cannot insert a folder into itself or one of its descendants" + ); + } + } + } else if (!targetParentGuid) { + targetParentGuid = existingItem.parentGuid; + } else if (existingItem.parentGuid != targetParentGuid) { + throw new Error( + "All bookmarks should be in the same folder if no parent is specified" + ); + } + + updateInfos.push({ existingItem, currIndex: existingItem.index }); + } + + let newParent = await fetchBookmark( + { guid: targetParentGuid }, + { db } + ); + + if (newParent._grandParentId == lazy.PlacesUtils.tagsFolderId) { + throw new Error("Can't move to a tags folder"); + } + + let newParentChildCount = newParent._childCount; + + await db.executeTransaction(async () => { + // Now that we have all the existing items, we can do the actual updates. + for (let i = 0; i < updateInfos.length; i++) { + let info = updateInfos[i]; + if (index != this.DEFAULT_INDEX) { + // If we're dropping on the same folder, then we may need to adjust + // the index to insert at the correct place. + if (info.existingItem.parentGuid == newParent.guid) { + if (index > info.existingItem.index) { + // If we're dragging down, we need to go one lower to insert at + // the real point as moving the element changes the index of + // everything below by 1. + index--; + } else if (index == info.existingItem.index) { + // This isn't moving so we skip it, but copy the data so we have + // an easy way for the notifications to check. + info.updatedItem = { ...info.existingItem }; + continue; + } + } + } + + // Never let the index go higher than the max count of the folder. + if (index == this.DEFAULT_INDEX || index >= newParentChildCount) { + index = newParentChildCount; + + // If this is moving within the same folder, then we need to drop the + // index by one to compensate for "removing" it, then re-inserting. + if (info.existingItem.parentGuid == newParent.guid) { + index--; + } + } + + info.updatedItem = await updateBookmark( + db, + { lastModified, index }, + info.existingItem, + info.currIndex, + newParent, + syncChangeDelta + ); + info.newParent = newParent; + + // For items moving within the same folder, we have to keep track + // of their indexes. Otherwise we run the risk of not correctly + // updating the indexes of other items in the folder. + // This section simulates the database write in moveBookmark, which + // allows us to avoid re-reading the database. + if (info.existingItem.parentGuid == newParent.guid) { + let sign = index < info.currIndex ? 1 : -1; + for (let j = 0; j < updateInfos.length; j++) { + if (j == i) { + continue; + } + if ( + updateInfos[j].currIndex >= + Math.min(info.currIndex, index) && + updateInfos[j].currIndex <= Math.max(info.currIndex, index) + ) { + updateInfos[j].currIndex += sign; + } + } + } + info.currIndex = index; + + // We only bump the parent count if we're moving from a different folder. + if (info.existingItem.parentGuid != newParent.guid) { + newParentChildCount++; + } + index++; + } + + await setAncestorsLastModified( + db, + newParent.guid, + lastModified, + syncChangeDelta + ); + }); + } + ); + + const notifications = []; + + // Updates complete, time to notify everyone. + for (let { updatedItem, existingItem, newParent } of updateInfos) { + // If the item was moved, notify bookmark-moved. + // We use the updatedItem.index here, rather than currIndex, as the views + // need to know where we inserted the item as opposed to where it ended + // up. + if ( + existingItem.parentGuid != updatedItem.parentGuid || + existingItem.index != updatedItem.index + ) { + notifications.push( + new PlacesBookmarkMoved({ + id: updatedItem._id, + itemType: updatedItem.type, + url: existingItem.url, + guid: updatedItem.guid, + parentGuid: updatedItem.parentGuid, + source, + index: updatedItem.index, + oldParentGuid: existingItem.parentGuid, + oldIndex: existingItem.index, + isTagging: + updatedItem.parentGuid === Bookmarks.tagsGuid || + newParent.parentGuid === Bookmarks.tagsGuid, + }) + ); + } + // Remove non-enumerable properties. + delete updatedItem.source; + } + + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + + return updateInfos.map(updateInfo => + Object.assign({}, updateInfo.updatedItem) + ); + })(); + }, + + /** + * Removes one or more bookmark-items. + * + * @param guidOrInfo This may be: + * - The globally unique identifier of the item to remove + * - an object representing the item, as defined above + * - an array of objects representing the items to be removed + * @param {Object} [options={}] + * Additional options that can be passed to the function. + * Currently supports the following properties: + * - preventRemovalOfNonEmptyFolders: Causes an exception to be + * thrown when attempting to remove a folder that is not empty. + * - source: The change source, forwarded to all bookmark observers. + * Defaults to nsINavBookmarksService::SOURCE_DEFAULT. + * + * @return {Promise} + * @resolves when the removal is complete + * @rejects if the provided guid doesn't match any existing bookmark. + * @throws if the arguments are invalid. + */ + remove(guidOrInfo, options = {}) { + let infos = guidOrInfo; + if (!infos) { + throw new Error("Input should be a valid object"); + } + if (!Array.isArray(guidOrInfo)) { + if (typeof guidOrInfo != "object") { + infos = [{ guid: guidOrInfo }]; + } else { + infos = [guidOrInfo]; + } + } + + if (!("source" in options)) { + options.source = Bookmarks.SOURCES.DEFAULT; + } + + let removeInfos = []; + for (let info of infos) { + // Disallow removing the root folders. + if ( + [ + Bookmarks.rootGuid, + Bookmarks.menuGuid, + Bookmarks.toolbarGuid, + Bookmarks.unfiledGuid, + Bookmarks.tagsGuid, + Bookmarks.mobileGuid, + ].includes(info.guid) + ) { + throw new Error("It's not possible to remove Places root folders."); + } + + // Even if we ignore any other unneeded property, we still validate any + // known property to reduce likelihood of hidden bugs. + let removeInfo = validateBookmarkObject("Bookmarks.jsm: remove", info); + removeInfos.push(removeInfo); + } + + return (async function () { + let removeItems = []; + for (let info of removeInfos) { + // We must be able to remove a bookmark even if it has an invalid url. + // In that case the item won't have a url property. + let item = await fetchBookmark(info, { ignoreInvalidURLs: true }); + if (!item) { + throw new Error("No bookmarks found for the provided GUID."); + } + + removeItems.push(item); + } + + await removeBookmarks(removeItems, options); + + // Notify bookmark-removed to listeners. + let notifications = []; + + for (let item of removeItems) { + let isUntagging = item._grandParentId == lazy.PlacesUtils.tagsFolderId; + let url = ""; + if (item.type == Bookmarks.TYPE_BOOKMARK) { + url = item.hasOwnProperty("url") ? item.url.href : null; + } + + notifications.push( + new PlacesBookmarkRemoved({ + id: item._id, + url, + title: item.title, + itemType: item.type, + parentId: item._parentId, + index: item.index, + guid: item.guid, + parentGuid: item.parentGuid, + source: options.source, + isTagging: isUntagging, + isDescendantRemoval: false, + }) + ); + + if (isUntagging) { + for (let entry of await fetchBookmarksByURL(item, { + concurrent: true, + })) { + notifications.push( + new PlacesBookmarkTags({ + id: entry._id, + itemType: entry.type, + url, + guid: entry.guid, + parentGuid: entry.parentGuid, + tags: entry._tags, + lastModified: entry.lastModified, + source: options.source, + isTagging: false, + }) + ); + } + } + } + + PlacesObservers.notifyListeners(notifications); + })(); + }, + + /** + * Removes ALL bookmarks, resetting the bookmarks storage to an empty tree. + * + * Note that roots are preserved, only their children will be removed. + * + * @param {Object} [options={}] + * Additional options. Currently supports the following properties: + * - source: The change source, forwarded to all bookmark observers. + * Defaults to nsINavBookmarksService::SOURCE_DEFAULT. + * + * @return {Promise} resolved when the removal is complete. + * @resolves once the removal is complete. + */ + eraseEverything(options = {}) { + if (!options.source) { + options.source = Bookmarks.SOURCES.DEFAULT; + } + + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: eraseEverything", + async function (db) { + let urls; + await db.executeTransaction(async function () { + urls = await removeFoldersContents( + db, + Bookmarks.userContentRoots, + options + ); + const time = lazy.PlacesUtils.toPRTime(new Date()); + const syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta( + options.source + ); + for (let folderGuid of Bookmarks.userContentRoots) { + await db.executeCached( + `UPDATE moz_bookmarks SET lastModified = :time, + syncChangeCounter = syncChangeCounter + :syncChangeDelta + WHERE id IN (SELECT id FROM moz_bookmarks WHERE guid = :folderGuid ) + `, + { folderGuid, time, syncChangeDelta } + ); + } + + await lazy.PlacesSyncUtils.bookmarks.resetSyncMetadata( + db, + options.source + ); + }); + + if (urls?.length) { + await lazy.PlacesUtils.keywords.eraseEverything(); + } + } + ); + }, + + /** + * Returns a list of recently bookmarked items. + * Only includes actual bookmarks. Excludes folders, separators and queries. + * + * @param {integer} numberOfItems + * The maximum number of bookmark items to return. + * + * @return {Promise} resolved when the listing is complete. + * @resolves to an array of recent bookmark-items. + * @rejects if an error happens while querying. + */ + getRecent(numberOfItems) { + if (numberOfItems === undefined) { + throw new Error("numberOfItems argument is required"); + } + if (typeof numberOfItems !== "number" || numberOfItems % 1 !== 0) { + throw new Error("numberOfItems argument must be an integer"); + } + if (numberOfItems <= 0) { + throw new Error("numberOfItems argument must be greater than zero"); + } + + return fetchRecentBookmarks(numberOfItems); + }, + + /** + * Fetches information about a bookmark-item. + * + * REMARK: any successful call to this method resolves to a single + * bookmark-item (or null), even when multiple bookmarks may exist + * (e.g. fetching by url). If you wish to retrieve all of the + * bookmarks for a given match, use the callback instead. + * + * Input can be either a guid or an object with one, and only one, of these + * filtering properties set: + * - guid + * retrieves the item with the specified guid. + * - parentGuid and index + * retrieves the item by its position. + * - url + * retrieves the most recent bookmark having the given URL. + * To retrieve ALL of the bookmarks for that URL, you must pass in an + * onResult callback, that will be invoked once for each found bookmark. + * - guidPrefix + * retrieves the most recent item with the specified guid prefix. + * To retrieve ALL of the bookmarks for that guid prefix, you must pass + * in an onResult callback, that will be invoked once for each bookmark. + * - tags + * Retrieves the most recent item with all the specified tags. + * The tags are matched in a case-insensitive way. + * To retrieve ALL of the bookmarks having these tags, pass in an + * onResult callback, that will be invoked once for each bookmark. + * Note, there can be multiple bookmarks for the same url, if you need + * unique tagged urls you can filter duplicates by accumulating in a Set. + * + * @param guidOrInfo + * The globally unique identifier of the item to fetch, or an + * object representing it, as defined above. + * @param onResult [optional] + * Callback invoked for each found bookmark. + * @param options [optional] + * an optional object whose properties describe options for the fetch: + * - concurrent: fetches concurrently to any writes, returning results + * faster. On the negative side, it may return stale + * information missing the currently ongoing write. + * - includePath: additionally fetches the path for the bookmarks. + * This is a potentially expensive operation. When + * set to true, the path property is set on results + * containing an array of {title, guid} objects + * ordered from root to leaf. + * + * @return {Promise} resolved when the fetch is complete. + * @resolves to an object representing the found item, as described above, or + * an array of such objects. if no item is found, the returned + * promise is resolved to null. + * @rejects if an error happens while fetching. + * @throws if the arguments are invalid. + * + * @note Any unknown property in the info object is ignored. Known properties + * may be overwritten. + */ + fetch(guidOrInfo, onResult = null, options = {}) { + if (onResult && typeof onResult != "function") { + throw new Error("onResult callback must be a valid function"); + } + let info = guidOrInfo; + if (!info) { + throw new Error("Input should be a valid object"); + } + if (typeof info != "object") { + info = { guid: guidOrInfo }; + } else if (Object.keys(info).length == 1) { + // Just a faster code path. + if ( + !["url", "guid", "parentGuid", "index", "guidPrefix", "tags"].includes( + Object.keys(info)[0] + ) + ) { + throw new Error(`Unexpected number of conditions provided: 0`); + } + } else { + // Only one condition at a time can be provided. + let conditionsCount = [ + v => v.hasOwnProperty("guid"), + v => v.hasOwnProperty("parentGuid") && v.hasOwnProperty("index"), + v => v.hasOwnProperty("url"), + v => v.hasOwnProperty("guidPrefix"), + v => v.hasOwnProperty("tags"), + ].reduce((old, fn) => (old + fn(info)) | 0, 0); + if (conditionsCount != 1) { + throw new Error( + `Unexpected number of conditions provided: ${conditionsCount}` + ); + } + } + + // Create a new options object with just the support properties, because + // we may augment it and hand it down to other methods. + options = { + concurrent: !!options.concurrent, + includePath: !!options.includePath, + }; + + let behavior = {}; + if (info.hasOwnProperty("parentGuid") || info.hasOwnProperty("index")) { + behavior = { + parentGuid: { requiredIf: b => b.hasOwnProperty("index") }, + index: { + validIf: b => + (typeof b.index == "number" && b.index >= 0) || + b.index == this.DEFAULT_INDEX, + }, + }; + } + + // Even if we ignore any other unneeded property, we still validate any + // known property to reduce likelihood of hidden bugs. + let fetchInfo = validateBookmarkObject( + "Bookmarks.jsm: fetch", + info, + behavior + ); + + return (async () => { + let results; + if (fetchInfo.hasOwnProperty("url")) { + results = await fetchBookmarksByURL(fetchInfo, options); + } else if (fetchInfo.hasOwnProperty("guid")) { + results = await fetchBookmark(fetchInfo, options); + } else if (fetchInfo.hasOwnProperty("parentGuid")) { + if (fetchInfo.hasOwnProperty("index")) { + results = await fetchBookmarkByPosition(fetchInfo, options); + } else { + results = await fetchBookmarksByParentGUID(fetchInfo, options); + } + } else if (fetchInfo.hasOwnProperty("guidPrefix")) { + results = await fetchBookmarksByGUIDPrefix(fetchInfo, options); + } else if (fetchInfo.hasOwnProperty("tags")) { + results = await fetchBookmarksByTags(fetchInfo, options); + } + + if (!results) { + return null; + } + + if (!Array.isArray(results)) { + results = [results]; + } + // Remove non-enumerable properties. + results = results.map(r => { + if (r.type == this.TYPE_FOLDER) { + r.childCount = r._childCount; + } + return Object.assign({}, r); + }); + + if (options.includePath) { + for (let result of results) { + let folderPath = await retrieveFullBookmarkPath(result.parentGuid); + if (folderPath) { + result.path = folderPath; + } + } + } + + // Ideally this should handle an incremental behavior and thus be invoked + // while we fetch. Though, the likelihood of 2 or more bookmarks for the + // same match is very low, so it's not worth the added code complication. + if (onResult) { + for (let result of results) { + try { + onResult(result); + } catch (ex) { + console.error(ex); + } + } + } + + return results[0]; + })(); + }, + + /** + * Retrieves an object representation of a bookmark-item, along with all of + * its descendants, if any. + * + * Each node in the tree is an object that extends the item representation + * described above with some additional properties: + * + * - [deprecated] id (number) + * the item's id. Defined only if aOptions.includeItemIds is set. + * - annos (array) + * the item's annotations. This is not set if there are no annotations + * set for the item. + * + * The root object of the tree also has the following properties set: + * - itemsCount (number, not enumerable) + * the number of items, including the root item itself, which are + * represented in the resolved object. + * + * Bookmarked URLs may also have the following properties: + * - tags (string) + * csv string of the bookmark's tags, if any. + * - charset (string) + * the last known charset of the bookmark, if any. + * - iconurl (URL) + * the bookmark's favicon URL, if any. + * + * Folders may also have the following properties: + * - children (array) + * the folder's children information, each of them having the same set of + * properties as above. + * + * @param [optional] guid + * the topmost item to be queried. If it's not passed, the Places + * root folder is queried: that is, you get a representation of the + * entire bookmarks hierarchy. + * @param [optional] options + * Options for customizing the query behavior, in the form of an + * object with any of the following properties: + * - excludeItemsCallback: a function for excluding items, along with + * their descendants. Given an item object (that has everything set + * apart its potential children data), it should return true if the + * item should be excluded. Once an item is excluded, the function + * isn't called for any of its descendants. This isn't called for + * the root item. + * WARNING: since the function may be called for each item, using + * this option can slow down the process significantly if the + * callback does anything that's not relatively trivial. It is + * highly recommended to avoid any synchronous I/O or DB queries. + * - includeItemIds: opt-in to include the deprecated id property. + * Use it if you must. It'll be removed once the switch to guids is + * complete. + * + * @return {Promise} resolved when the fetch is complete. + * @resolves to an object that represents either a single item or a + * bookmarks tree. if guid points to a non-existent item, the + * returned promise is resolved to null. + * @rejects if an error happens while fetching. + * @throws if the arguments are invalid. + */ + // TODO must implement these methods yet: + // PlacesUtils.promiseBookmarksTree() + fetchTree(guid = "", options = {}) { + throw new Error("Not yet implemented"); + }, + + /** + * Fetch all the existing tags, sorted alphabetically. + * @return {Promise} resolves to an array of objects representing tags, when + * fetching is complete. + * Each object looks like { + * name: the name of the tag, + * count: number of bookmarks with this tag + * } + */ + async fetchTags() { + // TODO: Once the tagging API is implemented in Bookmarks.jsm, we can cache + // the list of tags, instead of querying every time. + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT b.title AS name, count(*) AS count + FROM moz_bookmarks b + JOIN moz_bookmarks p ON b.parent = p.id + JOIN moz_bookmarks c ON c.parent = b.id + WHERE p.guid = :tagsGuid + GROUP BY name + ORDER BY name COLLATE nocase ASC + `, + { tagsGuid: this.tagsGuid } + ); + return rows.map(r => ({ + name: r.getResultByName("name"), + count: r.getResultByName("count"), + })); + }, + + /** + * Reorders contents of a folder based on a provided array of GUIDs. + * + * @param parentGuid + * The globally unique identifier of the folder whose contents should + * be reordered. + * @param orderedChildrenGuids + * Ordered array of the children's GUIDs. If this list contains + * non-existing entries they will be ignored. If the list is + * incomplete, and the current child list is already in order with + * respect to orderedChildrenGuids, no change is made. Otherwise, the + * new items are appended but maintain their current order relative to + * eachother. + * @param {Object} [options={}] + * Additional options. Currently supports the following properties: + * - lastModified: The last modified time to use for the folder and + reordered children. Defaults to the current time. + * - source: The change source, forwarded to all bookmark observers. + * Defaults to nsINavBookmarksService::SOURCE_DEFAULT. + * + * @return {Promise} resolved when reordering is complete. + * @rejects if an error happens while reordering. + * @throws if the arguments are invalid. + */ + reorder(parentGuid, orderedChildrenGuids, options = {}) { + let info = { guid: parentGuid }; + info = validateBookmarkObject("Bookmarks.jsm: reorder", info, { + guid: { required: true }, + }); + + if (!Array.isArray(orderedChildrenGuids) || !orderedChildrenGuids.length) { + throw new Error("Must provide a sorted array of children GUIDs."); + } + try { + orderedChildrenGuids.forEach(lazy.PlacesUtils.BOOKMARK_VALIDATORS.guid); + } catch (ex) { + throw new Error("Invalid GUID found in the sorted children array."); + } + + options.source = + "source" in options + ? lazy.PlacesUtils.BOOKMARK_VALIDATORS.source(options.source) + : Bookmarks.SOURCES.DEFAULT; + options.lastModified = + "lastModified" in options + ? lazy.PlacesUtils.BOOKMARK_VALIDATORS.lastModified( + options.lastModified + ) + : new Date(); + + return (async () => { + let parent = await fetchBookmark(info); + if (!parent || parent.type != this.TYPE_FOLDER) { + throw new Error("No folder found for the provided GUID."); + } + if (parent._childCount == 0) { + return; + } + + let sortedChildren = await reorderChildren( + parent, + orderedChildrenGuids, + options + ); + + const notifications = []; + for (let child of sortedChildren) { + notifications.push( + new PlacesBookmarkMoved({ + id: child._id, + itemType: child.type, + url: child.url?.href, + guid: child.guid, + parentGuid: child.parentGuid, + source: options.source, + index: child.index, + oldParentGuid: child.parentGuid, + oldIndex: child._oldIndex, + isTagging: + child.parentGuid === Bookmarks.tagsGuid || + parent.parentGuid === Bookmarks.tagsGuid, + }) + ); + } + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + })(); + }, + + /** + * Searches a list of bookmark-items by a search term, url or title. + * + * IMPORTANT: + * This is intended as an interim API for the web-extensions implementation. + * It will be removed as soon as we have a new querying API. + * + * Note also that this used to exclude separators but no longer does so. + * + * If you just want to search bookmarks by URL, use .fetch() instead. + * + * @param query + * Either a string to use as search term, or an object + * containing any of these keys: query, title or url with the + * corresponding string to match as value. + * The url property can be either a string or an nsIURI. + * + * @return {Promise} resolved when the search is complete. + * @resolves to an array of found bookmark-items. + * @rejects if an error happens while searching. + * @throws if the arguments are invalid. + * + * @note Any unknown property in the query object is ignored. + * Known properties may be overwritten. + */ + search(query) { + if (!query) { + throw new Error("Query object is required"); + } + if (typeof query === "string") { + query = { query }; + } + if (typeof query !== "object") { + throw new Error("Query must be an object or a string"); + } + if (query.query && typeof query.query !== "string") { + throw new Error("Query option must be a string"); + } + if (query.title && typeof query.title !== "string") { + throw new Error("Title option must be a string"); + } + + if (query.url) { + if (typeof query.url === "string" || URL.isInstance(query.url)) { + query.url = new URL(query.url).href; + } else if (query.url instanceof Ci.nsIURI) { + query.url = query.url.spec; + } else { + throw new Error("Url option must be a string or a URL object"); + } + } + + return queryBookmarks(query); + }, +}); + +// Globals. + +// Update implementation. + +/** + * Updates a single bookmark in the database. This should be called from within + * a transaction. + * + * @param {Object} db The pre-existing database connection. + * @param {Object} info A bookmark-item structure with new properties. + * @param {Object} item A bookmark-item structure representing the existing bookmark. + * @param {Integer} oldIndex The index of the item in the old parent. + * @param {Object} newParent The new parent folder (note: this may be the same as) + * the existing folder. + * @param {Integer} syncChangeDelta The change delta to be applied. + */ +async function updateBookmark( + db, + info, + item, + oldIndex, + newParent, + syncChangeDelta +) { + let tuples = new Map(); + tuples.set("lastModified", { + value: lazy.PlacesUtils.toPRTime(info.lastModified), + }); + if (info.hasOwnProperty("title")) { + tuples.set("title", { + value: info.title, + fragment: `title = NULLIF(:title, '')`, + }); + } + if (info.hasOwnProperty("dateAdded")) { + tuples.set("dateAdded", { + value: lazy.PlacesUtils.toPRTime(info.dateAdded), + }); + } + + if (info.hasOwnProperty("url")) { + // Ensure a page exists in moz_places for this URL. + await lazy.PlacesUtils.maybeInsertPlace(db, info.url); + // Update tuples for the update query. + tuples.set("url", { + value: info.url.href, + fragment: + "fk = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url)", + }); + } + + let newIndex = info.hasOwnProperty("index") ? info.index : item.index; + if (newParent) { + // For simplicity, update the index regardless. + tuples.set("position", { value: newIndex }); + + // For moving within the same parent, we've already updated the indexes. + if (newParent.guid == item.parentGuid) { + // Moving inside the original container. + // When moving "up", add 1 to each index in the interval. + // Otherwise when moving down, we subtract 1. + // Only the parent needs a sync change, which is handled in + // `setAncestorsLastModified`. + await db.executeCached( + `UPDATE moz_bookmarks + SET position = CASE WHEN :newIndex < :currIndex + THEN position + 1 + ELSE position - 1 + END + WHERE parent = :newParentId + AND position BETWEEN :lowIndex AND :highIndex + `, + { + newIndex, + currIndex: oldIndex, + newParentId: newParent._id, + lowIndex: Math.min(oldIndex, newIndex), + highIndex: Math.max(oldIndex, newIndex), + } + ); + } else { + // Moving across different containers. In this case, both parents and + // the child need sync changes. `setAncestorsLastModified`, below and in + // `update` and `moveToFolder`, handles the parents. The `needsSyncChange` + // check below handles the child. + tuples.set("parent", { value: newParent._id }); + await db.executeCached( + `UPDATE moz_bookmarks SET position = position - 1 + WHERE parent = :oldParentId + AND position >= :oldIndex + `, + { oldParentId: item._parentId, oldIndex } + ); + await db.executeCached( + `UPDATE moz_bookmarks SET position = position + 1 + WHERE parent = :newParentId + AND position >= :newIndex + `, + { newParentId: newParent._id, newIndex } + ); + + await setAncestorsLastModified( + db, + item.parentGuid, + info.lastModified, + syncChangeDelta + ); + } + } + + if (syncChangeDelta) { + // Sync stores child indices in the parent's record, so we only bump the + // item's counter if we're updating at least one more property in + // addition to the index, last modified time, and dateAdded. + let sizeThreshold = 1; + if (newIndex != oldIndex) { + ++sizeThreshold; + } + if (tuples.has("dateAdded")) { + ++sizeThreshold; + } + let needsSyncChange = tuples.size > sizeThreshold; + if (needsSyncChange) { + tuples.set("syncChangeDelta", { + value: syncChangeDelta, + fragment: "syncChangeCounter = syncChangeCounter + :syncChangeDelta", + }); + } + } + + let isTagging = item._grandParentId == lazy.PlacesUtils.tagsFolderId; + if (isTagging) { + // If we're updating a tag entry, bump the sync change counter for + // bookmarks with the tagged URL. + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + item.url, + syncChangeDelta + ); + if (info.hasOwnProperty("url")) { + // Changing the URL of a tag entry is equivalent to untagging the + // old URL and tagging the new one, so we bump the change counter + // for the new URL here. + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + info.url, + syncChangeDelta + ); + } + } + + let isChangingTagFolder = item._parentId == lazy.PlacesUtils.tagsFolderId; + if (isChangingTagFolder && syncChangeDelta) { + // If we're updating a tag folder (for example, changing a tag's title), + // bump the change counter for all tagged bookmarks. + await db.executeCached( + ` + UPDATE moz_bookmarks SET + syncChangeCounter = syncChangeCounter + :syncChangeDelta + WHERE type = :type AND + fk = (SELECT fk FROM moz_bookmarks WHERE parent = :parent) + `, + { syncChangeDelta, type: Bookmarks.TYPE_BOOKMARK, parent: item._id } + ); + } + + await db.executeCached( + `UPDATE moz_bookmarks + SET ${Array.from(tuples.keys()) + .map(v => tuples.get(v).fragment || `${v} = :${v}`) + .join(", ")} + WHERE guid = :guid + `, + Object.assign( + { guid: item.guid }, + [...tuples.entries()].reduce((p, c) => { + p[c[0]] = c[1].value; + return p; + }, {}) + ) + ); + + if (newParent) { + if (newParent.guid == item.parentGuid) { + // Mark all affected separators as changed + // Also bumps the change counter if the item itself is a separator + const startIndex = Math.min(newIndex, oldIndex); + await adjustSeparatorsSyncCounter( + db, + newParent._id, + startIndex, + syncChangeDelta + ); + } else { + // Mark all affected separators as changed + await adjustSeparatorsSyncCounter( + db, + item._parentId, + oldIndex, + syncChangeDelta + ); + await adjustSeparatorsSyncCounter( + db, + newParent._id, + newIndex, + syncChangeDelta + ); + } + } + + // If the parent changed, update related non-enumerable properties. + let additionalParentInfo = {}; + if (newParent) { + additionalParentInfo.parentGuid = newParent.guid; + Object.defineProperty(additionalParentInfo, "_parentId", { + value: newParent._id, + enumerable: false, + }); + Object.defineProperty(additionalParentInfo, "_grandParentId", { + value: newParent._parentId, + enumerable: false, + }); + } + + return mergeIntoNewObject(item, info, additionalParentInfo); +} + +// Insert implementation. + +function insertBookmark(item, parent) { + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: insertBookmark", + async function (db) { + // If a guid was not provided, generate one, so we won't need to fetch the + // bookmark just after having created it. + let hasExistingGuid = item.hasOwnProperty("guid"); + if (!hasExistingGuid) { + item.guid = lazy.PlacesUtils.history.makeGuid(); + } + + let isTagging = parent._parentId == lazy.PlacesUtils.tagsFolderId; + + await db.executeTransaction(async function transaction() { + if (item.type == Bookmarks.TYPE_BOOKMARK) { + // Ensure a page exists in moz_places for this URL. + // The IGNORE conflict can trigger on `guid`. + await lazy.PlacesUtils.maybeInsertPlace(db, item.url); + } + + // Adjust indices. + await db.executeCached( + `UPDATE moz_bookmarks SET position = position + 1 + WHERE parent = :parent + AND position >= :index + `, + { parent: parent._id, index: item.index } + ); + + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta(item.source); + let syncStatus = + lazy.PlacesSyncUtils.bookmarks.determineInitialSyncStatus( + item.source + ); + + // Insert the bookmark into the database. + await db.executeCached( + `INSERT INTO moz_bookmarks (fk, type, parent, position, title, + dateAdded, lastModified, guid, + syncChangeCounter, syncStatus) + VALUES (CASE WHEN :url ISNULL THEN NULL ELSE (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url) END, + :type, :parent, :index, NULLIF(:title, ''), :date_added, + :last_modified, :guid, :syncChangeCounter, :syncStatus) + `, + { + url: item.hasOwnProperty("url") ? item.url.href : null, + type: item.type, + parent: parent._id, + index: item.index, + title: item.title, + date_added: lazy.PlacesUtils.toPRTime(item.dateAdded), + last_modified: lazy.PlacesUtils.toPRTime(item.lastModified), + guid: item.guid, + syncChangeCounter: syncChangeDelta, + syncStatus, + } + ); + + // Mark all affected separators as changed + await adjustSeparatorsSyncCounter( + db, + parent._id, + item.index + 1, + syncChangeDelta + ); + + if (hasExistingGuid) { + // Remove stale tombstones if we're reinserting an item. + await db.executeCached( + `DELETE FROM moz_bookmarks_deleted WHERE guid = :guid`, + { guid: item.guid } + ); + } + + if (isTagging) { + // New tag entry; bump the change counter for bookmarks with the + // tagged URL. + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + item.url, + syncChangeDelta + ); + } + + await setAncestorsLastModified( + db, + item.parentGuid, + item.dateAdded, + syncChangeDelta + ); + }); + + return item; + } + ); +} + +function insertBookmarkTree(items, source, parent, urls, lastAddedForParent) { + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: insertBookmarkTree", + async function (db) { + await db.executeTransaction(async function transaction() { + await lazy.PlacesUtils.maybeInsertManyPlaces(db, urls); + + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta(source); + let syncStatus = + lazy.PlacesSyncUtils.bookmarks.determineInitialSyncStatus(source); + + let rootId = parent._id; + + items = items.map(item => ({ + url: item.url && item.url.href, + type: item.type, + parentGuid: item.parentGuid, + index: item.index, + title: item.title, + date_added: lazy.PlacesUtils.toPRTime(item.dateAdded), + last_modified: lazy.PlacesUtils.toPRTime(item.lastModified), + guid: item.guid, + syncChangeCounter: syncChangeDelta, + syncStatus, + rootId, + })); + await db.executeCached( + `INSERT INTO moz_bookmarks (fk, type, parent, position, title, + dateAdded, lastModified, guid, + syncChangeCounter, syncStatus) + VALUES (CASE WHEN :url ISNULL THEN NULL ELSE (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url) END, :type, + (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid), + IFNULL(:index, (SELECT count(*) FROM moz_bookmarks WHERE parent = :rootId)), + NULLIF(:title, ''), :date_added, :last_modified, :guid, + :syncChangeCounter, :syncStatus)`, + items + ); + + // Remove stale tombstones for new items. + for (let chunk of lazy.PlacesUtils.chunkArray( + items, + db.variableLimit + )) { + await db.executeCached( + `DELETE FROM moz_bookmarks_deleted + WHERE guid IN (${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk.map(item => item.guid) + ); + } + + await setAncestorsLastModified( + db, + parent.guid, + lastAddedForParent, + syncChangeDelta + ); + }); + + return items; + } + ); +} + +/** + * Handles special data on a bookmark, e.g. annotations, keywords, tags, charsets, + * inserting the data into the appropriate place. + * + * @param {Integer} itemId The ID of the item within the bookmarks database. + * @param {Object} item The bookmark item with possible special data to be inserted. + */ +async function handleBookmarkItemSpecialData(itemId, item) { + if ("keyword" in item && item.keyword) { + try { + await lazy.PlacesUtils.keywords.insert({ + keyword: item.keyword, + url: item.url, + postData: item.postData, + source: item.source, + }); + } catch (ex) { + console.error( + `Failed to insert keyword "${item.keyword} for ${item.url}": ${ex}` + ); + } + } + if ("tags" in item) { + try { + lazy.PlacesUtils.tagging.tagURI( + lazy.NetUtil.newURI(item.url), + item.tags, + item.source + ); + } catch (ex) { + // Invalid tag child, skip it. + console.error( + `Unable to set tags "${item.tags.join(", ")}" for ${item.url}: ${ex}` + ); + } + } + if ("charset" in item && item.charset) { + try { + // UTF-8 is the default. If we are passed the value then set it to null, + // to ensure any charset is removed from the database. + let charset = item.charset; + if (item.charset.toLowerCase() == "utf-8") { + charset = null; + } + + await lazy.PlacesUtils.history.update({ + url: item.url, + annotations: new Map([[lazy.PlacesUtils.CHARSET_ANNO, charset]]), + }); + } catch (ex) { + console.error( + `Failed to set charset "${item.charset}" for ${item.url}: ${ex}` + ); + } + } +} + +// Query implementation. + +async function queryBookmarks(info) { + let queryParams = { + tags_folder: lazy.PlacesUtils.tagsFolderId, + }; + // We're searching for bookmarks, so exclude tags. + let queryString = "WHERE b.parent <> :tags_folder"; + queryString += " AND p.parent <> :tags_folder"; + + if (info.title) { + queryString += " AND b.title = :title"; + queryParams.title = info.title; + } + + if (info.url) { + queryString += " AND h.url_hash = hash(:url) AND h.url = :url"; + queryParams.url = info.url; + } + + if (info.query) { + queryString += + " AND AUTOCOMPLETE_MATCH(:query, h.url, b.title, NULL, NULL, 1, 1, NULL, :matchBehavior, :searchBehavior, NULL) "; + queryParams.query = info.query; + queryParams.matchBehavior = MATCH_ANYWHERE_UNMODIFIED; + queryParams.searchBehavior = BEHAVIOR_BOOKMARK; + } + + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: queryBookmarks", + async function (db) { + // _id, _childCount, _grandParentId and _parentId fields + // are required to be in the result by the converting function + // hence setting them to NULL + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, + IFNULL(b.title, '') AS title, h.url AS url, b.parent, p.parent, + NULL AS _id, + NULL AS _childCount, + NULL AS _grandParentId, + NULL AS _parentId, + NULL AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + ${queryString} + `, + queryParams + ); + + return rowsToItemsArray(rows); + } + ); +} + +/** + * Internal fetch implementation. + * @param {object} info + * The bookmark item to remove. + * @param {object} options + * An options object supporting the following properties: + * @param {object} [options.concurrent] + * Whether to use the concurrent read-only connection. + * @param {object} [options.db] + * A specific connection to be used. + * @param {object} [options.ignoreInvalidURLs] + * Whether invalid URLs should be ignored or throw an exception. + * + */ +async function fetchBookmark(info, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + (SELECT count(*) FROM moz_bookmarks WHERE parent = b.id) AS _childCount, + p.parent AS _grandParentId, b.syncStatus AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + WHERE b.guid = :guid + `, + { guid: info.guid } + ); + + return rows.length + ? rowsToItemsArray(rows, !!options.ignoreInvalidURLs)[0] + : null; + }; + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + if (options.db) { + return query(options.db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmark", + query + ); +} + +async function fetchBookmarkByPosition(info, options = {}) { + let query = async function (db) { + let index = info.index == Bookmarks.DEFAULT_INDEX ? null : info.index; + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + (SELECT count(*) FROM moz_bookmarks WHERE parent = b.id) AS _childCount, + p.parent AS _grandParentId, b.syncStatus AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + WHERE p.guid = :parentGuid + AND b.position = IFNULL(:index, (SELECT count(*) - 1 + FROM moz_bookmarks + WHERE parent = p.id)) + `, + { parentGuid: info.parentGuid, index } + ); + + return rows.length ? rowsToItemsArray(rows)[0] : null; + }; + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmarkByPosition", + query + ); +} + +async function fetchBookmarksByTags(info, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + NULL AS _childCount, + p.parent AS _grandParentId, b.syncStatus AS _syncStatus, + (SELECT group_concat(pp.title) + FROM moz_bookmarks bb + JOIN moz_bookmarks pp ON bb.parent = pp.id + JOIN moz_bookmarks gg ON pp.parent = gg.id AND gg.guid = ? + WHERE bb.fk = h.id) AS _tags + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + JOIN moz_bookmarks g ON g.id = p.parent + JOIN moz_places h ON h.id = b.fk + WHERE g.guid <> ? AND b.fk IN ( + SELECT b2.fk FROM moz_bookmarks b2 + JOIN moz_bookmarks p2 ON p2.id = b2.parent + JOIN moz_bookmarks g2 ON g2.id = p2.parent + WHERE g2.guid = ? + AND lower(p2.title) IN ( + ${new Array(info.tags.length).fill("?").join(",")} + ) + GROUP BY b2.fk HAVING count(*) = ${info.tags.length} + ) + ORDER BY b.lastModified DESC + `, + [Bookmarks.tagsGuid, Bookmarks.tagsGuid, Bookmarks.tagsGuid].concat( + info.tags.map(t => t.toLowerCase()) + ) + ); + + return rows.length ? rowsToItemsArray(rows) : null; + }; + + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmarksByTags", + query + ); +} + +async function fetchBookmarksByGUIDPrefix(info, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + (SELECT count(*) FROM moz_bookmarks WHERE parent = b.id) AS _childCount, + p.parent AS _grandParentId, b.syncStatus AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + WHERE b.guid LIKE :guidPrefix + ORDER BY b.lastModified DESC + `, + { guidPrefix: info.guidPrefix + "%" } + ); + + return rows.length ? rowsToItemsArray(rows) : null; + }; + + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmarksByGUIDPrefix", + query + ); +} + +async function fetchBookmarksByURL(info, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `/* do not warn (bug no): not worth to add an index */ + SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + NULL AS _childCount, /* Unused for now */ + p.parent AS _grandParentId, b.syncStatus AS _syncStatus, + (SELECT group_concat(pp.title) + FROM moz_bookmarks bb + JOIN moz_bookmarks pp ON bb.parent = pp.id + JOIN moz_bookmarks gg ON pp.parent = gg.id AND gg.guid = :tagsGuid + WHERE bb.fk = h.id) AS _tags + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + JOIN moz_places h ON h.id = b.fk + WHERE h.url_hash = hash(:url) AND h.url = :url + AND _grandParentId <> :tagsFolderId + ORDER BY b.lastModified DESC + `, + { + url: info.url.href, + tagsFolderId: lazy.PlacesUtils.tagsFolderId, + tagsGuid: Bookmarks.tagsGuid, + } + ); + + return rows.length ? rowsToItemsArray(rows) : null; + }; + + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmarksByURL", + query + ); +} + +async function fetchBookmarksByParentGUID(info, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, + NULL AS _id, + NULL AS _parentId, + (SELECT count(*) FROM moz_bookmarks WHERE parent = b.id) AS _childCount, + NULL AS _grandParentId, + NULL AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + WHERE p.guid = :parentGuid + ORDER BY b.position ASC + `, + { parentGuid: info.parentGuid } + ); + + return rows.length ? rowsToItemsArray(rows) : null; + }; + + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchBookmarksByParentGUID", + query + ); +} + +function fetchRecentBookmarks(numberOfItems) { + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: fetchRecentBookmarks", + async function (db) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, + IFNULL(b.title, '') AS title, h.url AS url, NULL AS _id, + NULL AS _parentId, NULL AS _childCount, NULL AS _grandParentId, + NULL AS _syncStatus + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + JOIN moz_places h ON h.id = b.fk + WHERE p.parent <> :tagsFolderId + AND b.type = :type + AND url_hash NOT BETWEEN hash("place", "prefix_lo") + AND hash("place", "prefix_hi") + ORDER BY b.dateAdded DESC, b.ROWID DESC + LIMIT :numberOfItems + `, + { + tagsFolderId: lazy.PlacesUtils.tagsFolderId, + type: Bookmarks.TYPE_BOOKMARK, + numberOfItems, + } + ); + + return rows.length ? rowsToItemsArray(rows) : []; + } + ); +} + +async function fetchBookmarksByParent(db, info) { + let rows = await db.executeCached( + `SELECT b.guid, IFNULL(p.guid, '') AS parentGuid, b.position AS 'index', + b.dateAdded, b.lastModified, b.type, IFNULL(b.title, '') AS title, + h.url AS url, b.id AS _id, b.parent AS _parentId, + (SELECT count(*) FROM moz_bookmarks WHERE parent = b.id) AS _childCount, + p.parent AS _grandParentId, b.syncStatus AS _syncStatus + FROM moz_bookmarks b + LEFT JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON h.id = b.fk + WHERE p.guid = :parentGuid + ORDER BY b.position ASC + `, + { parentGuid: info.parentGuid } + ); + + return rowsToItemsArray(rows); +} + +// Remove implementation. + +function removeBookmarks(items, options) { + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: removeBookmarks", + async function (db) { + let urls = []; + + await db.executeTransaction(async function transaction() { + // We use a map for its de-duplication properties. + let parents = new Map(); + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta( + options.source + ); + + for (let item of items) { + parents.set(item.parentGuid, item._parentId); + + // If it's a folder, remove its contents first. + if (item.type == Bookmarks.TYPE_FOLDER) { + if ( + options.preventRemovalOfNonEmptyFolders && + item._childCount > 0 + ) { + throw new Error("Cannot remove a non-empty folder."); + } + urls = urls.concat( + await removeFoldersContents(db, [item.guid], options) + ); + } + } + + for (let chunk of lazy.PlacesUtils.chunkArray( + items, + db.variableLimit + )) { + // Remove the bookmarks. + await db.executeCached( + `DELETE FROM moz_bookmarks + WHERE guid IN (${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk.map(item => item.guid) + ); + } + + for (let [parentGuid, parentId] of parents.entries()) { + // Now recalculate the positions. + await db.executeCached( + `WITH positions(id, pos, seq) AS ( + SELECT id, position AS pos, + (row_number() OVER (ORDER BY position)) - 1 AS seq + FROM moz_bookmarks + WHERE parent = :parentId + ) + UPDATE moz_bookmarks + SET position = (SELECT seq FROM positions WHERE positions.id = moz_bookmarks.id) + WHERE id IN (SELECT id FROM positions WHERE seq <> pos) + `, + { parentId } + ); + + // Mark this parent as changed. + await setAncestorsLastModified( + db, + parentGuid, + new Date(), + syncChangeDelta + ); + } + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + // For the notifications, we may need to adjust indexes if there are more + // than one of the same item in the folder. This makes sure that we notify + // the index of the item when it was removed, rather than the original index. + for (let j = i + 1; j < items.length; j++) { + if ( + items[j]._parentId == item._parentId && + items[j].index > item.index + ) { + items[j].index--; + } + } + if (item._grandParentId == lazy.PlacesUtils.tagsFolderId) { + // If we're removing a tag entry, increment the change counter for all + // bookmarks with the tagged URL. + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + item.url, + syncChangeDelta + ); + } + + await adjustSeparatorsSyncCounter( + db, + item._parentId, + item.index, + syncChangeDelta + ); + } + + // Write tombstones for the removed items. + await insertTombstones(db, items, syncChangeDelta); + }); + + urls = urls.concat(items.map(item => item.url).filter(item => item)); + if (urls.length) { + await lazy.PlacesUtils.keywords.removeFromURLsIfNotBookmarked(urls); + } + } + ); +} + +// Reorder implementation. + +function reorderChildren(parent, orderedChildrenGuids, options) { + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: reorderChildren", + db => + db.executeTransaction(async function () { + // Fetch old indices for the notifications. + const oldIndices = new Map(); + ( + await db.executeCached( + `SELECT guid, position FROM moz_bookmarks WHERE parent = :parentId`, + { parentId: parent._id } + ) + ).forEach(r => + oldIndices.set( + r.getResultByName("guid"), + r.getResultByName("position") + ) + ); + // By the time the caller collects guids and the time reorder is invoked + // new bookmarks may appear, and the passed-in list becomes incomplete. + // To avoid unnecessary work then skip reorder if children are already + // in the requested sort order. + let lastIndex = 0, + needReorder = false; + for (let guid of orderedChildrenGuids) { + let requestedIndex = oldIndices.get(guid); + if (requestedIndex === undefined) { + // doesn't exist, just ignore it. + continue; + } + if (requestedIndex < lastIndex) { + needReorder = true; + break; + } + lastIndex = requestedIndex; + } + if (!needReorder) { + return []; + } + + const valuesFragment = orderedChildrenGuids + .map((g, i) => `("${g}", ${i})`) + .join(); + await db.execute( + `UPDATE moz_bookmarks + SET position = sorted.pos, + lastModified = :lastModified + FROM ( + WITH fixed(guid, pos) AS ( + VALUES ${valuesFragment} + ) + SELECT b.id, + row_number() OVER (ORDER BY CASE WHEN fixed.pos IS NULL THEN 1 ELSE 0 END ASC, fixed.pos ASC, position ASC) - 1 AS pos + FROM moz_bookmarks b + LEFT JOIN fixed ON b.guid = fixed.guid + WHERE parent = :parentId + ) AS sorted + WHERE sorted.id = moz_bookmarks.id`, + { + parentId: parent._id, + lastModified: lazy.PlacesUtils.toPRTime(options.lastModified), + } + ); + + let syncChangeDelta = + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta( + options.source + ); + await setAncestorsLastModified( + db, + parent.guid, + options.lastModified, + syncChangeDelta + ); + // Fetch bookmarks for the notifications, adding _oldIndex to each. + return ( + await fetchBookmarksByParent(db, { + parentGuid: parent.guid, + }) + ).map(c => { + // We're not returning these objects to the caller, but just in case + // we'd decide to do it in the future, make sure this will be removed. + // See rowsToItemsArray() for additional details. + Object.defineProperty(c, "_oldIndex", { + value: oldIndices.get(c.guid) || 0, + enumerable: false, + configurable: true, + }); + return c; + }); + }) + ); +} + +// Helpers. + +/** + * Merges objects into a new object, included non-enumerable properties. + * + * @param sources + * source objects to merge. + * @return a new object including all properties from the source objects. + */ +function mergeIntoNewObject(...sources) { + let dest = {}; + for (let src of sources) { + for (let prop of Object.getOwnPropertyNames(src)) { + Object.defineProperty( + dest, + prop, + Object.getOwnPropertyDescriptor(src, prop) + ); + } + } + return dest; +} + +/** + * Remove properties that have the same value across two bookmark objects. + * + * @param dest + * destination bookmark object. + * @param src + * source bookmark object. + * @return a cleaned up bookmark object. + * @note "guid" is never removed. + */ +function removeSameValueProperties(dest, src) { + for (let prop in dest) { + let remove = false; + switch (prop) { + case "lastModified": + case "dateAdded": + remove = + src.hasOwnProperty(prop) && + dest[prop].getTime() == src[prop].getTime(); + break; + case "url": + remove = src.hasOwnProperty(prop) && dest[prop].href == src[prop].href; + break; + default: + remove = dest[prop] == src[prop]; + } + if (remove && prop != "guid") { + delete dest[prop]; + } + } +} + +/** + * Convert an array of mozIStorageRow objects to an array of bookmark objects. + * + * @param {Array} rows + * the array of mozIStorageRow objects. + * @param {Boolean} ignoreInvalidURLs + * whether to ignore invalid urls (leaving the url property undefined) + * or throw. + * @return an array of bookmark objects. + */ +function rowsToItemsArray(rows, ignoreInvalidURLs = false) { + return rows.map(row => { + let item = {}; + for (let prop of ["guid", "index", "type", "title"]) { + item[prop] = row.getResultByName(prop); + } + for (let prop of ["dateAdded", "lastModified"]) { + let value = row.getResultByName(prop); + if (value) { + item[prop] = lazy.PlacesUtils.toDate(value); + } + } + let parentGuid = row.getResultByName("parentGuid"); + if (parentGuid) { + item.parentGuid = parentGuid; + } + let url = row.getResultByName("url"); + if (url) { + try { + item.url = new URL(url); + } catch (ex) { + if (!ignoreInvalidURLs) { + throw ex; + } + } + } + + // All the private properties below this point should not be returned to the + // API consumer, thus they are non-enumerable and removed through + // Object.assign just before the object is returned. + // Configurable is set to support mergeIntoNewObject overwrites. + + for (let prop of [ + "_id", + "_parentId", + "_childCount", + "_grandParentId", + "_syncStatus", + ]) { + let val = row.getResultByName(prop); + if (val !== null) { + Object.defineProperty(item, prop, { + value: val, + enumerable: false, + configurable: true, + }); + } + } + + try { + let tags = row.getResultByName("_tags"); + Object.defineProperty(item, "_tags", { + value: tags + ? tags + .split(",") + .sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase())) + : [], + enumerable: false, + configurable: true, + }); + } catch (ex) { + // `tags` not fetched, don't add it. + } + + return item; + }); +} + +function validateBookmarkObject(name, input, behavior) { + return lazy.PlacesUtils.validateItemProperties( + name, + lazy.PlacesUtils.BOOKMARK_VALIDATORS, + input, + behavior + ); +} + +/** + * Updates lastModified for all the ancestors of a given folder GUID. + * + * @param db + * the Sqlite.sys.mjs connection handle. + * @param folderGuid + * the GUID of the folder whose ancestors should be updated. + * @param time + * a Date object to use for the update. + * + * @note the folder itself is also updated. + */ +var setAncestorsLastModified = async function ( + db, + folderGuid, + time, + syncChangeDelta +) { + await db.executeCached( + `WITH RECURSIVE + ancestors(aid) AS ( + SELECT id FROM moz_bookmarks WHERE guid = :guid + UNION ALL + SELECT parent FROM moz_bookmarks + JOIN ancestors ON id = aid + WHERE type = :type + ) + UPDATE moz_bookmarks SET lastModified = :time + WHERE id IN ancestors + `, + { + guid: folderGuid, + type: Bookmarks.TYPE_FOLDER, + time: lazy.PlacesUtils.toPRTime(time), + } + ); + + if (syncChangeDelta) { + // Flag the folder as having a change. + await db.executeCached( + ` + UPDATE moz_bookmarks SET + syncChangeCounter = syncChangeCounter + :syncChangeDelta + WHERE guid = :guid`, + { guid: folderGuid, syncChangeDelta } + ); + } +}; + +/** + * Remove all descendants of one or more bookmark folders. + * + * @param {Object} db + * the Sqlite.sys.mjs connection handle. + * @param {Array} folderGuids + * array of folder guids. + * @return {Array} + * An array of the affected urls. + */ +var removeFoldersContents = async function (db, folderGuids, options) { + let syncChangeDelta = lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta( + options.source + ); + + let itemsRemoved = []; + for (let folderGuid of folderGuids) { + let rows = await db.executeCached( + `WITH RECURSIVE + descendants(did) AS ( + SELECT b.id FROM moz_bookmarks b + JOIN moz_bookmarks p ON b.parent = p.id + WHERE p.guid = :folderGuid + UNION ALL + SELECT id FROM moz_bookmarks + JOIN descendants ON parent = did + ) + SELECT b.id AS _id, b.parent AS _parentId, b.position AS 'index', + b.type, url, b.guid, p.guid AS parentGuid, b.dateAdded, + b.lastModified, IFNULL(b.title, '') AS title, + p.parent AS _grandParentId, NULL AS _childCount, + b.syncStatus AS _syncStatus + FROM descendants + /* The usage of CROSS JOIN is not random, it tells the optimizer + to retain the original rows order, so the hierarchy is respected */ + CROSS JOIN moz_bookmarks b ON did = b.id + JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_places h ON b.fk = h.id`, + { folderGuid } + ); + + itemsRemoved = itemsRemoved.concat(rowsToItemsArray(rows, true)); + + await db.executeCached( + `WITH RECURSIVE + descendants(did) AS ( + SELECT b.id FROM moz_bookmarks b + JOIN moz_bookmarks p ON b.parent = p.id + WHERE p.guid = :folderGuid + UNION ALL + SELECT id FROM moz_bookmarks + JOIN descendants ON parent = did + ) + DELETE FROM moz_bookmarks WHERE id IN descendants`, + { folderGuid } + ); + } + + // Write tombstones for removed items. + await insertTombstones(db, itemsRemoved, syncChangeDelta); + + // Bump the change counter for all tagged bookmarks when removing tag + // folders. + await addSyncChangesForRemovedTagFolders(db, itemsRemoved, syncChangeDelta); + + // TODO (Bug 1087576): this may leave orphan tags behind. + + // Send onItemRemoved notifications to listeners. + // TODO (Bug 1087580): for the case of eraseEverything, this should send a + // single clear bookmarks notification rather than notifying for each + // bookmark. + + // Notify listeners in reverse order to serve children before parents. + let { source = Bookmarks.SOURCES.DEFAULT } = options; + let notifications = []; + for (let item of itemsRemoved.reverse()) { + let isUntagging = item._grandParentId == lazy.PlacesUtils.tagsFolderId; + let url = ""; + if (item.type == Bookmarks.TYPE_BOOKMARK) { + url = item.hasOwnProperty("url") ? item.url.href : null; + } + notifications.push( + new PlacesBookmarkRemoved({ + id: item._id, + url, + title: item.title, + parentId: item._parentId, + index: item.index, + itemType: item.type, + guid: item.guid, + parentGuid: item.parentGuid, + source, + isTagging: isUntagging, + isDescendantRemoval: + !lazy.PlacesUtils.bookmarks.userContentRoots.includes( + item.parentGuid + ), + }) + ); + + if (isUntagging) { + for (let entry of await fetchBookmarksByURL(item, true)) { + notifications.push( + new PlacesBookmarkTags({ + id: entry._id, + itemType: entry.type, + url, + guid: entry.guid, + parentGuid: entry.parentGuid, + tags: entry._tags, + lastModified: entry.lastModified, + source, + isTagging: false, + }) + ); + } + } + } + + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + + return itemsRemoved.filter(item => "url" in item).map(item => item.url); +}; + +// Indicates whether we should write a tombstone for an item that has been +// uploaded to the server. We ignore "NEW" and "UNKNOWN" items: "NEW" items +// haven't been uploaded yet, and "UNKNOWN" items need a full reconciliation +// with the server. +function needsTombstone(item) { + return item._syncStatus == Bookmarks.SYNC_STATUS.NORMAL; +} + +// Inserts tombstones for removed synced items. +function insertTombstones(db, itemsRemoved, syncChangeDelta) { + if (!syncChangeDelta) { + return Promise.resolve(); + } + let syncedItems = itemsRemoved.filter(needsTombstone); + if (!syncedItems.length) { + return Promise.resolve(); + } + let dateRemoved = lazy.PlacesUtils.toPRTime(Date.now()); + let valuesTable = syncedItems + .map( + item => `( + ${JSON.stringify(item.guid)}, + ${dateRemoved} + )` + ) + .join(","); + return db.execute(` + INSERT INTO moz_bookmarks_deleted (guid, dateRemoved) + VALUES ${valuesTable}`); +} + +// Bumps the change counter for all bookmarks with URLs referenced in removed +// tag folders. +var addSyncChangesForRemovedTagFolders = async function ( + db, + itemsRemoved, + syncChangeDelta +) { + if (!syncChangeDelta) { + return; + } + for (let item of itemsRemoved) { + let isUntagging = item._grandParentId == lazy.PlacesUtils.tagsFolderId; + if (isUntagging) { + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + item.url, + syncChangeDelta + ); + } + } +}; + +function adjustSeparatorsSyncCounter( + db, + parentId, + startIndex, + syncChangeDelta +) { + if (!syncChangeDelta) { + return Promise.resolve(); + } + + return db.executeCached( + ` + UPDATE moz_bookmarks + SET syncChangeCounter = syncChangeCounter + :delta + WHERE parent = :parent AND position >= :start_index + AND type = :item_type + `, + { + delta: syncChangeDelta, + parent: parentId, + start_index: startIndex, + item_type: Bookmarks.TYPE_SEPARATOR, + } + ); +} + +/** + * Return the full path, from parent to root folder, of a bookmark. + * + * @param guid + * The globally unique identifier of the item to determine the full + * bookmark path for. + * @param options [optional] + * an optional object whose properties describe options for the query: + * - concurrent: Queries concurrently to any writes, returning results + * faster. On the negative side, it may return stale + * information missing the currently ongoing write. + * - db: A specific connection to be used. + * @return {Promise} resolved when the query is complete. + * @resolves to an array of {guid, title} objects that represent the full path + * from parent to root for the passed in bookmark. + * @rejects if an error happens while querying. + */ +async function retrieveFullBookmarkPath(guid, options = {}) { + let query = async function (db) { + let rows = await db.executeCached( + `WITH RECURSIVE parents(guid, _id, _parent, title) AS + (SELECT guid, id AS _id, parent AS _parent, + IFNULL(title, '') AS title + FROM moz_bookmarks + WHERE guid = :pguid + UNION ALL + SELECT b.guid, b.id AS _id, b.parent AS _parent, + IFNULL(b.title, '') AS title + FROM moz_bookmarks b + INNER JOIN parents ON b.id=parents._parent) + SELECT * FROM parents WHERE guid != :rootGuid; + `, + { pguid: guid, rootGuid: lazy.PlacesUtils.bookmarks.rootGuid } + ); + + return rows.reverse().map(r => ({ + guid: r.getResultByName("guid"), + title: r.getResultByName("title"), + })); + }; + + if (options.concurrent) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + return query(db); + } + return lazy.PlacesUtils.withConnectionWrapper( + "Bookmarks.jsm: retrieveFullBookmarkPath", + query + ); +} diff --git a/toolkit/components/places/Database.cpp b/toolkit/components/places/Database.cpp new file mode 100644 index 0000000000..b614b6b460 --- /dev/null +++ b/toolkit/components/places/Database.cpp @@ -0,0 +1,2902 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Attributes.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/SpinEventLoopUntil.h" +#include "mozilla/JSONStringWriteFuncs.h" +#include "mozilla/StaticPrefs_places.h" + +#include "Database.h" + +#include "nsIInterfaceRequestorUtils.h" +#include "nsIFile.h" + +#include "nsNavBookmarks.h" +#include "nsNavHistory.h" +#include "nsPlacesTables.h" +#include "nsPlacesIndexes.h" +#include "nsPlacesTriggers.h" +#include "nsPlacesMacros.h" +#include "nsVariant.h" +#include "SQLFunctions.h" +#include "Helpers.h" +#include "nsFaviconService.h" + +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "prenv.h" +#include "prsystem.h" +#include "nsPrintfCString.h" +#include "mozilla/Preferences.h" +#include "mozilla/Services.h" +#include "mozilla/Unused.h" +#include "mozIStorageService.h" +#include "prtime.h" + +#include "nsXULAppAPI.h" + +// Time between corrupt database backups. +#define RECENT_BACKUP_TIME_MICROSEC (int64_t)86400 * PR_USEC_PER_SEC // 24H + +// Filename of the database. +#define DATABASE_FILENAME u"places.sqlite"_ns +// Filename of the icons database. +#define DATABASE_FAVICONS_FILENAME u"favicons.sqlite"_ns + +// Set to the database file name when it was found corrupt by a previous +// maintenance run. +#define PREF_FORCE_DATABASE_REPLACEMENT \ + "places.database.replaceDatabaseOnStartup" + +// Whether on corruption we should try to fix the database by cloning it. +#define PREF_DATABASE_CLONEONCORRUPTION "places.database.cloneOnCorruption" + +// Set to specify the size of the places database growth increments in kibibytes +#define PREF_GROWTH_INCREMENT_KIB "places.database.growthIncrementKiB" + +// Set to disable the default robust storage and use volatile, in-memory +// storage without robust transaction flushing guarantees. This makes +// SQLite use much less I/O at the cost of losing data when things crash. +// The pref is only honored if an environment variable is set. The env +// variable is intentionally named something scary to help prevent someone +// from thinking it is a useful performance optimization they should enable. +#define PREF_DISABLE_DURABILITY "places.database.disableDurability" + +#define PREF_PREVIEWS_ENABLED "places.previews.enabled" + +#define ENV_ALLOW_CORRUPTION \ + "ALLOW_PLACES_DATABASE_TO_LOSE_DATA_AND_BECOME_CORRUPT" + +#define PREF_MIGRATE_V52_ORIGIN_FRECENCIES \ + "places.database.migrateV52OriginFrecencies" + +// Maximum size for the WAL file. +// For performance reasons this should be as large as possible, so that more +// transactions can fit into it, and the checkpoint cost is paid less often. +// At the same time, since we use synchronous = NORMAL, an fsync happens only +// at checkpoint time, so we don't want the WAL to grow too much and risk to +// lose all the contained transactions on a crash. +#define DATABASE_MAX_WAL_BYTES 2048000 + +// Since exceeding the journal limit will cause a truncate, we allow a slightly +// larger limit than DATABASE_MAX_WAL_BYTES to reduce the number of truncates. +// This is the number of bytes the journal can grow over the maximum wal size +// before being truncated. +#define DATABASE_JOURNAL_OVERHEAD_BYTES 2048000 + +#define BYTES_PER_KIBIBYTE 1024 + +// How much time Sqlite can wait before returning a SQLITE_BUSY error. +#define DATABASE_BUSY_TIMEOUT_MS 100 + +// This annotation is no longer used & is obsolete, but here for migration. +#define LAST_USED_ANNO "bookmarkPropertiesDialog/folderLastUsed"_ns +// This is key in the meta table that the LAST_USED_ANNO is migrated to. +#define LAST_USED_FOLDERS_META_KEY "places/bookmarks/edit/lastusedfolder"_ns + +// We use a fixed title for the mobile root to avoid marking the database as +// corrupt if we can't look up the localized title in the string bundle. Sync +// sets the title to the localized version when it creates the left pane query. +#define MOBILE_ROOT_TITLE "mobile" + +// Legacy item annotation used by the old Sync engine. +#define SYNC_PARENT_ANNO "sync/parent" + +using namespace mozilla; + +namespace mozilla::places { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// +//// Helpers + +/** + * Get the filename for a corrupt database. + */ +nsString getCorruptFilename(const nsString& aDbFilename) { + return aDbFilename + u".corrupt"_ns; +} +/** + * Get the filename for a recover database. + */ +nsString getRecoverFilename(const nsString& aDbFilename) { + return aDbFilename + u".recover"_ns; +} + +/** + * Checks whether exists a corrupt database file created not longer than + * RECENT_BACKUP_TIME_MICROSEC ago. + */ +bool isRecentCorruptFile(const nsCOMPtr& aCorruptFile) { + MOZ_ASSERT(NS_IsMainThread()); + bool fileExists = false; + if (NS_FAILED(aCorruptFile->Exists(&fileExists)) || !fileExists) { + return false; + } + PRTime lastMod = 0; + return NS_SUCCEEDED(aCorruptFile->GetLastModifiedTime(&lastMod)) && + lastMod > 0 && (PR_Now() - lastMod) <= RECENT_BACKUP_TIME_MICROSEC; +} + +/** + * Removes a file, optionally adding a suffix to the file name. + */ +void RemoveFileSwallowsErrors(const nsCOMPtr& aFile, + const nsString& aSuffix = u""_ns) { + nsCOMPtr file; + MOZ_ALWAYS_SUCCEEDS(aFile->Clone(getter_AddRefs(file))); + if (!aSuffix.IsEmpty()) { + nsAutoString newFileName; + file->GetLeafName(newFileName); + newFileName.Append(aSuffix); + MOZ_ALWAYS_SUCCEEDS(file->SetLeafName(newFileName)); + } + DebugOnly rv = file->Remove(false); + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "Failed to remove file."); +} + +/** + * Sets the connection journal mode to one of the JOURNAL_* types. + * + * @param aDBConn + * The database connection. + * @param aJournalMode + * One of the JOURNAL_* types. + * @returns the current journal mode. + * @note this may return a different journal mode than the required one, since + * setting it may fail. + */ +enum JournalMode SetJournalMode(nsCOMPtr& aDBConn, + enum JournalMode aJournalMode) { + MOZ_ASSERT(NS_IsMainThread()); + nsAutoCString journalMode; + switch (aJournalMode) { + default: + MOZ_FALLTHROUGH_ASSERT("Trying to set an unknown journal mode."); + // Fall through to the default DELETE journal. + case JOURNAL_DELETE: + journalMode.AssignLiteral("delete"); + break; + case JOURNAL_TRUNCATE: + journalMode.AssignLiteral("truncate"); + break; + case JOURNAL_MEMORY: + journalMode.AssignLiteral("memory"); + break; + case JOURNAL_WAL: + journalMode.AssignLiteral("wal"); + break; + } + + nsCOMPtr statement; + nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA journal_mode = "); + query.Append(journalMode); + aDBConn->CreateStatement(query, getter_AddRefs(statement)); + NS_ENSURE_TRUE(statement, JOURNAL_DELETE); + + bool hasResult = false; + if (NS_SUCCEEDED(statement->ExecuteStep(&hasResult)) && hasResult && + NS_SUCCEEDED(statement->GetUTF8String(0, journalMode))) { + if (journalMode.EqualsLiteral("delete")) { + return JOURNAL_DELETE; + } + if (journalMode.EqualsLiteral("truncate")) { + return JOURNAL_TRUNCATE; + } + if (journalMode.EqualsLiteral("memory")) { + return JOURNAL_MEMORY; + } + if (journalMode.EqualsLiteral("wal")) { + return JOURNAL_WAL; + } + MOZ_ASSERT(false, "Got an unknown journal mode."); + } + + return JOURNAL_DELETE; +} + +nsresult CreateRoot(nsCOMPtr& aDBConn, + const nsCString& aRootName, const nsCString& aGuid, + const nsCString& titleString, const int32_t position, + int64_t& newId) { + MOZ_ASSERT(NS_IsMainThread()); + + // A single creation timestamp for all roots so that the root folder's + // last modification time isn't earlier than its childrens' creation time. + static PRTime timestamp = 0; + if (!timestamp) timestamp = RoundedPRNow(); + + // Create a new bookmark folder for the root. + nsCOMPtr stmt; + nsresult rv = aDBConn->CreateStatement( + nsLiteralCString( + "INSERT INTO moz_bookmarks " + "(type, position, title, dateAdded, lastModified, guid, parent, " + "syncChangeCounter, syncStatus) " + "VALUES (:item_type, :item_position, :item_title," + ":date_added, :last_modified, :guid, " + "IFNULL((SELECT id FROM moz_bookmarks WHERE parent = 0), 0), " + "1, :sync_status)"), + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) return rv; + + rv = stmt->BindInt32ByName("item_type"_ns, + nsINavBookmarksService::TYPE_FOLDER); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindInt32ByName("item_position"_ns, position); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindUTF8StringByName("item_title"_ns, titleString); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindInt64ByName("date_added"_ns, timestamp); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindInt64ByName("last_modified"_ns, timestamp); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindUTF8StringByName("guid"_ns, aGuid); + if (NS_FAILED(rv)) return rv; + rv = stmt->BindInt32ByName("sync_status"_ns, + nsINavBookmarksService::SYNC_STATUS_NEW); + if (NS_FAILED(rv)) return rv; + rv = stmt->Execute(); + if (NS_FAILED(rv)) return rv; + + newId = nsNavBookmarks::sLastInsertedItemId; + return NS_OK; +} + +nsresult SetupDurability(nsCOMPtr& aDBConn, + int32_t aDBPageSize) { + nsresult rv; + if (PR_GetEnv(ENV_ALLOW_CORRUPTION) && + Preferences::GetBool(PREF_DISABLE_DURABILITY, false)) { + // Volatile storage was requested. Use the in-memory journal (no + // filesystem I/O) and don't sync the filesystem after writing. + SetJournalMode(aDBConn, JOURNAL_MEMORY); + rv = aDBConn->ExecuteSimpleSQL("PRAGMA synchronous = OFF"_ns); + NS_ENSURE_SUCCESS(rv, rv); + } else { + // Be sure to set journal mode after page_size. WAL would prevent the + // change otherwise. + if (JOURNAL_WAL == SetJournalMode(aDBConn, JOURNAL_WAL)) { + // Set the WAL journal size limit. + int32_t checkpointPages = + static_cast(DATABASE_MAX_WAL_BYTES / aDBPageSize); + nsAutoCString checkpointPragma("PRAGMA wal_autocheckpoint = "); + checkpointPragma.AppendInt(checkpointPages); + rv = aDBConn->ExecuteSimpleSQL(checkpointPragma); + NS_ENSURE_SUCCESS(rv, rv); + } else { + // Ignore errors, if we fail here the database could be considered corrupt + // and we won't be able to go on, even if it's just matter of a bogus file + // system. The default mode (DELETE) will be fine in such a case. + (void)SetJournalMode(aDBConn, JOURNAL_TRUNCATE); + + // Set synchronous to FULL to ensure maximum data integrity, even in + // case of crashes or unclean shutdowns. + rv = aDBConn->ExecuteSimpleSQL("PRAGMA synchronous = FULL"_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + // The journal is usually free to grow for performance reasons, but it never + // shrinks back. Since the space taken may be problematic, limit its size. + nsAutoCString journalSizePragma("PRAGMA journal_size_limit = "); + journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + + DATABASE_JOURNAL_OVERHEAD_BYTES); + (void)aDBConn->ExecuteSimpleSQL(journalSizePragma); + + // Grow places in |growthIncrementKiB| increments to limit fragmentation on + // disk. By default, it's 5 MB. + int32_t growthIncrementKiB = + Preferences::GetInt(PREF_GROWTH_INCREMENT_KIB, 5 * BYTES_PER_KIBIBYTE); + if (growthIncrementKiB > 0) { + (void)aDBConn->SetGrowthIncrement(growthIncrementKiB * BYTES_PER_KIBIBYTE, + ""_ns); + } + return NS_OK; +} + +nsresult AttachDatabase(nsCOMPtr& aDBConn, + const nsACString& aPath, const nsACString& aName) { + nsCOMPtr stmt; + nsresult rv = aDBConn->CreateStatement("ATTACH DATABASE :path AS "_ns + aName, + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("path"_ns, aPath); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // The journal limit must be set apart for each database. + nsAutoCString journalSizePragma("PRAGMA favicons.journal_size_limit = "); + journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + + DATABASE_JOURNAL_OVERHEAD_BYTES); + Unused << aDBConn->ExecuteSimpleSQL(journalSizePragma); + + return NS_OK; +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// +//// Database + +PLACES_FACTORY_SINGLETON_IMPLEMENTATION(Database, gDatabase) + +NS_IMPL_ISUPPORTS(Database, nsIObserver, nsISupportsWeakReference) + +Database::Database() + : mMainThreadStatements(mMainConn), + mMainThreadAsyncStatements(mMainConn), + mAsyncThreadStatements(mMainConn), + mDBPageSize(0), + mDatabaseStatus(nsINavHistoryService::DATABASE_STATUS_OK), + mClosed(false), + mClientsShutdown(new ClientsShutdownBlocker()), + mConnectionShutdown(new ConnectionShutdownBlocker(this)), + mMaxUrlLength(0), + mCacheObservers(TOPIC_PLACES_INIT_COMPLETE), + mRootId(-1), + mMenuRootId(-1), + mTagsRootId(-1), + mUnfiledRootId(-1), + mToolbarRootId(-1), + mMobileRootId(-1) { + MOZ_ASSERT(!XRE_IsContentProcess(), + "Cannot instantiate Places in the content process"); + // Attempting to create two instances of the service? + MOZ_ASSERT(!gDatabase); + gDatabase = this; +} + +already_AddRefed +Database::GetProfileChangeTeardownPhase() { + nsCOMPtr asyncShutdownSvc = + services::GetAsyncShutdownService(); + MOZ_ASSERT(asyncShutdownSvc); + if (NS_WARN_IF(!asyncShutdownSvc)) { + return nullptr; + } + + // Consumers of Places should shutdown before us, at profile-change-teardown. + nsCOMPtr shutdownPhase; + DebugOnly rv = + asyncShutdownSvc->GetProfileChangeTeardown(getter_AddRefs(shutdownPhase)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + return shutdownPhase.forget(); +} + +already_AddRefed +Database::GetProfileBeforeChangePhase() { + nsCOMPtr asyncShutdownSvc = + services::GetAsyncShutdownService(); + MOZ_ASSERT(asyncShutdownSvc); + if (NS_WARN_IF(!asyncShutdownSvc)) { + return nullptr; + } + + // Consumers of Places should shutdown before us, at profile-change-teardown. + nsCOMPtr shutdownPhase; + DebugOnly rv = + asyncShutdownSvc->GetProfileBeforeChange(getter_AddRefs(shutdownPhase)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + return shutdownPhase.forget(); +} + +Database::~Database() = default; + +already_AddRefed Database::GetAsyncStatement( + const nsACString& aQuery) { + if (PlacesShutdownBlocker::sIsStarted || NS_FAILED(EnsureConnection())) { + return nullptr; + } + + MOZ_ASSERT(NS_IsMainThread()); + return mMainThreadAsyncStatements.GetCachedStatement(aQuery); +} + +already_AddRefed Database::GetStatement( + const nsACString& aQuery) { + if (PlacesShutdownBlocker::sIsStarted) { + return nullptr; + } + if (NS_IsMainThread()) { + if (NS_FAILED(EnsureConnection())) { + return nullptr; + } + return mMainThreadStatements.GetCachedStatement(aQuery); + } + // In the async case, the connection must have been started on the main-thread + // already. + MOZ_ASSERT(mMainConn); + return mAsyncThreadStatements.GetCachedStatement(aQuery); +} + +already_AddRefed Database::GetClientsShutdown() { + if (mClientsShutdown) return mClientsShutdown->GetClient(); + return nullptr; +} + +already_AddRefed Database::GetConnectionShutdown() { + if (mConnectionShutdown) return mConnectionShutdown->GetClient(); + return nullptr; +} + +// static +already_AddRefed Database::GetDatabase() { + if (PlacesShutdownBlocker::sIsStarted) { + return nullptr; + } + return GetSingleton(); +} + +nsresult Database::Init() { + MOZ_ASSERT(NS_IsMainThread()); + + // DO NOT FAIL HERE, otherwise we would never break the cycle between this + // object and the shutdown blockers, causing unexpected leaks. + + { + // First of all Places clients should block profile-change-teardown. + nsCOMPtr shutdownPhase = + GetProfileChangeTeardownPhase(); + MOZ_ASSERT(shutdownPhase); + if (shutdownPhase) { + DebugOnly rv = shutdownPhase->AddBlocker( + static_cast(mClientsShutdown.get()), + NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, u""_ns); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + } + + { + // Then connection closing should block profile-before-change. + nsCOMPtr shutdownPhase = + GetProfileBeforeChangePhase(); + MOZ_ASSERT(shutdownPhase); + if (shutdownPhase) { + DebugOnly rv = shutdownPhase->AddBlocker( + static_cast(mConnectionShutdown.get()), + NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, u""_ns); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + } + + // Finally observe profile shutdown notifications. + nsCOMPtr os = mozilla::services::GetObserverService(); + if (os) { + (void)os->AddObserver(this, TOPIC_PROFILE_CHANGE_TEARDOWN, true); + } + return NS_OK; +} + +nsresult Database::EnsureConnection() { + // Run this only once. + if (mMainConn || + mDatabaseStatus == nsINavHistoryService::DATABASE_STATUS_LOCKED) { + return NS_OK; + } + // Don't try to create a database too late. + if (PlacesShutdownBlocker::sIsStarted) { + return NS_ERROR_FAILURE; + } + + MOZ_ASSERT(NS_IsMainThread(), + "Database initialization must happen on the main-thread"); + + { + bool initSucceeded = false; + auto notify = MakeScopeExit([&]() { + // If the database connection cannot be opened, it may just be locked + // by third parties. Set a locked state. + if (!initSucceeded) { + mMainConn = nullptr; + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_LOCKED; + } + // Notify at the next tick, to avoid re-entrancy problems. + NS_DispatchToMainThread( + NewRunnableMethod("places::Database::EnsureConnection()", this, + &Database::NotifyConnectionInitalized)); + }); + + nsCOMPtr storage = + do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID); + NS_ENSURE_STATE(storage); + + nsCOMPtr profileDir; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr databaseFile; + rv = profileDir->Clone(getter_AddRefs(databaseFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = databaseFile->Append(DATABASE_FILENAME); + NS_ENSURE_SUCCESS(rv, rv); + bool databaseExisted = false; + rv = databaseFile->Exists(&databaseExisted); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoString corruptDbName; + if (NS_SUCCEEDED(Preferences::GetString(PREF_FORCE_DATABASE_REPLACEMENT, + corruptDbName)) && + !corruptDbName.IsEmpty()) { + // If this pref is set, maintenance required a database replacement, due + // to integrity corruption. Be sure to clear the pref to avoid handling it + // more than once. + (void)Preferences::ClearUser(PREF_FORCE_DATABASE_REPLACEMENT); + + // The database is corrupt, backup and replace it with a new one. + nsCOMPtr fileToBeReplaced; + bool fileExists = false; + if (NS_SUCCEEDED(profileDir->Clone(getter_AddRefs(fileToBeReplaced))) && + NS_SUCCEEDED(fileToBeReplaced->Exists(&fileExists)) && fileExists) { + rv = BackupAndReplaceDatabaseFile(storage, corruptDbName, true, false); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + // Open the database file. If it does not exist a new one will be created. + // Use an unshared connection, it will consume more memory but avoid shared + // cache contentions across threads. + rv = storage->OpenUnsharedDatabase(databaseFile, + mozIStorageService::CONNECTION_DEFAULT, + getter_AddRefs(mMainConn)); + if (NS_SUCCEEDED(rv) && !databaseExisted) { + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CREATE; + } else if (rv == NS_ERROR_FILE_CORRUPTED) { + // The database is corrupt, backup and replace it with a new one. + rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FILENAME, true, true); + // Fallback to catch-all handler. + } + NS_ENSURE_SUCCESS(rv, rv); + + // Initialize the database schema. In case of failure the existing schema + // is is corrupt or incoherent, thus the database should be replaced. + bool databaseMigrated = false; + rv = SetupDatabaseConnection(storage); + bool shouldTryToCloneDb = true; + if (NS_SUCCEEDED(rv)) { + // Failing to initialize the schema may indicate a corruption. + rv = InitSchema(&databaseMigrated); + if (NS_FAILED(rv)) { + // Cloning the db on a schema migration may not be a good idea, since we + // may end up cloning the schema problems. + shouldTryToCloneDb = false; + if (rv == NS_ERROR_STORAGE_BUSY || rv == NS_ERROR_FILE_IS_LOCKED || + rv == NS_ERROR_FILE_NO_DEVICE_SPACE || + rv == NS_ERROR_OUT_OF_MEMORY) { + // The database is not corrupt, though some migration step failed. + // This may be caused by concurrent use of sync and async Storage APIs + // or by a system issue. + // The best we can do is trying again. If it should still fail, Places + // won't work properly and will be handled as LOCKED. + rv = InitSchema(&databaseMigrated); + if (NS_FAILED(rv)) { + rv = NS_ERROR_FILE_IS_LOCKED; + } + } else { + rv = NS_ERROR_FILE_CORRUPTED; + } + } + } + if (NS_WARN_IF(NS_FAILED(rv))) { + if (rv != NS_ERROR_FILE_IS_LOCKED) { + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CORRUPT; + } + // Some errors may not indicate a database corruption, for those cases we + // just bail out without throwing away a possibly valid places.sqlite. + if (rv == NS_ERROR_FILE_CORRUPTED) { + // Since we don't know which database is corrupt, we must replace both. + rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FAVICONS_FILENAME, + false, false); + NS_ENSURE_SUCCESS(rv, rv); + rv = BackupAndReplaceDatabaseFile(storage, DATABASE_FILENAME, + shouldTryToCloneDb, true); + NS_ENSURE_SUCCESS(rv, rv); + // Try to initialize the new database again. + rv = SetupDatabaseConnection(storage); + NS_ENSURE_SUCCESS(rv, rv); + rv = InitSchema(&databaseMigrated); + } + // Bail out if we couldn't fix the database. + NS_ENSURE_SUCCESS(rv, rv); + } + + if (databaseMigrated) { + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_UPGRADED; + } + + // Initialize here all the items that are not part of the on-disk database, + // like views, temp triggers or temp tables. The database should not be + // considered corrupt if any of the following fails. + + rv = InitTempEntities(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = CheckRoots(); + NS_ENSURE_SUCCESS(rv, rv); + + initSucceeded = true; + } + return NS_OK; +} + +nsresult Database::NotifyConnectionInitalized() { + MOZ_ASSERT(NS_IsMainThread()); + // Notify about Places initialization. + nsCOMArray entries; + mCacheObservers.GetEntries(entries); + for (int32_t idx = 0; idx < entries.Count(); ++idx) { + MOZ_ALWAYS_SUCCEEDS( + entries[idx]->Observe(nullptr, TOPIC_PLACES_INIT_COMPLETE, nullptr)); + } + nsCOMPtr obs = mozilla::services::GetObserverService(); + if (obs) { + MOZ_ALWAYS_SUCCEEDS( + obs->NotifyObservers(nullptr, TOPIC_PLACES_INIT_COMPLETE, nullptr)); + } + return NS_OK; +} + +nsresult Database::EnsureFaviconsDatabaseAttached( + const nsCOMPtr& aStorage) { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr databaseFile; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(databaseFile)); + NS_ENSURE_STATE(databaseFile); + nsresult rv = databaseFile->Append(DATABASE_FAVICONS_FILENAME); + NS_ENSURE_SUCCESS(rv, rv); + nsString iconsPath; + rv = databaseFile->GetPath(iconsPath); + NS_ENSURE_SUCCESS(rv, rv); + + bool fileExists = false; + if (NS_SUCCEEDED(databaseFile->Exists(&fileExists)) && fileExists) { + return AttachDatabase(mMainConn, NS_ConvertUTF16toUTF8(iconsPath), + "favicons"_ns); + } + + // Open the database file, this will also create it. + nsCOMPtr conn; + rv = aStorage->OpenUnsharedDatabase(databaseFile, + mozIStorageService::CONNECTION_DEFAULT, + getter_AddRefs(conn)); + NS_ENSURE_SUCCESS(rv, rv); + + { + // Ensure we'll close the connection when done. + auto cleanup = MakeScopeExit([&]() { + // We cannot use AsyncClose() here, because by the time we try to ATTACH + // this database, its transaction could be still be running and that would + // cause the ATTACH query to fail. + MOZ_ALWAYS_TRUE(NS_SUCCEEDED(conn->Close())); + }); + + // Enable incremental vacuum for this database. Since it will contain even + // large blobs and can be cleared with history, it's worth to have it. + // Note that it will be necessary to manually use PRAGMA incremental_vacuum. + rv = conn->ExecuteSimpleSQL("PRAGMA auto_vacuum = INCREMENTAL"_ns); + NS_ENSURE_SUCCESS(rv, rv); + +#if !defined(HAVE_64BIT_BUILD) + // Ensure that temp tables are held in memory, not on disk, on 32 bit + // platforms. + rv = conn->ExecuteSimpleSQL("PRAGMA temp_store = MEMORY"_ns); + NS_ENSURE_SUCCESS(rv, rv); +#endif + + int32_t defaultPageSize; + rv = conn->GetDefaultPageSize(&defaultPageSize); + NS_ENSURE_SUCCESS(rv, rv); + rv = SetupDurability(conn, defaultPageSize); + NS_ENSURE_SUCCESS(rv, rv); + + // We are going to update the database, so everything from now on should be + // in a transaction for performances. + mozStorageTransaction transaction(conn, false); + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + rv = conn->ExecuteSimpleSQL(CREATE_MOZ_ICONS); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ICONS_ICONURLHASH); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(CREATE_MOZ_PAGES_W_ICONS); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PAGES_W_ICONS_ICONURLHASH); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(CREATE_MOZ_ICONS_TO_PAGES); + NS_ENSURE_SUCCESS(rv, rv); + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + // The scope exit will take care of closing the connection. + } + + rv = AttachDatabase(mMainConn, NS_ConvertUTF16toUTF8(iconsPath), + "favicons"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::BackupAndReplaceDatabaseFile( + nsCOMPtr& aStorage, const nsString& aDbFilename, + bool aTryToClone, bool aReopenConnection) { + MOZ_ASSERT(NS_IsMainThread()); + + if (aDbFilename.Equals(DATABASE_FILENAME)) { + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_CORRUPT; + } else { + // Due to OS file lockings, attached databases can't be cloned properly, + // otherwise trying to reattach them later would fail. + aTryToClone = false; + } + + nsCOMPtr profDir; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profDir)); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr databaseFile; + rv = profDir->Clone(getter_AddRefs(databaseFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = databaseFile->Append(aDbFilename); + NS_ENSURE_SUCCESS(rv, rv); + + // If we already failed in the last 24 hours avoid to create another corrupt + // file, since doing so, in some situation, could cause us to create a new + // corrupt file at every try to access any Places service. That is bad + // because it would quickly fill the user's disk space without any notice. + nsCOMPtr corruptFile; + rv = profDir->Clone(getter_AddRefs(corruptFile)); + NS_ENSURE_SUCCESS(rv, rv); + nsString corruptFilename = getCorruptFilename(aDbFilename); + rv = corruptFile->Append(corruptFilename); + NS_ENSURE_SUCCESS(rv, rv); + if (!isRecentCorruptFile(corruptFile)) { + // Ensure we never create more than one corrupt file. + nsCOMPtr corruptFile; + rv = profDir->Clone(getter_AddRefs(corruptFile)); + NS_ENSURE_SUCCESS(rv, rv); + nsString corruptFilename = getCorruptFilename(aDbFilename); + rv = corruptFile->Append(corruptFilename); + NS_ENSURE_SUCCESS(rv, rv); + rv = corruptFile->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) { + return rv; + } + + nsCOMPtr backup; + Unused << aStorage->BackupDatabaseFile(databaseFile, corruptFilename, + profDir, getter_AddRefs(backup)); + } + + // If anything fails from this point on, we have a stale connection or + // database file, and there's not much more we can do. + // The only thing we can try to do is to replace the database on the next + // startup, and report the problem through telemetry. + { + enum eCorruptDBReplaceStage : int8_t { + stage_closing = 0, + stage_removing, + stage_reopening, + stage_replaced, + stage_cloning, + stage_cloned + }; + eCorruptDBReplaceStage stage = stage_closing; + auto guard = MakeScopeExit([&]() { + // In case we failed to close the connection or remove the database file, + // we want to try again at the next startup. + if (stage == stage_closing || stage == stage_removing) { + Preferences::SetString(PREF_FORCE_DATABASE_REPLACEMENT, aDbFilename); + } + // Report the corruption through telemetry. + Telemetry::Accumulate( + Telemetry::PLACES_DATABASE_CORRUPTION_HANDLING_STAGE, + static_cast(stage)); + }); + + // Close database connection if open. + if (mMainConn) { + rv = mMainConn->SpinningSynchronousClose(); + NS_ENSURE_SUCCESS(rv, rv); + mMainConn = nullptr; + } + + // Remove the broken database. + stage = stage_removing; + rv = databaseFile->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) { + return rv; + } + + // Create a new database file and try to clone tables from the corrupt one. + bool cloned = false; + if (aTryToClone && + Preferences::GetBool(PREF_DATABASE_CLONEONCORRUPTION, true)) { + stage = stage_cloning; + rv = TryToCloneTablesFromCorruptDatabase(aStorage, databaseFile); + if (NS_SUCCEEDED(rv)) { + // If we cloned successfully, we should not consider the database + // corrupt anymore, otherwise we could reimport default bookmarks. + mDatabaseStatus = nsINavHistoryService::DATABASE_STATUS_OK; + cloned = true; + } + } + + if (aReopenConnection) { + // Use an unshared connection, it will consume more memory but avoid + // shared cache contentions across threads. + stage = stage_reopening; + rv = aStorage->OpenUnsharedDatabase( + databaseFile, mozIStorageService::CONNECTION_DEFAULT, + getter_AddRefs(mMainConn)); + NS_ENSURE_SUCCESS(rv, rv); + } + + stage = cloned ? stage_cloned : stage_replaced; + } + + return NS_OK; +} + +nsresult Database::TryToCloneTablesFromCorruptDatabase( + const nsCOMPtr& aStorage, + const nsCOMPtr& aDatabaseFile) { + MOZ_ASSERT(NS_IsMainThread()); + + nsAutoString filename; + nsresult rv = aDatabaseFile->GetLeafName(filename); + + nsCOMPtr corruptFile; + rv = aDatabaseFile->Clone(getter_AddRefs(corruptFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = corruptFile->SetLeafName(getCorruptFilename(filename)); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoString path; + rv = corruptFile->GetPath(path); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr recoverFile; + rv = aDatabaseFile->Clone(getter_AddRefs(recoverFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = recoverFile->SetLeafName(getRecoverFilename(filename)); + NS_ENSURE_SUCCESS(rv, rv); + // Ensure there's no previous recover file. + rv = recoverFile->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) { + return rv; + } + + nsCOMPtr conn; + auto guard = MakeScopeExit([&]() { + if (conn) { + Unused << conn->Close(); + } + RemoveFileSwallowsErrors(recoverFile); + }); + + rv = aStorage->OpenUnsharedDatabase(recoverFile, + mozIStorageService::CONNECTION_DEFAULT, + getter_AddRefs(conn)); + NS_ENSURE_SUCCESS(rv, rv); + rv = AttachDatabase(conn, NS_ConvertUTF16toUTF8(path), "corrupt"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + mozStorageTransaction transaction(conn, false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + // Copy the schema version. + nsCOMPtr stmt; + (void)conn->CreateStatement("PRAGMA corrupt.user_version"_ns, + getter_AddRefs(stmt)); + NS_ENSURE_TRUE(stmt, NS_ERROR_OUT_OF_MEMORY); + bool hasResult; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + int32_t schemaVersion = stmt->AsInt32(0); + rv = conn->SetSchemaVersion(schemaVersion); + NS_ENSURE_SUCCESS(rv, rv); + + // Recreate the tables. + rv = conn->CreateStatement( + nsLiteralCString( + "SELECT name, sql FROM corrupt.sqlite_master " + "WHERE type = 'table' AND name BETWEEN 'moz_' AND 'moza'"), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + nsAutoCString name; + rv = stmt->GetUTF8String(0, name); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString query; + rv = stmt->GetUTF8String(1, query); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(query); + NS_ENSURE_SUCCESS(rv, rv); + // Copy the table contents. + rv = conn->ExecuteSimpleSQL("INSERT INTO main."_ns + name + + " SELECT * FROM corrupt."_ns + name); + if (NS_FAILED(rv)) { + rv = conn->ExecuteSimpleSQL("INSERT INTO main."_ns + name + + " SELECT * FROM corrupt."_ns + name + + " ORDER BY rowid DESC"_ns); + } + NS_ENSURE_SUCCESS(rv, rv); + } + + // Recreate the indices. Doing this after data addition is faster. + rv = conn->CreateStatement( + nsLiteralCString( + "SELECT sql FROM corrupt.sqlite_master " + "WHERE type <> 'table' AND name BETWEEN 'moz_' AND 'moza'"), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + hasResult = false; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + nsAutoCString query; + rv = stmt->GetUTF8String(0, query); + NS_ENSURE_SUCCESS(rv, rv); + rv = conn->ExecuteSimpleSQL(query); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = stmt->Finalize(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + MOZ_ALWAYS_SUCCEEDS(conn->Close()); + conn = nullptr; + rv = recoverFile->RenameTo(nullptr, filename); + NS_ENSURE_SUCCESS(rv, rv); + + RemoveFileSwallowsErrors(corruptFile); + RemoveFileSwallowsErrors(corruptFile, u"-wal"_ns); + RemoveFileSwallowsErrors(corruptFile, u"-shm"_ns); + + guard.release(); + return NS_OK; +} + +nsresult Database::SetupDatabaseConnection( + nsCOMPtr& aStorage) { + MOZ_ASSERT(NS_IsMainThread()); + + // Using immediate transactions allows the main connection to retry writes + // that fail with `SQLITE_BUSY` because a cloned connection has locked the + // database for writing. + nsresult rv = mMainConn->SetDefaultTransactionType( + mozIStorageConnection::TRANSACTION_IMMEDIATE); + NS_ENSURE_SUCCESS(rv, rv); + + // WARNING: any statement executed before setting the journal mode must be + // finalized, since SQLite doesn't allow changing the journal mode if there + // is any outstanding statement. + + { + // Get the page size. This may be different than the default if the + // database file already existed with a different page size. + nsCOMPtr statement; + rv = mMainConn->CreateStatement( + nsLiteralCString(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + bool hasResult = false; + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FILE_CORRUPTED); + rv = statement->GetInt32(0, &mDBPageSize); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && mDBPageSize > 0, + NS_ERROR_FILE_CORRUPTED); + } + +#if !defined(HAVE_64BIT_BUILD) + // Ensure that temp tables are held in memory, not on disk, on 32 bit + // platforms. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA temp_store = MEMORY")); + NS_ENSURE_SUCCESS(rv, rv); +#endif + + rv = SetupDurability(mMainConn, mDBPageSize); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString busyTimeoutPragma("PRAGMA busy_timeout = "); + busyTimeoutPragma.AppendInt(DATABASE_BUSY_TIMEOUT_MS); + (void)mMainConn->ExecuteSimpleSQL(busyTimeoutPragma); + + // Enable FOREIGN KEY support. This is a strict requirement. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA foreign_keys = ON")); + NS_ENSURE_SUCCESS(rv, NS_ERROR_FILE_CORRUPTED); +#ifdef DEBUG + { + // There are a few cases where setting foreign_keys doesn't work: + // * in the middle of a multi-statement transaction + // * if the SQLite library in use doesn't support them + // Since we need foreign_keys, let's at least assert in debug mode. + nsCOMPtr stmt; + mMainConn->CreateStatement("PRAGMA foreign_keys"_ns, getter_AddRefs(stmt)); + bool hasResult = false; + if (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + int32_t fkState = stmt->AsInt32(0); + MOZ_ASSERT(fkState, "Foreign keys should be enabled"); + } + } +#endif + + // Attach the favicons database to the main connection. + rv = EnsureFaviconsDatabaseAttached(aStorage); + if (NS_FAILED(rv)) { + // The favicons database may be corrupt. Try to replace and reattach it. + nsCOMPtr iconsFile; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(iconsFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = iconsFile->Append(DATABASE_FAVICONS_FILENAME); + NS_ENSURE_SUCCESS(rv, rv); + rv = iconsFile->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) { + return rv; + } + rv = EnsureFaviconsDatabaseAttached(aStorage); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Create favicons temp entities. + rv = mMainConn->ExecuteSimpleSQL(CREATE_ICONS_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + // We use our functions during migration, so initialize them now. + rv = InitFunctions(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::InitSchema(bool* aDatabaseMigrated) { + MOZ_ASSERT(NS_IsMainThread()); + *aDatabaseMigrated = false; + + // Get the database schema version. + int32_t currentSchemaVersion; + nsresult rv = mMainConn->GetSchemaVersion(¤tSchemaVersion); + NS_ENSURE_SUCCESS(rv, rv); + bool databaseInitialized = currentSchemaVersion > 0; + + if (databaseInitialized && currentSchemaVersion == DATABASE_SCHEMA_VERSION) { + // The database is up to date and ready to go. + return NS_OK; + } + + auto guard = MakeScopeExit([&]() { + // These run at the end of the migration, out of the transaction, + // regardless of its success. + MigrateV52OriginFrecencies(); + }); + + // We are going to update the database, so everything from now on should be in + // a transaction for performances. + mozStorageTransaction transaction(mMainConn, false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + if (databaseInitialized) { + // Migration How-to: + // + // 1. increment PLACES_SCHEMA_VERSION. + // 2. implement a method that performs upgrade to your version from the + // previous one. + // + // NOTE: The downgrade process is pretty much complicated by the fact old + // versions cannot know what a new version is going to implement. + // The only thing we will do for downgrades is setting back the schema + // version, so that next upgrades will run again the migration step. + + if (currentSchemaVersion < DATABASE_SCHEMA_VERSION) { + *aDatabaseMigrated = true; + + if (currentSchemaVersion < 43) { + // These are versions older than Firefox 60 ESR that are not supported + // anymore. In this case it's safer to just replace the database. + return NS_ERROR_FILE_CORRUPTED; + } + + // Firefox 60 uses schema version 43. - This is an ESR. + + if (currentSchemaVersion < 44) { + rv = MigrateV44Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 45) { + rv = MigrateV45Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 46) { + rv = MigrateV46Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 47) { + rv = MigrateV47Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 61 uses schema version 47. + + if (currentSchemaVersion < 48) { + rv = MigrateV48Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 49) { + rv = MigrateV49Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 50) { + rv = MigrateV50Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 51) { + rv = MigrateV51Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 52) { + rv = MigrateV52Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 62 uses schema version 52. + // Firefox 68 uses schema version 52. - This is an ESR. + + if (currentSchemaVersion < 53) { + rv = MigrateV53Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 69 uses schema version 53 + // Firefox 78 uses schema version 53 - This is an ESR. + + if (currentSchemaVersion < 54) { + rv = MigrateV54Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 81 uses schema version 54 + + if (currentSchemaVersion < 55) { + rv = MigrateV55Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 56) { + rv = MigrateV56Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 57) { + rv = MigrateV57Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 91 uses schema version 57 + + // The schema 58 migration is no longer needed. + + // Firefox 92 uses schema version 58 + + // The schema 59 migration is no longer needed. + + // Firefox 94 uses schema version 59 + + if (currentSchemaVersion < 60) { + rv = MigrateV60Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 96 uses schema version 60 + + if (currentSchemaVersion < 61) { + rv = MigrateV61Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // The schema 62 migration is no longer needed. + + // Firefox 97 uses schema version 62 + + // The schema 63 migration is no longer needed. + + // Firefox 98 uses schema version 63 + + // The schema 64 migration is no longer needed. + + // Firefox 99 uses schema version 64 + + // The schema 65 migration is no longer needed. + + // The schema 66 migration is no longer needed. + + // Firefox 100 uses schema version 66 + + if (currentSchemaVersion < 67) { + rv = MigrateV67Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // The schema 68 migration is no longer needed. + + // Firefox 103 uses schema version 68 + + if (currentSchemaVersion < 69) { + rv = MigrateV69Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 104 uses schema version 69 + + if (currentSchemaVersion < 70) { + rv = MigrateV70Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (currentSchemaVersion < 71) { + rv = MigrateV71Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 110 uses schema version 71 + + if (currentSchemaVersion < 72) { + rv = MigrateV72Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 111 uses schema version 72 + + if (currentSchemaVersion < 73) { + rv = MigrateV73Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 114 uses schema version 73 + + if (currentSchemaVersion < 74) { + rv = MigrateV74Up(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Firefox 115 uses schema version 74 + + // Schema Upgrades must add migration code here. + // >>> IMPORTANT! <<< + // NEVER MIX UP SYNC AND ASYNC EXECUTION IN MIGRATORS, YOU MAY LOCK THE + // CONNECTION AND CAUSE FURTHER STEPS TO FAIL. + // In case, set a bool and do the async work in the ScopeExit guard just + // before the migration steps. + } + } else { + // This is a new database, so we have to create all the tables and indices. + + // moz_origins. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ORIGINS); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_places. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_URL_HASH); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_REVHOST); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_VISITCOUNT); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_FRECENCY); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_LASTVISITDATE); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_GUID); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ORIGIN_ID); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ALT_FRECENCY); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_historyvisits. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_HISTORYVISITS); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_PLACEDATE); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_FROMVISIT); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_HISTORYVISITS_VISITDATE); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_inputhistory. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_INPUTHISTORY); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_bookmarks. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_BOOKMARKS); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_BOOKMARKS_DELETED); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PLACETYPE); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PARENTPOSITION); + NS_ENSURE_SUCCESS(rv, rv); + rv = + mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_PLACELASTMODIFIED); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_DATEADDED); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_BOOKMARKS_GUID); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_keywords. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_KEYWORDS); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_KEYWORDS_PLACEPOSTDATA); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_anno_attributes. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ANNO_ATTRIBUTES); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_annos. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ANNOS); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ANNOS_PLACEATTRIBUTE); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_items_annos. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ITEMS_ANNOS); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_ITEMSANNOS_PLACEATTRIBUTE); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_meta. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_META); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_places_metadata + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_IDX_MOZ_PLACES_METADATA_PLACECREATED); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_METADATA_REFERRER); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_places_metadata_search_queries + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SEARCH_QUERIES); + NS_ENSURE_SUCCESS(rv, rv); + + // moz_previews_tombstones + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PREVIEWS_TOMBSTONES); + NS_ENSURE_SUCCESS(rv, rv); + + // The bookmarks roots get initialized in CheckRoots(). + } + + // Set the schema version to the current one. + rv = mMainConn->SetSchemaVersion(DATABASE_SCHEMA_VERSION); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + // ANY FAILURE IN THIS METHOD WILL CAUSE US TO MARK THE DATABASE AS CORRUPT + // AND TRY TO REPLACE IT. + // DO NOT PUT HERE ANYTHING THAT IS NOT RELATED TO INITIALIZATION OR MODIFYING + // THE DISK DATABASE. + + return NS_OK; +} + +nsresult Database::CheckRoots() { + MOZ_ASSERT(NS_IsMainThread()); + + // If the database has just been created, skip straight to the part where + // we create the roots. + if (mDatabaseStatus == nsINavHistoryService::DATABASE_STATUS_CREATE) { + return EnsureBookmarkRoots(0, /* shouldReparentRoots */ false); + } + + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + nsLiteralCString("SELECT guid, id, position, parent FROM moz_bookmarks " + "WHERE guid IN ( " + "'" ROOT_GUID "', '" MENU_ROOT_GUID + "', '" TOOLBAR_ROOT_GUID "', " + "'" TAGS_ROOT_GUID "', '" UNFILED_ROOT_GUID + "', '" MOBILE_ROOT_GUID "' )"), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult; + nsAutoCString guid; + int32_t maxPosition = 0; + bool shouldReparentRoots = false; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + rv = stmt->GetUTF8String(0, guid); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t parentId = stmt->AsInt64(3); + + if (guid.EqualsLiteral(ROOT_GUID)) { + mRootId = stmt->AsInt64(1); + shouldReparentRoots |= parentId != 0; + } else { + maxPosition = std::max(stmt->AsInt32(2), maxPosition); + + if (guid.EqualsLiteral(MENU_ROOT_GUID)) { + mMenuRootId = stmt->AsInt64(1); + } else if (guid.EqualsLiteral(TOOLBAR_ROOT_GUID)) { + mToolbarRootId = stmt->AsInt64(1); + } else if (guid.EqualsLiteral(TAGS_ROOT_GUID)) { + mTagsRootId = stmt->AsInt64(1); + } else if (guid.EqualsLiteral(UNFILED_ROOT_GUID)) { + mUnfiledRootId = stmt->AsInt64(1); + } else if (guid.EqualsLiteral(MOBILE_ROOT_GUID)) { + mMobileRootId = stmt->AsInt64(1); + } + shouldReparentRoots |= parentId != mRootId; + } + } + + rv = EnsureBookmarkRoots(maxPosition + 1, shouldReparentRoots); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::EnsureBookmarkRoots(const int32_t startPosition, + bool shouldReparentRoots) { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + if (mRootId < 1) { + // The first root's title is an empty string. + rv = CreateRoot(mMainConn, "places"_ns, "root________"_ns, ""_ns, 0, + mRootId); + + if (NS_FAILED(rv)) return rv; + } + + int32_t position = startPosition; + + // For the other roots, the UI doesn't rely on the value in the database, so + // just set it to something simple to make it easier for humans to read. + if (mMenuRootId < 1) { + rv = CreateRoot(mMainConn, "menu"_ns, "menu________"_ns, "menu"_ns, + position, mMenuRootId); + if (NS_FAILED(rv)) return rv; + position++; + } + + if (mToolbarRootId < 1) { + rv = CreateRoot(mMainConn, "toolbar"_ns, "toolbar_____"_ns, "toolbar"_ns, + position, mToolbarRootId); + if (NS_FAILED(rv)) return rv; + position++; + } + + if (mTagsRootId < 1) { + rv = CreateRoot(mMainConn, "tags"_ns, "tags________"_ns, "tags"_ns, + position, mTagsRootId); + if (NS_FAILED(rv)) return rv; + position++; + } + + if (mUnfiledRootId < 1) { + rv = CreateRoot(mMainConn, "unfiled"_ns, "unfiled_____"_ns, "unfiled"_ns, + position, mUnfiledRootId); + if (NS_FAILED(rv)) return rv; + position++; + } + + if (mMobileRootId < 1) { + int64_t mobileRootId = CreateMobileRoot(); + if (mobileRootId <= 0) return NS_ERROR_FAILURE; + { + nsCOMPtr mobileRootSyncStatusStmt; + rv = mMainConn->CreateStatement( + nsLiteralCString("UPDATE moz_bookmarks SET syncStatus = " + ":sync_status WHERE id = :id"), + getter_AddRefs(mobileRootSyncStatusStmt)); + if (NS_FAILED(rv)) return rv; + + rv = mobileRootSyncStatusStmt->BindInt32ByName( + "sync_status"_ns, nsINavBookmarksService::SYNC_STATUS_NEW); + if (NS_FAILED(rv)) return rv; + rv = mobileRootSyncStatusStmt->BindInt64ByName("id"_ns, mobileRootId); + if (NS_FAILED(rv)) return rv; + + rv = mobileRootSyncStatusStmt->Execute(); + if (NS_FAILED(rv)) return rv; + + mMobileRootId = mobileRootId; + } + } + + if (!shouldReparentRoots) { + return NS_OK; + } + + // At least one root had the wrong parent, so we need to ensure that + // all roots are parented correctly, fix their positions, and bump the + // Sync change counter. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "CREATE TEMP TRIGGER moz_ensure_bookmark_roots_trigger " + "AFTER UPDATE OF parent ON moz_bookmarks FOR EACH ROW " + "WHEN OLD.parent <> NEW.parent " + "BEGIN " + "UPDATE moz_bookmarks SET " + "syncChangeCounter = syncChangeCounter + 1 " + "WHERE id IN (OLD.parent, NEW.parent, NEW.id); " + + "UPDATE moz_bookmarks SET " + "position = position - 1 " + "WHERE parent = OLD.parent AND position >= OLD.position; " + + // Fix the positions of the root's old siblings. Since we've already + // moved the root, we need to exclude it from the subquery. + "UPDATE moz_bookmarks SET " + "position = IFNULL((SELECT MAX(position) + 1 FROM moz_bookmarks " + "WHERE parent = NEW.parent AND " + "id <> NEW.id), 0)" + "WHERE id = NEW.id; " + "END")); + if (NS_FAILED(rv)) return rv; + auto guard = MakeScopeExit([&]() { + Unused << mMainConn->ExecuteSimpleSQL( + "DROP TRIGGER moz_ensure_bookmark_roots_trigger"_ns); + }); + + nsCOMPtr reparentStmt; + rv = mMainConn->CreateStatement( + nsLiteralCString( + "UPDATE moz_bookmarks SET " + "parent = CASE id WHEN :root_id THEN 0 ELSE :root_id END " + "WHERE id IN (:root_id, :menu_root_id, :toolbar_root_id, " + ":tags_root_id, " + ":unfiled_root_id, :mobile_root_id)"), + getter_AddRefs(reparentStmt)); + if (NS_FAILED(rv)) return rv; + + rv = reparentStmt->BindInt64ByName("root_id"_ns, mRootId); + if (NS_FAILED(rv)) return rv; + rv = reparentStmt->BindInt64ByName("menu_root_id"_ns, mMenuRootId); + if (NS_FAILED(rv)) return rv; + rv = reparentStmt->BindInt64ByName("toolbar_root_id"_ns, mToolbarRootId); + if (NS_FAILED(rv)) return rv; + rv = reparentStmt->BindInt64ByName("tags_root_id"_ns, mTagsRootId); + if (NS_FAILED(rv)) return rv; + rv = reparentStmt->BindInt64ByName("unfiled_root_id"_ns, mUnfiledRootId); + if (NS_FAILED(rv)) return rv; + rv = reparentStmt->BindInt64ByName("mobile_root_id"_ns, mMobileRootId); + if (NS_FAILED(rv)) return rv; + + rv = reparentStmt->Execute(); + if (NS_FAILED(rv)) return rv; + + return NS_OK; +} + +nsresult Database::InitFunctions() { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = GetUnreversedHostFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = MatchAutoCompleteFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = CalculateFrecencyFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = GenerateGUIDFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = IsValidGUIDFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = FixupURLFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = StoreLastInsertedIdFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = HashFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = GetQueryParamFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = GetPrefixFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = GetHostAndPortFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = StripPrefixAndUserinfoFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = IsFrecencyDecayingFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = NoteSyncChangeFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = InvalidateDaysOfHistoryFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = MD5HexFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + rv = SetShouldStartFrecencyRecalculationFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + + if (StaticPrefs::places_frecency_pages_alternative_featureGate_AtStartup()) { + rv = CalculateAltFrecencyFunction::create(mMainConn); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult Database::InitTempEntities() { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = + mMainConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + // Add the triggers that update the moz_origins table as necessary. + rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSINSERT_TEMP); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_UPDATEORIGINSINSERT_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSDELETE_TEMP); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_UPDATEORIGINSDELETE_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + if (Preferences::GetBool(PREF_PREVIEWS_ENABLED, false)) { + rv = mMainConn->ExecuteSimpleSQL( + CREATE_PLACES_AFTERDELETE_WPREVIEWS_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + } else { + rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = mMainConn->ExecuteSimpleSQL(CREATE_UPDATEORIGINSUPDATE_TEMP); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_UPDATEORIGINSUPDATE_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_AFTERUPDATE_FRECENCY_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_PLACES_AFTERUPDATE_RECALC_FRECENCY_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mMainConn->ExecuteSimpleSQL( + CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mMainConn->ExecuteSimpleSQL( + CREATE_KEYWORDS_FOREIGNCOUNT_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_KEYWORDS_FOREIGNCOUNT_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + CREATE_KEYWORDS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = + mMainConn->ExecuteSimpleSQL(CREATE_BOOKMARKS_DELETED_AFTERINSERT_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + rv = + mMainConn->ExecuteSimpleSQL(CREATE_BOOKMARKS_DELETED_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mMainConn->ExecuteSimpleSQL(CREATE_PLACES_METADATA_AFTERDELETE_TRIGGER); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV44Up() { + // We need to remove any non-builtin roots and their descendants. + + // Install a temp trigger to clean up linked tables when the main + // bookmarks are deleted. + nsresult rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "CREATE TEMP TRIGGER moz_migrate_bookmarks_trigger " + "AFTER DELETE ON moz_bookmarks FOR EACH ROW " + "BEGIN " + // Insert tombstones. + "INSERT OR IGNORE INTO moz_bookmarks_deleted (guid, dateRemoved) " + "VALUES (OLD.guid, strftime('%s', 'now', 'localtime', 'utc') * 1000000); " + // Remove old annotations for the bookmarks. + "DELETE FROM moz_items_annos " + "WHERE item_id = OLD.id; " + // Decrease the foreign_count in moz_places. + "UPDATE moz_places " + "SET foreign_count = foreign_count - 1 " + "WHERE id = OLD.fk; " + "END ")); + if (NS_FAILED(rv)) return rv; + + // This trigger listens for moz_places deletes, and updates moz_annos and + // moz_keywords accordingly. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "CREATE TEMP TRIGGER moz_migrate_annos_trigger " + "AFTER UPDATE ON moz_places FOR EACH ROW " + // Only remove from moz_places if we don't have any remaining keywords + // pointing to this place, and it hasn't been visited. Note: orphan + // keywords are tidied up below. + "WHEN NEW.visit_count = 0 AND " + " NEW.foreign_count = (SELECT COUNT(*) FROM moz_keywords WHERE place_id " + "= NEW.id) " + "BEGIN " + // No more references to the place, so we can delete the place itself. + "DELETE FROM moz_places " + "WHERE id = NEW.id; " + // Delete annotations relating to the place. + "DELETE FROM moz_annos " + "WHERE place_id = NEW.id; " + // Delete keywords relating to the place. + "DELETE FROM moz_keywords " + "WHERE place_id = NEW.id; " + "END ")); + if (NS_FAILED(rv)) return rv; + + // Listens to moz_keyword deletions, to ensure moz_places gets the + // foreign_count updated corrrectly. + rv = mMainConn->ExecuteSimpleSQL( + nsLiteralCString("CREATE TEMP TRIGGER moz_migrate_keyword_trigger " + "AFTER DELETE ON moz_keywords FOR EACH ROW " + "BEGIN " + // If we remove a keyword, then reduce the foreign_count. + "UPDATE moz_places " + "SET foreign_count = foreign_count - 1 " + "WHERE id = OLD.place_id; " + "END ")); + if (NS_FAILED(rv)) return rv; + + // First of all, find the non-builtin roots. + nsCOMPtr deleteStmt; + rv = mMainConn->CreateStatement( + nsLiteralCString("WITH RECURSIVE " + "itemsToRemove(id, guid) AS ( " + "SELECT b.id, b.guid FROM moz_bookmarks b " + "JOIN moz_bookmarks p ON b.parent = p.id " + "WHERE p.guid = 'root________' AND " + "b.guid NOT IN ('menu________', 'toolbar_____', " + "'tags________', 'unfiled_____', 'mobile______') " + "UNION ALL " + "SELECT b.id, b.guid FROM moz_bookmarks b " + "JOIN itemsToRemove d ON d.id = b.parent " + "WHERE b.guid NOT IN ('menu________', 'toolbar_____', " + "'tags________', 'unfiled_____', 'mobile______') " + ") " + "DELETE FROM moz_bookmarks " + "WHERE id IN (SELECT id FROM itemsToRemove) "), + getter_AddRefs(deleteStmt)); + if (NS_FAILED(rv)) return rv; + + rv = deleteStmt->Execute(); + if (NS_FAILED(rv)) return rv; + + // Before we remove the triggers, check for keywords attached to places which + // no longer have a bookmark to them. We do this before removing the triggers, + // so that we can make use of the keyword trigger to update the counts in + // moz_places. + rv = mMainConn->ExecuteSimpleSQL( + nsLiteralCString("DELETE FROM moz_keywords WHERE place_id IN ( " + "SELECT h.id FROM moz_keywords k " + "JOIN moz_places h ON h.id = k.place_id " + "GROUP BY place_id HAVING h.foreign_count = count(*) " + ")")); + if (NS_FAILED(rv)) return rv; + + // Now remove the temp triggers. + rv = mMainConn->ExecuteSimpleSQL( + "DROP TRIGGER moz_migrate_bookmarks_trigger "_ns); + if (NS_FAILED(rv)) return rv; + rv = + mMainConn->ExecuteSimpleSQL("DROP TRIGGER moz_migrate_annos_trigger "_ns); + if (NS_FAILED(rv)) return rv; + rv = mMainConn->ExecuteSimpleSQL( + "DROP TRIGGER moz_migrate_keyword_trigger "_ns); + if (NS_FAILED(rv)) return rv; + + // Cleanup any orphan annotation attributes. + rv = mMainConn->ExecuteSimpleSQL( + nsLiteralCString("DELETE FROM moz_anno_attributes WHERE id IN ( " + "SELECT id FROM moz_anno_attributes n " + "EXCEPT " + "SELECT DISTINCT anno_attribute_id FROM moz_annos " + "EXCEPT " + "SELECT DISTINCT anno_attribute_id FROM moz_items_annos " + ")")); + if (NS_FAILED(rv)) return rv; + + return NS_OK; +} + +nsresult Database::MigrateV45Up() { + nsCOMPtr metaTableStmt; + nsresult rv = mMainConn->CreateStatement("SELECT 1 FROM moz_meta"_ns, + getter_AddRefs(metaTableStmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_META); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::MigrateV46Up() { + // Convert the existing queries. For simplicity we assume the user didn't + // edit these queries, and just do a 1:1 conversion. + nsresult rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "UPDATE moz_places " + "SET url = IFNULL('place:tag=' || ( " + "SELECT title FROM moz_bookmarks " + "WHERE id = CAST(get_query_param(substr(url, 7), 'folder') AS INT) " + "), url) " + "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi') " + "AND url LIKE '%type=7%' " + "AND EXISTS(SELECT 1 FROM moz_bookmarks " + "WHERE id = CAST(get_query_param(substr(url, 7), 'folder') AS INT)) ")); + + // Recalculate hashes for all tag queries. + rv = mMainConn->ExecuteSimpleSQL( + nsLiteralCString("UPDATE moz_places SET url_hash = hash(url) " + "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi') " + "AND url LIKE '%tag=%' ")); + NS_ENSURE_SUCCESS(rv, rv); + + // Update Sync fields for all tag queries. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 " + "WHERE fk IN ( " + "SELECT id FROM moz_places " + "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi') " + "AND url LIKE '%tag=%' " + ") ")); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult Database::MigrateV47Up() { + // v46 may have mistakenly set some url to NULL, we must fix those. + // Since the original url was an invalid query, we replace NULLs with an + // empty query. + nsresult rv = mMainConn->ExecuteSimpleSQL( + nsLiteralCString("UPDATE moz_places " + "SET url = 'place:excludeItems=1', url_hash = " + "hash('place:excludeItems=1') " + "WHERE url ISNULL ")); + NS_ENSURE_SUCCESS(rv, rv); + // Update Sync fields for these queries. + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 " + "WHERE fk IN ( " + "SELECT id FROM moz_places " + "WHERE url_hash = hash('place:excludeItems=1') " + "AND url = 'place:excludeItems=1' " + ") ")); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult Database::MigrateV48Up() { + // Create and populate moz_origins. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement("SELECT * FROM moz_origins; "_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_ORIGINS); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "INSERT OR IGNORE INTO moz_origins (prefix, host, frecency) " + "SELECT get_prefix(url), get_host_and_port(url), -1 " + "FROM moz_places; ")); + NS_ENSURE_SUCCESS(rv, rv); + + // Add and populate moz_places.origin_id. + rv = mMainConn->CreateStatement("SELECT origin_id FROM moz_places; "_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "ALTER TABLE moz_places " + "ADD COLUMN origin_id INTEGER REFERENCES moz_origins(id); ")); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ORIGIN_ID); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "UPDATE moz_places " + "SET origin_id = ( " + "SELECT id FROM moz_origins " + "WHERE prefix = get_prefix(url) AND host = get_host_and_port(url) " + "); ")); + NS_ENSURE_SUCCESS(rv, rv); + + // From this point on, nobody should use moz_hosts again. Empty it so that we + // don't leak the user's history, but don't remove it yet so that the user can + // downgrade. + // This can fail, if moz_hosts doesn't exist anymore, that is what happens in + // case of downgrade+upgrade. + Unused << mMainConn->ExecuteSimpleSQL("DELETE FROM moz_hosts; "_ns); + + return NS_OK; +} + +nsresult Database::MigrateV49Up() { + // These hidden preferences were added along with the v48 migration as part of + // the frecency stats implementation but are now replaced with entries in the + // moz_meta table. + Unused << Preferences::ClearUser("places.frecency.stats.count"); + Unused << Preferences::ClearUser("places.frecency.stats.sum"); + Unused << Preferences::ClearUser("places.frecency.stats.sumOfSquares"); + return NS_OK; +} + +nsresult Database::MigrateV50Up() { + // Convert the existing queries. We don't have REGEX available, so the + // simplest thing to do is to pull the urls out, and process them manually. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + nsLiteralCString("SELECT id, url FROM moz_places " + "WHERE url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi') " + "AND url LIKE '%folder=%' "), + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) return rv; + + AutoTArray, 32> placeURLs; + + bool hasMore = false; + nsCString url; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) { + int64_t placeId; + rv = stmt->GetInt64(0, &placeId); + if (NS_FAILED(rv)) return rv; + rv = stmt->GetUTF8String(1, url); + if (NS_FAILED(rv)) return rv; + + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + placeURLs.AppendElement(std::make_pair(placeId, url)); + } + + if (placeURLs.IsEmpty()) { + return NS_OK; + } + + int64_t placeId; + for (uint32_t i = 0; i < placeURLs.Length(); ++i) { + placeId = placeURLs[i].first; + url = placeURLs[i].second; + + rv = ConvertOldStyleQuery(url); + // Something bad happened, and we can't convert it, so just continue. + if (NS_WARN_IF(NS_FAILED(rv))) { + continue; + } + + nsCOMPtr updateStmt; + rv = mMainConn->CreateStatement( + nsLiteralCString("UPDATE moz_places " + "SET url = :url, url_hash = hash(:url) " + "WHERE id = :placeId "), + getter_AddRefs(updateStmt)); + if (NS_FAILED(rv)) return rv; + + rv = URIBinder::Bind(updateStmt, "url"_ns, url); + if (NS_FAILED(rv)) return rv; + rv = updateStmt->BindInt64ByName("placeId"_ns, placeId); + if (NS_FAILED(rv)) return rv; + + rv = updateStmt->Execute(); + if (NS_FAILED(rv)) return rv; + + // Update Sync fields for these queries. + nsCOMPtr syncStmt; + rv = mMainConn->CreateStatement( + nsLiteralCString("UPDATE moz_bookmarks SET syncChangeCounter = " + "syncChangeCounter + 1 " + "WHERE fk = :placeId "), + getter_AddRefs(syncStmt)); + if (NS_FAILED(rv)) return rv; + + rv = syncStmt->BindInt64ByName("placeId"_ns, placeId); + if (NS_FAILED(rv)) return rv; + + rv = syncStmt->Execute(); + if (NS_FAILED(rv)) return rv; + } + + return NS_OK; +} + +nsresult Database::MigrateV51Up() { + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + nsLiteralCString("SELECT b.guid FROM moz_anno_attributes n " + "JOIN moz_items_annos a ON n.id = a.anno_attribute_id " + "JOIN moz_bookmarks b ON a.item_id = b.id " + "WHERE n.name = :anno_name ORDER BY a.content DESC"), + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + MOZ_ASSERT(false, + "Should succeed unless item annotations table has been removed"); + return NS_OK; + }; + + rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO); + NS_ENSURE_SUCCESS(rv, rv); + + JSONStringWriteFunc json; + JSONWriter jw{json}; + jw.StartArrayProperty(nullptr, JSONWriter::SingleLineStyle); + + bool hasAtLeastOne = false; + bool hasMore = false; + uint32_t length; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) { + hasAtLeastOne = true; + const char* stmtString = stmt->AsSharedUTF8String(0, &length); + jw.StringElement(Span(stmtString, length)); + } + jw.EndArray(); + + // If we don't have any, just abort early and save the extra work. + if (!hasAtLeastOne) { + return NS_OK; + } + + rv = mMainConn->CreateStatement( + nsLiteralCString("INSERT OR REPLACE INTO moz_meta " + "VALUES (:key, :value) "), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->BindUTF8StringByName("key"_ns, LAST_USED_FOLDERS_META_KEY); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("value"_ns, json.StringCRef()); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Clean up the now redundant annotations. + rv = mMainConn->CreateStatement( + nsLiteralCString( + "DELETE FROM moz_items_annos WHERE anno_attribute_id = " + "(SELECT id FROM moz_anno_attributes WHERE name = :anno_name) "), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mMainConn->CreateStatement( + nsLiteralCString( + "DELETE FROM moz_anno_attributes WHERE name = :anno_name "), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("anno_name"_ns, LAST_USED_ANNO); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +namespace { + +class MigrateV52OriginFrecenciesRunnable final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + explicit MigrateV52OriginFrecenciesRunnable(mozIStorageConnection* aDBConn); + + private: + nsCOMPtr mDBConn; +}; + +MigrateV52OriginFrecenciesRunnable::MigrateV52OriginFrecenciesRunnable( + mozIStorageConnection* aDBConn) + : Runnable("places::MigrateV52OriginFrecenciesRunnable"), + mDBConn(aDBConn) {} + +NS_IMETHODIMP +MigrateV52OriginFrecenciesRunnable::Run() { + if (NS_IsMainThread()) { + // Migration done. Clear the pref. + Unused << Preferences::ClearUser(PREF_MIGRATE_V52_ORIGIN_FRECENCIES); + + // Now that frecencies have been migrated, recalculate the origin frecency + // stats. + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(navHistory); + nsresult rv = navHistory->RecalculateOriginFrecencyStats(nullptr); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + // We do the work in chunks, or the wal journal may grow too much. + nsresult rv = mDBConn->ExecuteSimpleSQL(nsLiteralCString( + "UPDATE moz_origins " + "SET frecency = ( " + "SELECT CAST(TOTAL(frecency) AS INTEGER) " + "FROM moz_places " + "WHERE frecency > 0 AND moz_places.origin_id = moz_origins.id " + ") " + "WHERE id IN ( " + "SELECT id " + "FROM moz_origins " + "WHERE frecency < 0 " + "LIMIT 400 " + ") ")); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr selectStmt; + rv = mDBConn->CreateStatement(nsLiteralCString("SELECT 1 " + "FROM moz_origins " + "WHERE frecency < 0 " + "LIMIT 1 "), + getter_AddRefs(selectStmt)); + NS_ENSURE_SUCCESS(rv, rv); + bool hasResult = false; + rv = selectStmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + if (hasResult) { + // There are more results to handle. Re-dispatch to the same thread for the + // next chunk. + return NS_DispatchToCurrentThread(this); + } + + // Re-dispatch to the main-thread to flip the migration pref. + return NS_DispatchToMainThread(this); +} + +} // namespace + +void Database::MigrateV52OriginFrecencies() { + MOZ_ASSERT(NS_IsMainThread()); + + if (!Preferences::GetBool(PREF_MIGRATE_V52_ORIGIN_FRECENCIES)) { + // The migration has already been completed. + return; + } + + RefPtr runnable( + new MigrateV52OriginFrecenciesRunnable(mMainConn)); + nsCOMPtr target(do_GetInterface(mMainConn)); + MOZ_ASSERT(target); + if (target) { + Unused << target->Dispatch(runnable, NS_DISPATCH_NORMAL); + } +} + +nsresult Database::MigrateV52Up() { + // Before this migration, moz_origin.frecency is the max frecency of all + // places with the origin. After this migration, it's the sum of frecencies + // of all places with the origin. + // + // Setting this pref will cause InitSchema to begin async migration, via + // MigrateV52OriginFrecencies. When that migration is done, origin frecency + // stats are recalculated (see MigrateV52OriginFrecenciesRunnable::Run). + Unused << Preferences::SetBool(PREF_MIGRATE_V52_ORIGIN_FRECENCIES, true); + + // Set all origin frecencies to -1 so that MigrateV52OriginFrecenciesRunnable + // will migrate them. + nsresult rv = + mMainConn->ExecuteSimpleSQL("UPDATE moz_origins SET frecency = -1 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + + // This migration also renames these moz_meta keys that keep track of frecency + // stats. (That happens when stats are recalculated.) Delete the old ones. + rv = + mMainConn->ExecuteSimpleSQL(nsLiteralCString("DELETE FROM moz_meta " + "WHERE key IN ( " + "'frecency_count', " + "'frecency_sum', " + "'frecency_sum_of_squares' " + ") ")); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV53Up() { + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement("SELECT 1 FROM moz_items_annos"_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + // Likely we removed the table. + return NS_OK; + } + + // Remove all item annotations but SYNC_PARENT_ANNO. + rv = mMainConn->CreateStatement( + nsLiteralCString( + "DELETE FROM moz_items_annos " + "WHERE anno_attribute_id NOT IN ( " + " SELECT id FROM moz_anno_attributes WHERE name = :anno_name " + ") "), + getter_AddRefs(stmt)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("anno_name"_ns, + nsLiteralCString(SYNC_PARENT_ANNO)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "DELETE FROM moz_anno_attributes WHERE id IN ( " + " SELECT id FROM moz_anno_attributes " + " EXCEPT " + " SELECT DISTINCT anno_attribute_id FROM moz_annos " + " EXCEPT " + " SELECT DISTINCT anno_attribute_id FROM moz_items_annos " + ")")); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV54Up() { + // Add an expiration column to moz_icons_to_pages. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT expire_ms FROM moz_icons_to_pages"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_icons_to_pages " + "ADD COLUMN expire_ms INTEGER NOT NULL DEFAULT 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Set all the zero-ed entries as expired today, they won't be removed until + // the next related page load. + rv = mMainConn->ExecuteSimpleSQL( + "UPDATE moz_icons_to_pages " + "SET expire_ms = strftime('%s','now','localtime','start " + "of day','utc') * 1000 " + "WHERE expire_ms = 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV55Up() { + // Add places metadata tables. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT id FROM moz_places_metadata"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + // Create the tables. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA); + NS_ENSURE_SUCCESS(rv, rv); + // moz_places_metadata_search_queries. + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PLACES_METADATA_SEARCH_QUERIES); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult Database::MigrateV56Up() { + // Add places metadata (place_id, created_at) index. + return mMainConn->ExecuteSimpleSQL( + CREATE_IDX_MOZ_PLACES_METADATA_PLACECREATED); +} + +nsresult Database::MigrateV57Up() { + // Add the scrolling columns to the metadata. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT scrolling_time FROM moz_places_metadata"_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places_metadata " + "ADD COLUMN scrolling_time INTEGER NOT NULL DEFAULT 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = mMainConn->CreateStatement( + "SELECT scrolling_distance FROM moz_places_metadata"_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places_metadata " + "ADD COLUMN scrolling_distance INTEGER NOT NULL DEFAULT 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::MigrateV60Up() { + // Add the site_name column to moz_places. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT site_name FROM moz_places"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places ADD COLUMN site_name TEXT"_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::MigrateV61Up() { + // Add previews tombstones table if necessary. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT hash FROM moz_previews_tombstones"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL(CREATE_MOZ_PREVIEWS_TOMBSTONES); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::MigrateV67Up() { + // Align all input field in moz_inputhistory to lowercase. If there are + // multiple records that expresses the same input, use maximum use_count from + // them to carry on the experience of the past. + nsCOMPtr stmt; + nsresult rv = mMainConn->ExecuteSimpleSQL( + "INSERT INTO moz_inputhistory " + "SELECT place_id, LOWER(input), use_count FROM moz_inputhistory " + " WHERE LOWER(input) <> input " + "ON CONFLICT DO " + " UPDATE SET use_count = MAX(use_count, EXCLUDED.use_count)"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DELETE FROM moz_inputhistory WHERE LOWER(input) <> input"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV69Up() { + // Add source and annotation column to places table. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT source FROM moz_historyvisits"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_historyvisits " + "ADD COLUMN source INTEGER DEFAULT 0 NOT NULL"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_historyvisits " + "ADD COLUMN triggeringPlaceId INTEGER"_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult Database::MigrateV70Up() { + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT recalc_frecency FROM moz_places LIMIT 1 "_ns, + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + // Add recalc_frecency column, indicating frecency has to be recalculated. + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places " + "ADD COLUMN recalc_frecency INTEGER NOT NULL DEFAULT 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + + // We must do the following updates regardless, for downgrade/upgrade cases. + + // moz_origins frecency is, at the time of this migration, the sum of all the + // positive frecencies of pages linked to that origin. Frecencies that were + // set to negative to request recalculation are thus not accounted for, and + // since we're about to flip them to positive we should add them to their + // origin. Then we must also update origins stats. + // We ignore frecency = -1 because it's just an indication to recalculate + // frecency and not an actual frecency value that was flipped, thus it would + // not make sense to count it for the origin. + rv = mMainConn->ExecuteSimpleSQL( + "UPDATE moz_origins " + "SET frecency = frecency + abs_frecency " + "FROM (SELECT origin_id, ABS(frecency) AS abs_frecency FROM moz_places " + "WHERE frecency < -1) AS places " + "WHERE moz_origins.id = places.origin_id"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = RecalculateOriginFrecencyStatsInternal(); + NS_ENSURE_SUCCESS(rv, rv); + + // Now set recalc_frecency = 1 and positive frecency to any page having a + // negative frecency. + // Note we don't flip frecency = -1, since we skipped it above when updating + // origins, and it remains an acceptable value yet, until the recalculation. + rv = mMainConn->ExecuteSimpleSQL( + "UPDATE moz_places " + "SET recalc_frecency = 1, " + " frecency = CASE WHEN frecency = -1 THEN -1 ELSE ABS(frecency) END " + "WHERE frecency < 0 "_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV71Up() { + // Fix the foreign counts. We ignore failures as the tables may not exist. + mMainConn->ExecuteSimpleSQL( + "UPDATE moz_places " + "SET foreign_count = foreign_count - 1 " + "WHERE id in (SELECT place_id FROM moz_places_metadata_snapshots)"_ns); + mMainConn->ExecuteSimpleSQL( + "UPDATE moz_places " + "SET foreign_count = foreign_count - 1 " + "WHERE id in (SELECT place_id FROM moz_session_to_places)"_ns); + + // Remove unused snapshots and session tables and indexes. + nsresult rv = mMainConn->ExecuteSimpleSQL( + "DROP INDEX IF EXISTS moz_places_metadata_snapshots_pinnedindex"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP INDEX IF EXISTS moz_places_metadata_snapshots_extra_typeindex"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_places_metadata_groups_to_snapshots"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_places_metadata_snapshots_groups"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_places_metadata_snapshots_extra"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_places_metadata_snapshots"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_session_to_places"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "DROP TABLE IF EXISTS moz_session_metadata"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult Database::MigrateV72Up() { + // Recalculate frecency of unvisited bookmarks. + nsresult rv = mMainConn->ExecuteSimpleSQL( + "UPDATE moz_places " + "SET recalc_frecency = 1 " + "WHERE foreign_count > 0 AND visit_count = 0"_ns); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult Database::MigrateV73Up() { + // Add recalc_frecency, alt_frecency and recalc_alt_frecency to moz_origins. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT recalc_frecency FROM moz_origins"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_origins " + "ADD COLUMN recalc_frecency INTEGER NOT NULL DEFAULT 0"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_origins " + "ADD COLUMN alt_frecency INTEGER"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_origins " + "ADD COLUMN recalc_alt_frecency INTEGER NOT NULL DEFAULT 0"_ns); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::MigrateV74Up() { + // Add alt_frecency and recalc_alt_frecency to moz_places. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + "SELECT alt_frecency FROM moz_places"_ns, getter_AddRefs(stmt)); + if (NS_FAILED(rv)) { + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places " + "ADD COLUMN alt_frecency INTEGER"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL( + "ALTER TABLE moz_places " + "ADD COLUMN recalc_alt_frecency INTEGER NOT NULL DEFAULT 0"_ns); + NS_ENSURE_SUCCESS(rv, rv); + rv = mMainConn->ExecuteSimpleSQL(CREATE_IDX_MOZ_PLACES_ALT_FRECENCY); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult Database::RecalculateOriginFrecencyStatsInternal() { + return mMainConn->ExecuteSimpleSQL(nsLiteralCString( + "INSERT OR REPLACE INTO moz_meta(key, value) VALUES " + "( " + "'" MOZ_META_KEY_ORIGIN_FRECENCY_COUNT + "' , " + "(SELECT COUNT(*) FROM moz_origins WHERE frecency > 0) " + "), " + "( " + "'" MOZ_META_KEY_ORIGIN_FRECENCY_SUM + "', " + "(SELECT TOTAL(frecency) FROM moz_origins WHERE frecency > 0) " + "), " + "( " + "'" MOZ_META_KEY_ORIGIN_FRECENCY_SUM_OF_SQUARES + "' , " + "(SELECT TOTAL(frecency * frecency) FROM moz_origins WHERE frecency > 0) " + ") ")); +} + +nsresult Database::ConvertOldStyleQuery(nsCString& aURL) { + AutoTArray tokens; + nsresult rv = TokenizeQueryString(aURL, &tokens); + NS_ENSURE_SUCCESS(rv, rv); + + AutoTArray newTokens; + bool invalid = false; + nsAutoCString guid; + + for (uint32_t j = 0; j < tokens.Length(); ++j) { + const QueryKeyValuePair& kvp = tokens[j]; + + if (!kvp.key.EqualsLiteral("folder")) { + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + newTokens.AppendElement(kvp); + continue; + } + + int64_t itemId = kvp.value.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + // We have the folder's ID, now to find its GUID. + nsCOMPtr stmt; + nsresult rv = mMainConn->CreateStatement( + nsLiteralCString("SELECT guid FROM moz_bookmarks " + "WHERE id = :itemId "), + getter_AddRefs(stmt)); + if (NS_FAILED(rv)) return rv; + + rv = stmt->BindInt64ByName("itemId"_ns, itemId); + if (NS_FAILED(rv)) return rv; + + bool hasMore = false; + if (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) { + rv = stmt->GetUTF8String(0, guid); + if (NS_FAILED(rv)) return rv; + } + } else if (kvp.value.EqualsLiteral("PLACES_ROOT")) { + guid = nsLiteralCString(ROOT_GUID); + } else if (kvp.value.EqualsLiteral("BOOKMARKS_MENU")) { + guid = nsLiteralCString(MENU_ROOT_GUID); + } else if (kvp.value.EqualsLiteral("TAGS")) { + guid = nsLiteralCString(TAGS_ROOT_GUID); + } else if (kvp.value.EqualsLiteral("UNFILED_BOOKMARKS")) { + guid = nsLiteralCString(UNFILED_ROOT_GUID); + } else if (kvp.value.EqualsLiteral("TOOLBAR")) { + guid = nsLiteralCString(TOOLBAR_ROOT_GUID); + } else if (kvp.value.EqualsLiteral("MOBILE_BOOKMARKS")) { + guid = nsLiteralCString(MOBILE_ROOT_GUID); + } + + QueryKeyValuePair* newPair; + if (guid.IsEmpty()) { + // This is invalid, so we'll change this key/value pair to something else + // so that the query remains a valid url. + newPair = new QueryKeyValuePair("invalidOldParentId"_ns, kvp.value); + invalid = true; + } else { + newPair = new QueryKeyValuePair("parent"_ns, guid); + } + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + newTokens.AppendElement(*newPair); + delete newPair; + } + + if (invalid) { + // One or more of the folders don't exist, replace with an empty query. + newTokens.AppendElement(QueryKeyValuePair("excludeItems"_ns, "1"_ns)); + } + + TokensToQueryString(newTokens, aURL); + return NS_OK; +} + +int64_t Database::CreateMobileRoot() { + MOZ_ASSERT(NS_IsMainThread()); + + // Create the mobile root, ignoring conflicts if one already exists (for + // example, if the user downgraded to an earlier release channel). + nsCOMPtr createStmt; + nsresult rv = mMainConn->CreateStatement( + nsLiteralCString( + "INSERT OR IGNORE INTO moz_bookmarks " + "(type, title, dateAdded, lastModified, guid, position, parent) " + "SELECT :item_type, :item_title, :timestamp, :timestamp, :guid, " + "IFNULL((SELECT MAX(position) + 1 FROM moz_bookmarks p WHERE " + "p.parent = b.id), 0), b.id " + "FROM moz_bookmarks b WHERE b.parent = 0"), + getter_AddRefs(createStmt)); + if (NS_FAILED(rv)) return -1; + + rv = createStmt->BindInt32ByName("item_type"_ns, + nsINavBookmarksService::TYPE_FOLDER); + if (NS_FAILED(rv)) return -1; + rv = createStmt->BindUTF8StringByName("item_title"_ns, + nsLiteralCString(MOBILE_ROOT_TITLE)); + if (NS_FAILED(rv)) return -1; + rv = createStmt->BindInt64ByName("timestamp"_ns, RoundedPRNow()); + if (NS_FAILED(rv)) return -1; + rv = createStmt->BindUTF8StringByName("guid"_ns, + nsLiteralCString(MOBILE_ROOT_GUID)); + if (NS_FAILED(rv)) return -1; + + rv = createStmt->Execute(); + if (NS_FAILED(rv)) return -1; + + // Find the mobile root ID. We can't use the last inserted ID because the + // root might already exist, and we ignore on conflict. + nsCOMPtr findIdStmt; + rv = mMainConn->CreateStatement( + "SELECT id FROM moz_bookmarks WHERE guid = :guid"_ns, + getter_AddRefs(findIdStmt)); + if (NS_FAILED(rv)) return -1; + + rv = findIdStmt->BindUTF8StringByName("guid"_ns, + nsLiteralCString(MOBILE_ROOT_GUID)); + if (NS_FAILED(rv)) return -1; + + bool hasResult = false; + rv = findIdStmt->ExecuteStep(&hasResult); + if (NS_FAILED(rv) || !hasResult) return -1; + + int64_t rootId; + rv = findIdStmt->GetInt64(0, &rootId); + if (NS_FAILED(rv)) return -1; + + return rootId; +} + +void Database::Shutdown() { + // As the last step in the shutdown path, finalize the database handle. + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(!mClosed); + + // Break cycles with the shutdown blockers. + mClientsShutdown = nullptr; + nsCOMPtr connectionShutdown = + std::move(mConnectionShutdown); + + if (!mMainConn) { + // The connection has never been initialized. Just mark it as closed. + mClosed = true; + (void)connectionShutdown->Complete(NS_OK, nullptr); + return; + } + +#ifdef DEBUG + { + bool hasResult; + nsCOMPtr stmt; + + // Sanity check for missing guids. + nsresult rv = + mMainConn->CreateStatement(nsLiteralCString("SELECT 1 " + "FROM moz_places " + "WHERE guid IS NULL "), + getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found a page without a GUID!"); + rv = mMainConn->CreateStatement(nsLiteralCString("SELECT 1 " + "FROM moz_bookmarks " + "WHERE guid IS NULL "), + getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found a bookmark without a GUID!"); + + // Sanity check for unrounded dateAdded and lastModified values (bug + // 1107308). + rv = mMainConn->CreateStatement( + nsLiteralCString( + "SELECT 1 " + "FROM moz_bookmarks " + "WHERE dateAdded % 1000 > 0 OR lastModified % 1000 > 0 LIMIT 1"), + getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found unrounded dates!"); + + // Sanity check url_hash + rv = mMainConn->CreateStatement( + "SELECT 1 FROM moz_places WHERE url_hash = 0"_ns, getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found a place without a hash!"); + + // Sanity check unique urls + rv = mMainConn->CreateStatement( + nsLiteralCString( + "SELECT 1 FROM moz_places GROUP BY url HAVING count(*) > 1 "), + getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found a duplicate url!"); + + // Sanity check NULL urls + rv = mMainConn->CreateStatement( + "SELECT 1 FROM moz_places WHERE url ISNULL "_ns, getter_AddRefs(stmt)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + rv = stmt->ExecuteStep(&hasResult); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + MOZ_ASSERT(!hasResult, "Found a NULL url!"); + } +#endif + + mMainThreadStatements.FinalizeStatements(); + mMainThreadAsyncStatements.FinalizeStatements(); + + RefPtr> event = + new FinalizeStatementCacheProxy( + mAsyncThreadStatements, NS_ISUPPORTS_CAST(nsIObserver*, this)); + DispatchToAsyncThread(event); + + mClosed = true; + + // Execute PRAGMA optimized as last step, this will ensure proper database + // performance across restarts. + nsCOMPtr ps; + MOZ_ALWAYS_SUCCEEDS(mMainConn->ExecuteSimpleSQLAsync( + "PRAGMA optimize(0x02)"_ns, nullptr, getter_AddRefs(ps))); + + if (NS_FAILED(mMainConn->AsyncClose(connectionShutdown))) { + mozilla::Unused << connectionShutdown->Complete(NS_ERROR_UNEXPECTED, + nullptr); + } + mMainConn = nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsIObserver + +NS_IMETHODIMP +Database::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + MOZ_ASSERT(NS_IsMainThread()); + if (strcmp(aTopic, TOPIC_PROFILE_CHANGE_TEARDOWN) == 0) { + // Tests simulating shutdown may cause multiple notifications. + if (PlacesShutdownBlocker::sIsStarted) { + return NS_OK; + } + + nsCOMPtr os = services::GetObserverService(); + NS_ENSURE_STATE(os); + + // If shutdown happens in the same mainthread loop as init, observers could + // handle the places-init-complete notification after xpcom-shutdown, when + // the connection does not exist anymore. Removing those observers would + // be less expensive but may cause their RemoveObserver calls to throw. + // Thus notify the topic now, so they stop listening for it. + nsCOMPtr e; + if (NS_SUCCEEDED(os->EnumerateObservers(TOPIC_PLACES_INIT_COMPLETE, + getter_AddRefs(e))) && + e) { + bool hasMore = false; + while (NS_SUCCEEDED(e->HasMoreElements(&hasMore)) && hasMore) { + nsCOMPtr supports; + if (NS_SUCCEEDED(e->GetNext(getter_AddRefs(supports)))) { + nsCOMPtr observer = do_QueryInterface(supports); + (void)observer->Observe(observer, TOPIC_PLACES_INIT_COMPLETE, + nullptr); + } + } + } + + // Notify all Places users that we are about to shutdown. + (void)os->NotifyObservers(nullptr, TOPIC_PLACES_SHUTDOWN, nullptr); + } else if (strcmp(aTopic, TOPIC_SIMULATE_PLACES_SHUTDOWN) == 0) { + // This notification is (and must be) only used by tests that are trying + // to simulate Places shutdown out of the normal shutdown path. + + // Tests simulating shutdown may cause re-entrance. + if (PlacesShutdownBlocker::sIsStarted) { + return NS_OK; + } + + // We are simulating a shutdown, so invoke the shutdown blockers, + // wait for them, then proceed with connection shutdown. + // Since we are already going through shutdown, but it's not the real one, + // we won't need to block the real one anymore, so we can unblock it. + { + nsCOMPtr shutdownPhase = + GetProfileChangeTeardownPhase(); + if (shutdownPhase) { + shutdownPhase->RemoveBlocker(mClientsShutdown.get()); + } + (void)mClientsShutdown->BlockShutdown(nullptr); + } + + // Spin the events loop until the clients are done. + // Note, this is just for tests, specifically test_clearHistory_shutdown.js + SpinEventLoopUntil("places:Database::Observe(SIMULATE_PLACES_SHUTDOWN)"_ns, + [&]() { + return mClientsShutdown->State() == + PlacesShutdownBlocker::States::RECEIVED_DONE; + }); + + { + nsCOMPtr shutdownPhase = + GetProfileBeforeChangePhase(); + if (shutdownPhase) { + shutdownPhase->RemoveBlocker(mConnectionShutdown.get()); + } + (void)mConnectionShutdown->BlockShutdown(nullptr); + } + } + return NS_OK; +} + +} // namespace mozilla::places diff --git a/toolkit/components/places/Database.h b/toolkit/components/places/Database.h new file mode 100644 index 0000000000..1dfb16c0ca --- /dev/null +++ b/toolkit/components/places/Database.h @@ -0,0 +1,389 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_Database_h_ +#define mozilla_places_Database_h_ + +#include "MainThreadUtils.h" +#include "nsWeakReference.h" +#include "nsIInterfaceRequestorUtils.h" +#include "nsIObserver.h" +#include "mozilla/storage.h" +#include "mozilla/storage/StatementCache.h" +#include "mozilla/Attributes.h" +#include "nsIEventTarget.h" +#include "Shutdown.h" +#include "nsCategoryCache.h" + +// This is the schema version. Update it at any schema change and add a +// corresponding migrateVxx method below. +#define DATABASE_SCHEMA_VERSION 74 + +// Fired after Places inited. +#define TOPIC_PLACES_INIT_COMPLETE "places-init-complete" +// This topic is received when the profile is about to be lost. Places does +// initial shutdown work and notifies TOPIC_PLACES_SHUTDOWN to all listeners. +// Any shutdown work that requires the Places APIs should happen here. +#define TOPIC_PROFILE_CHANGE_TEARDOWN "profile-change-teardown" +// Fired when Places is shutting down. Any code should stop accessing Places +// APIs after this notification. If you need to listen for Places shutdown +// you should only use this notification, next ones are intended only for +// internal Places use. +#define TOPIC_PLACES_SHUTDOWN "places-shutdown" +// Fired when the connection has gone, nothing will work from now on. +#define TOPIC_PLACES_CONNECTION_CLOSED "places-connection-closed" + +// Simulate profile-before-change. This topic may only be used by +// calling `observe` directly on the database. Used for testing only. +#define TOPIC_SIMULATE_PLACES_SHUTDOWN "test-simulate-places-shutdown" + +class mozIStorageService; +class nsIAsyncShutdownClient; +class nsIRunnable; + +namespace mozilla::places { + +enum JournalMode { + // Default SQLite journal mode. + JOURNAL_DELETE = 0 + // Can reduce fsyncs on Linux when journal is deleted (See bug 460315). + // We fallback to this mode when WAL is unavailable. + , + JOURNAL_TRUNCATE + // Unsafe in case of crashes on database swap or low memory. + , + JOURNAL_MEMORY + // Can reduce number of fsyncs. We try to use this mode by default. + , + JOURNAL_WAL +}; + +class ClientsShutdownBlocker; +class ConnectionShutdownBlocker; + +class Database final : public nsIObserver, public nsSupportsWeakReference { + using StatementCache = mozilla::storage::StatementCache; + using AsyncStatementCache = + mozilla::storage::StatementCache; + + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIOBSERVER + + Database(); + + /** + * Initializes the database connection and the schema. + * In case of corruption the database is copied to a backup file and replaced. + */ + nsresult Init(); + + /** + * The AsyncShutdown client used by clients of this API to be informed of + * shutdown. + */ + already_AddRefed GetClientsShutdown(); + + /** + * The AsyncShutdown client used by clients of this API to be informed of + * connection shutdown. + */ + already_AddRefed GetConnectionShutdown(); + + /** + * Getter to use when instantiating the class. + * + * @return Singleton instance of this class. + */ + static already_AddRefed GetDatabase(); + + /** + * Actually initialized the connection on first need. + */ + nsresult EnsureConnection(); + + /** + * Notifies that the connection has been initialized. + */ + nsresult NotifyConnectionInitalized(); + + /** + * Returns last known database status. + * + * @return one of the nsINavHistoryService::DATABASE_STATUS_* constants. + */ + uint16_t GetDatabaseStatus() { + mozilla::Unused << EnsureConnection(); + return mDatabaseStatus; + } + + /** + * Returns a pointer to the storage connection. + * + * @return The connection handle. + */ + mozIStorageConnection* MainConn() { + mozilla::Unused << EnsureConnection(); + return mMainConn; + } + + /** + * Dispatches a runnable to the connection async thread, to be serialized + * with async statements. + * + * @param aEvent + * The runnable to be dispatched. + */ + void DispatchToAsyncThread(nsIRunnable* aEvent) { + if (mClosed || NS_FAILED(EnsureConnection())) { + return; + } + nsCOMPtr target = do_GetInterface(mMainConn); + if (target) { + (void)target->Dispatch(aEvent, NS_DISPATCH_NORMAL); + } + } + + ////////////////////////////////////////////////////////////////////////////// + //// Statements Getters. + + /** + * Gets a cached synchronous statement. + * + * @param aQuery + * SQL query literal. + * @return The cached statement. + * @note Always null check the result. + * @note Always use a scoper to reset the statement. + */ + template + already_AddRefed GetStatement(const char (&aQuery)[N]) { + nsDependentCString query(aQuery, N - 1); + return GetStatement(query); + } + + /** + * Gets a cached synchronous statement. + * + * @param aQuery + * nsCString of SQL query. + * @return The cached statement. + * @note Always null check the result. + * @note Always use a scoper to reset the statement. + */ + already_AddRefed GetStatement(const nsACString& aQuery); + + /** + * Gets a cached asynchronous statement. + * + * @param aQuery + * SQL query literal. + * @return The cached statement. + * @note Always null check the result. + * @note AsyncStatements are automatically reset on execution. + */ + template + already_AddRefed GetAsyncStatement( + const char (&aQuery)[N]) { + nsDependentCString query(aQuery, N - 1); + return GetAsyncStatement(query); + } + + /** + * Gets a cached asynchronous statement. + * + * @param aQuery + * nsCString of SQL query. + * @return The cached statement. + * @note Always null check the result. + * @note AsyncStatements are automatically reset on execution. + */ + already_AddRefed GetAsyncStatement( + const nsACString& aQuery); + + int64_t GetTagsFolderId() { + mozilla::Unused << EnsureConnection(); + return mTagsRootId; + } + nsresult RecalculateOriginFrecencyStatsInternal(); + + protected: + /** + * Finalizes the cached statements and closes the database connection. + * A TOPIC_PLACES_CONNECTION_CLOSED notification is fired when done. + */ + void Shutdown(); + + /** + * Ensure the favicons database file exists. + * + * @param aStorage + * mozStorage service instance. + */ + nsresult EnsureFaviconsDatabaseAttached( + const nsCOMPtr& aStorage); + + /** + * Creates a database backup and replaces the original file with a new + * one. + * + * @param aStorage + * mozStorage service instance. + * @param aDbfilename + * the database file name to replace. + * @param aTryToClone + * whether we should try to clone a corrupt database. + * @param aReopenConnection + * whether we should open a new connection to the replaced database. + */ + nsresult BackupAndReplaceDatabaseFile(nsCOMPtr& aStorage, + const nsString& aDbFilename, + bool aTryToClone, + bool aReopenConnection); + + /** + * Tries to recover tables and their contents from a corrupt database. + * + * @param aStorage + * mozStorage service instance. + * @param aDatabaseFile + * nsIFile pointing to the places.sqlite file considered corrupt. + */ + nsresult TryToCloneTablesFromCorruptDatabase( + const nsCOMPtr& aStorage, + const nsCOMPtr& aDatabaseFile); + + /** + * Set up the connection environment through PRAGMAs. + * Will return NS_ERROR_FILE_CORRUPTED if any critical setting fails. + * + * @param aStorage + * mozStorage service instance. + */ + nsresult SetupDatabaseConnection(nsCOMPtr& aStorage); + + /** + * Initializes the schema. This performs any necessary migrations for the + * database. All migration is done inside a transaction that is rolled back + * if any error occurs. + * @param aDatabaseMigrated + * Whether a schema upgrade happened. + */ + nsresult InitSchema(bool* aDatabaseMigrated); + + /** + * Checks the root bookmark folders are present, and saves the IDs for them. + */ + nsresult CheckRoots(); + + /** + * Creates bookmark roots in a new DB. + */ + nsresult EnsureBookmarkRoots(const int32_t startPosition, + bool shouldReparentRoots); + + /** + * Initializes additionale SQLite functions, defined in SQLFunctions.h + */ + nsresult InitFunctions(); + + /** + * Initializes temp entities, like triggers, tables, views... + */ + nsresult InitTempEntities(); + + /** + * Helpers used by schema upgrades. + */ + nsresult MigrateV44Up(); + nsresult MigrateV45Up(); + nsresult MigrateV46Up(); + nsresult MigrateV47Up(); + nsresult MigrateV48Up(); + nsresult MigrateV49Up(); + nsresult MigrateV50Up(); + nsresult MigrateV51Up(); + nsresult MigrateV52Up(); + nsresult MigrateV53Up(); + nsresult MigrateV54Up(); + nsresult MigrateV55Up(); + nsresult MigrateV56Up(); + nsresult MigrateV57Up(); + nsresult MigrateV60Up(); + nsresult MigrateV61Up(); + nsresult MigrateV67Up(); + nsresult MigrateV69Up(); + nsresult MigrateV70Up(); + nsresult MigrateV71Up(); + nsresult MigrateV72Up(); + nsresult MigrateV73Up(); + nsresult MigrateV74Up(); + + void MigrateV52OriginFrecencies(); + + nsresult UpdateBookmarkRootTitles(); + + friend class ConnectionShutdownBlocker; + + int64_t CreateMobileRoot(); + nsresult ConvertOldStyleQuery(nsCString& aURL); + + private: + ~Database(); + + /** + * Singleton getter, invoked by class instantiation. + */ + static already_AddRefed GetSingleton(); + + static Database* gDatabase; + + nsCOMPtr mMainConn; + + mutable StatementCache mMainThreadStatements; + mutable AsyncStatementCache mMainThreadAsyncStatements; + mutable StatementCache mAsyncThreadStatements; + + int32_t mDBPageSize; + uint16_t mDatabaseStatus; + bool mClosed; + + /** + * Phases for shutting down the Database. + * See Shutdown.h for further details about the shutdown procedure. + */ + already_AddRefed GetProfileChangeTeardownPhase(); + already_AddRefed GetProfileBeforeChangePhase(); + + /** + * Blockers in charge of waiting for the Places clients and then shutting + * down the mozStorage connection. + * See Shutdown.h for further details about the shutdown procedure. + * + * Cycles with these are broken in `Shutdown()`. + */ + RefPtr mClientsShutdown; + RefPtr mConnectionShutdown; + + // Maximum length of a stored url. + // For performance reasons we don't store very long urls in history, since + // they are slower to search through and cause abnormal database growth, + // affecting the awesomebar fetch time. + uint32_t mMaxUrlLength; + + // Used to initialize components on places startup. + nsCategoryCache mCacheObservers; + + // Used to cache the places folder Ids when the connection is started. + int64_t mRootId; + int64_t mMenuRootId; + int64_t mTagsRootId; + int64_t mUnfiledRootId; + int64_t mToolbarRootId; + int64_t mMobileRootId; +}; + +} // namespace mozilla::places + +#endif // mozilla_places_Database_h_ diff --git a/toolkit/components/places/ExtensionSearchHandler.sys.mjs b/toolkit/components/places/ExtensionSearchHandler.sys.mjs new file mode 100644 index 0000000000..a04e40b6c6 --- /dev/null +++ b/toolkit/components/places/ExtensionSearchHandler.sys.mjs @@ -0,0 +1,336 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Used to keep track of all of the registered keywords, where each keyword is +// mapped to a KeywordInfo instance. +let gKeywordMap = new Map(); + +// Used to keep track of the active input session. +let gActiveInputSession = null; + +// Used to keep track of who has control over the active suggestion callback +// so older callbacks can be ignored. The callback ID should increment whenever +// the input changes or the input session ends. +let gCurrentCallbackID = 0; + +// Handles keeping track of information associated to the registered keyword. +class KeywordInfo { + constructor(extension, description) { + this._extension = extension; + this._description = description; + } + + get description() { + return this._description; + } + + set description(desc) { + this._description = desc; + } + + get extension() { + return this._extension; + } +} + +// Responsible for handling communication between the extension and the urlbar. +class InputSession { + constructor(keyword, extension) { + this._keyword = keyword; + this._extension = extension; + this._suggestionsCallback = null; + this._searchFinishedCallback = null; + } + + get keyword() { + return this._keyword; + } + + addSuggestions(suggestions) { + if (this._suggestionsCallback) { + this._suggestionsCallback(suggestions); + } + } + + start(eventName) { + this._extension.emit(eventName); + } + + update(eventName, text, suggestionsCallback, searchFinishedCallback) { + // Check to see if an existing input session needs to be ended first. + if (this._searchFinishedCallback) { + this._searchFinishedCallback(); + } + this._searchFinishedCallback = searchFinishedCallback; + this._suggestionsCallback = suggestionsCallback; + this._extension.emit(eventName, text, ++gCurrentCallbackID); + } + + cancel(eventName) { + if (this._searchFinishedCallback) { + this._searchFinishedCallback(); + this._searchFinishedCallback = null; + this._suggestionsCallback = null; + } + this._extension.emit(eventName); + } + + end(eventName, text, disposition) { + if (this._searchFinishedCallback) { + this._searchFinishedCallback(); + this._searchFinishedCallback = null; + this._suggestionsCallback = null; + } + this._extension.emit(eventName, text, disposition); + } +} + +export var ExtensionSearchHandler = Object.freeze({ + MSG_INPUT_STARTED: "webext-omnibox-input-started", + MSG_INPUT_CHANGED: "webext-omnibox-input-changed", + MSG_INPUT_ENTERED: "webext-omnibox-input-entered", + MSG_INPUT_CANCELLED: "webext-omnibox-input-cancelled", + MSG_INPUT_DELETED: "webext-omnibox-input-deleted", + + /** + * Registers a keyword. + * + * @param {string} keyword The keyword to register. + * @param {Extension} extension The extension registering the keyword. + */ + registerKeyword(keyword, extension) { + if (gKeywordMap.has(keyword)) { + throw new Error( + `The keyword provided is already registered: "${keyword}"` + ); + } + gKeywordMap.set(keyword, new KeywordInfo(extension, extension.name)); + }, + + /** + * Unregisters a keyword. + * + * @param {string} keyword The keyword to unregister. + */ + unregisterKeyword(keyword) { + if (!gKeywordMap.has(keyword)) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + gActiveInputSession = null; + gKeywordMap.delete(keyword); + }, + + /** + * Checks if a keyword is registered. + * + * @param {string} keyword The word to check. + * @return {boolean} true if the word is a registered keyword. + */ + isKeywordRegistered(keyword) { + return gKeywordMap.has(keyword); + }, + + /** + * @return {boolean} true if there is an active input session. + */ + hasActiveInputSession() { + return gActiveInputSession != null; + }, + + /** + * @param {string} keyword The keyword to look up. + * @return {string} the description to use for the heuristic result. + */ + getDescription(keyword) { + if (!gKeywordMap.has(keyword)) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + return gKeywordMap.get(keyword).description; + }, + + /** + * Sets the default suggestion for the registered keyword. The suggestion's + * description will be used for the comment in the heuristic result. + * + * @param {string} keyword The keyword. + * @param {string} description The description to use for the heuristic result. + */ + setDefaultSuggestion(keyword, { description }) { + if (!gKeywordMap.has(keyword)) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + gKeywordMap.get(keyword).description = description; + }, + + /** + * Adds suggestions for the registered keyword. This function will throw if + * the keyword provided is not registered or active, or if the callback ID + * provided is no longer equal to the active callback ID. + * + * @param {string} keyword The keyword. + * @param {integer} id The ID of the suggestion callback. + * @param {Array} suggestions An array of suggestions to provide to the urlbar. + */ + addSuggestions(keyword, id, suggestions) { + if (!gKeywordMap.has(keyword)) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + + if (!gActiveInputSession || gActiveInputSession.keyword != keyword) { + throw new Error( + `The keyword provided is not apart of an active input session: "${keyword}"` + ); + } + + if (id != gCurrentCallbackID) { + throw new Error( + `The callback is no longer active for the keyword provided: "${keyword}"` + ); + } + + gActiveInputSession.addSuggestions(suggestions); + }, + + /** + * Called when the input in the urlbar begins with ``. + * + * If the keyword is inactive, MSG_INPUT_STARTED is emitted and the + * keyword is marked as active. If the keyword is followed by any text, + * MSG_INPUT_CHANGED is fired with the current callback ID that can be + * used to provide suggestions to the urlbar while the callback ID is active. + * The callback is invalidated when either the input changes or the urlbar blurs. + * + * @param {object} data An object that contains + * {string} keyword The keyword to handle. + * {string} text The search text in the urlbar. + * {boolean} inPrivateWindow privateness of window search + * is occuring in. + * @param {Function} callback The callback used to provide search suggestions. + * @return {Promise} promise that resolves when the current search is complete. + */ + handleSearch(data, callback) { + let { keyword, text } = data; + let keywordInfo = gKeywordMap.get(keyword); + if (!keywordInfo) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + + let { extension } = keywordInfo; + if (data.inPrivateWindow && !extension.privateBrowsingAllowed) { + return Promise.resolve(false); + } + + if (gActiveInputSession && gActiveInputSession.keyword != keyword) { + throw new Error("A different input session is already ongoing"); + } + + if (!text || !text.startsWith(`${keyword} `)) { + throw new Error(`The text provided must start with: "${keyword} "`); + } + + if (!callback) { + throw new Error("A callback must be provided"); + } + + // The search text in the urlbar currently starts with , and + // we only want the text that follows. + text = text.substring(keyword.length + 1); + + // We fire MSG_INPUT_STARTED once we have , and only fire + // MSG_INPUT_CHANGED when we have text to process. This is different from Chrome's + // behavior, which always fires MSG_INPUT_STARTED right before MSG_INPUT_CHANGED + // first fires, but this is a bug in Chrome according to https://crbug.com/258911. + if (!gActiveInputSession) { + gActiveInputSession = new InputSession(keyword, extension); + gActiveInputSession.start(this.MSG_INPUT_STARTED); + + // Resolve early if there is no text to process. There can be text to process when + // the input starts if the user copy/pastes the text into the urlbar. + if (!text.length) { + return Promise.resolve(); + } + } + + return new Promise(resolve => { + gActiveInputSession.update( + this.MSG_INPUT_CHANGED, + text, + callback, + resolve + ); + }); + }, + + /** + * Called when the user clicks on a suggestion that was added by + * an extension. MSG_INPUT_ENTERED is emitted to the extension with + * the keyword, the current search string, and info about how the + * the search should be handled. This ends the active input session. + * + * @param {string} keyword The keyword associated to the suggestion. + * @param {string} text The search text in the urlbar. + * @param {string} where How the page should be opened. Accepted values are: + * "current": open the page in the same tab. + * "tab": open the page in a new foreground tab. + * "tabshifted": open the page in a new background tab. + */ + handleInputEntered(keyword, text, where) { + if (!gKeywordMap.has(keyword)) { + throw new Error(`The keyword provided is not registered: "${keyword}"`); + } + + if (!gActiveInputSession) { + throw new Error("There is no active input session"); + } + + if (gActiveInputSession && gActiveInputSession.keyword != keyword) { + throw new Error("A different input session is already ongoing"); + } + + if (!text || !text.startsWith(`${keyword} `)) { + throw new Error(`The text provided must start with: "${keyword} "`); + } + + let dispositionMap = { + current: "currentTab", + tab: "newForegroundTab", + tabshifted: "newBackgroundTab", + }; + let disposition = dispositionMap[where]; + + if (!disposition) { + throw new Error(`Invalid "where" argument: ${where}`); + } + + // The search text in the urlbar currently starts with , and + // we only want to send the text that follows. + text = text.substring(keyword.length + 1); + + gActiveInputSession.end(this.MSG_INPUT_ENTERED, text, disposition); + gActiveInputSession = null; + }, + + /** + * Called when the user deletes a suggestion that was added by + * an extension. MSG_INPUT_DELETED is emitted to the extension with + * the description of the suggestion that was deleted. + * + * @param {string} text The description of the suggestion. + */ + handleInputDeleted(text) { + return gActiveInputSession.update(this.MSG_INPUT_DELETED, text); + }, + + /** + * If the user has ended the keyword input session without accepting the input, + * MSG_INPUT_CANCELLED is emitted and the input session is ended. + */ + handleInputCancelled() { + if (!gActiveInputSession) { + throw new Error("There is no active input session"); + } + gActiveInputSession.cancel(this.MSG_INPUT_CANCELLED); + gActiveInputSession = null; + }, +}); diff --git a/toolkit/components/places/FaviconHelpers.cpp b/toolkit/components/places/FaviconHelpers.cpp new file mode 100644 index 0000000000..05990c878c --- /dev/null +++ b/toolkit/components/places/FaviconHelpers.cpp @@ -0,0 +1,1261 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FaviconHelpers.h" + +#include "nsICacheEntry.h" +#include "nsICachingChannel.h" +#include "nsIClassOfService.h" +#include "nsIAsyncVerifyRedirectCallback.h" +#include "nsIHttpChannel.h" +#include "nsIPrincipal.h" + +#include "nsComponentManagerUtils.h" +#include "nsNavHistory.h" +#include "nsFaviconService.h" + +#include "mozilla/dom/PlacesFavicon.h" +#include "mozilla/dom/PlacesObservers.h" +#include "mozilla/storage.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Telemetry.h" +#include "mozilla/StaticPrefs_network.h" +#include "nsNetUtil.h" +#include "nsPrintfCString.h" +#include "nsStreamUtils.h" +#include "nsStringStream.h" +#include "nsIPrivateBrowsingChannel.h" +#include "nsISupportsPriority.h" +#include +#include +#include "mozilla/gfx/2D.h" +#include "imgIContainer.h" +#include "ImageOps.h" +#include "imgIEncoder.h" + +using namespace mozilla; +using namespace mozilla::places; +using namespace mozilla::storage; + +namespace mozilla { +namespace places { + +namespace { + +/** + * Fetches information about a page from the database. + * + * @param aDB + * Database connection to history tables. + * @param _page + * Page that should be fetched. + */ +nsresult FetchPageInfo(const RefPtr& aDB, PageData& _page) { + MOZ_ASSERT(_page.spec.Length(), "Must have a non-empty spec!"); + MOZ_ASSERT(!NS_IsMainThread()); + + // The subquery finds the bookmarked uri we want to set the icon for, + // walking up redirects. + nsCString query = nsPrintfCString( + "SELECT h.id, pi.id, h.guid, ( " + "WITH RECURSIVE " + "destinations(visit_type, from_visit, place_id, rev_host, bm) AS ( " + "SELECT v.visit_type, v.from_visit, p.id, p.rev_host, b.id " + "FROM moz_places p " + "LEFT JOIN moz_historyvisits v ON v.place_id = p.id " + "LEFT JOIN moz_bookmarks b ON b.fk = p.id " + "WHERE p.id = h.id " + "UNION " + "SELECT src.visit_type, src.from_visit, src.place_id, p.rev_host, b.id " + "FROM moz_places p " + "JOIN moz_historyvisits src ON src.place_id = p.id " + "JOIN destinations dest ON dest.from_visit = src.id AND dest.visit_type " + "IN (%d, %d) " + "LEFT JOIN moz_bookmarks b ON b.fk = src.place_id " + "WHERE instr(p.rev_host, dest.rev_host) = 1 " + "OR instr(dest.rev_host, p.rev_host) = 1 " + ") " + "SELECT url " + "FROM moz_places p " + "JOIN destinations r ON r.place_id = p.id " + "WHERE bm NOTNULL " + "LIMIT 1 " + "), fixup_url(get_unreversed_host(h.rev_host)) AS host " + "FROM moz_places h " + "LEFT JOIN moz_pages_w_icons pi ON page_url_hash = hash(:page_url) AND " + "page_url = :page_url " + "WHERE h.url_hash = hash(:page_url) AND h.url = :page_url", + nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT, + nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY); + + nsCOMPtr stmt = aDB->GetStatement(query); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = URIBinder::Bind(stmt, "page_url"_ns, _page.spec); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasResult) { + // The page does not exist. + return NS_ERROR_NOT_AVAILABLE; + } + + rv = stmt->GetInt64(0, &_page.placeId); + NS_ENSURE_SUCCESS(rv, rv); + // May be null, and in such a case this will be 0. + _page.id = stmt->AsInt64(1); + rv = stmt->GetUTF8String(2, _page.guid); + NS_ENSURE_SUCCESS(rv, rv); + // Bookmarked url can be nullptr. + bool isNull; + rv = stmt->GetIsNull(3, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + // The page could not be bookmarked. + if (!isNull) { + rv = stmt->GetUTF8String(3, _page.bookmarkedSpec); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (_page.host.IsEmpty()) { + rv = stmt->GetUTF8String(4, _page.host); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (!_page.canAddToHistory) { + // Either history is disabled or the scheme is not supported. In such a + // case we want to update the icon only if the page is bookmarked. + + if (_page.bookmarkedSpec.IsEmpty()) { + // The page is not bookmarked. Since updating the icon with a disabled + // history would be a privacy leak, bail out as if the page did not exist. + return NS_ERROR_NOT_AVAILABLE; + } else { + // The page, or a redirect to it, is bookmarked. If the bookmarked spec + // is different from the requested one, use it. + if (!_page.bookmarkedSpec.Equals(_page.spec)) { + _page.spec = _page.bookmarkedSpec; + rv = FetchPageInfo(aDB, _page); + NS_ENSURE_SUCCESS(rv, rv); + } + } + } + + return NS_OK; +} + +/** + * Stores information about an icon in the database. + * + * @param aDB + * Database connection to history tables. + * @param aIcon + * Icon that should be stored. + * @param aMustReplace + * If set to true, the function will bail out with NS_ERROR_NOT_AVAILABLE + * if it can't find a previous stored icon to replace. + * @note Should be wrapped in a transaction. + */ +nsresult SetIconInfo(const RefPtr& aDB, IconData& aIcon, + bool aMustReplace = false) { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(aIcon.payloads.Length() > 0); + MOZ_ASSERT(!aIcon.spec.IsEmpty()); + MOZ_ASSERT(aIcon.expiration > 0); + + // There are multiple cases possible at this point: + // 1. We must insert some payloads and no payloads exist in the table. This + // would be a straight INSERT. + // 2. The table contains the same number of payloads we are inserting. This + // would be a straight UPDATE. + // 3. The table contains more payloads than we are inserting. This would be + // an UPDATE and a DELETE. + // 4. The table contains less payloads than we are inserting. This would be + // an UPDATE and an INSERT. + // We can't just remove all the old entries and insert the new ones, cause + // we'd lose the referential integrity with pages. For the same reason we + // cannot use INSERT OR REPLACE, since it's implemented as DELETE AND INSERT. + // Thus, we follow this strategy: + // * SELECT all existing icon ids + // * For each payload, either UPDATE OR INSERT reusing icon ids. + // * If any previous icon ids is leftover, DELETE it. + + nsCOMPtr selectStmt = aDB->GetStatement( + "SELECT id FROM moz_icons " + "WHERE fixed_icon_url_hash = hash(fixup_url(:url)) " + "AND icon_url = :url "); + NS_ENSURE_STATE(selectStmt); + mozStorageStatementScoper scoper(selectStmt); + nsresult rv = URIBinder::Bind(selectStmt, "url"_ns, aIcon.spec); + NS_ENSURE_SUCCESS(rv, rv); + std::deque ids; + bool hasResult = false; + while (NS_SUCCEEDED(selectStmt->ExecuteStep(&hasResult)) && hasResult) { + int64_t id = selectStmt->AsInt64(0); + MOZ_ASSERT(id > 0); + ids.push_back(id); + } + if (aMustReplace && ids.empty()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr insertStmt = aDB->GetStatement( + "INSERT INTO moz_icons " + "(icon_url, fixed_icon_url_hash, width, root, expire_ms, data) " + "VALUES (:url, hash(fixup_url(:url)), :width, :root, :expire, :data) "); + NS_ENSURE_STATE(insertStmt); + // ReplaceFaviconData may replace data for an already existing icon, and in + // that case it won't have the page uri at hand, thus it can't tell if the + // icon is a root icon or not. For that reason, never overwrite a root = 1. + nsCOMPtr updateStmt = aDB->GetStatement( + "UPDATE moz_icons SET width = :width, " + "expire_ms = :expire, " + "data = :data, " + "root = (root OR :root) " + "WHERE id = :id "); + NS_ENSURE_STATE(updateStmt); + + for (auto& payload : aIcon.payloads) { + // Sanity checks. + MOZ_ASSERT(payload.mimeType.EqualsLiteral(PNG_MIME_TYPE) || + payload.mimeType.EqualsLiteral(SVG_MIME_TYPE), + "Only png and svg payloads are supported"); + MOZ_ASSERT(!payload.mimeType.EqualsLiteral(SVG_MIME_TYPE) || + payload.width == UINT16_MAX, + "SVG payloads should have max width"); + MOZ_ASSERT(payload.width > 0, "Payload should have a width"); +#ifdef DEBUG + // Done to ensure we fetch the id. See the MOZ_ASSERT below. + payload.id = 0; +#endif + if (!ids.empty()) { + // Pop the first existing id for reuse. + int64_t id = ids.front(); + ids.pop_front(); + mozStorageStatementScoper scoper(updateStmt); + rv = updateStmt->BindInt64ByName("id"_ns, id); + NS_ENSURE_SUCCESS(rv, rv); + rv = updateStmt->BindInt32ByName("width"_ns, payload.width); + NS_ENSURE_SUCCESS(rv, rv); + rv = updateStmt->BindInt64ByName("expire"_ns, aIcon.expiration / 1000); + NS_ENSURE_SUCCESS(rv, rv); + rv = updateStmt->BindInt32ByName("root"_ns, aIcon.rootIcon); + NS_ENSURE_SUCCESS(rv, rv); + rv = updateStmt->BindBlobByName("data"_ns, TO_INTBUFFER(payload.data), + payload.data.Length()); + NS_ENSURE_SUCCESS(rv, rv); + rv = updateStmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + // Set the new payload id. + payload.id = id; + } else { + // Insert a new entry. + mozStorageStatementScoper scoper(insertStmt); + rv = URIBinder::Bind(insertStmt, "url"_ns, aIcon.spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = insertStmt->BindInt32ByName("width"_ns, payload.width); + NS_ENSURE_SUCCESS(rv, rv); + + rv = insertStmt->BindInt32ByName("root"_ns, aIcon.rootIcon); + NS_ENSURE_SUCCESS(rv, rv); + rv = insertStmt->BindInt64ByName("expire"_ns, aIcon.expiration / 1000); + NS_ENSURE_SUCCESS(rv, rv); + rv = insertStmt->BindBlobByName("data"_ns, TO_INTBUFFER(payload.data), + payload.data.Length()); + NS_ENSURE_SUCCESS(rv, rv); + rv = insertStmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + // Set the new payload id. + payload.id = nsFaviconService::sLastInsertedIconId; + } + MOZ_ASSERT(payload.id > 0, "Payload should have an id"); + } + + if (!ids.empty()) { + // Remove any old leftover payload. + nsAutoCString sql("DELETE FROM moz_icons WHERE id IN ("); + for (int64_t id : ids) { + sql.AppendInt(id); + sql.AppendLiteral(","); + } + sql.AppendLiteral(" 0)"); // Non-existing id to match the trailing comma. + nsCOMPtr stmt = aDB->GetStatement(sql); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +/** + * Fetches information on a icon url from the database. + * + * @param aDBConn + * Database connection to history tables. + * @param aPreferredWidth + * The preferred size to fetch. + * @param _icon + * Icon that should be fetched. + */ +nsresult FetchIconInfo(const RefPtr& aDB, uint16_t aPreferredWidth, + IconData& _icon) { + MOZ_ASSERT(_icon.spec.Length(), "Must have a non-empty spec!"); + MOZ_ASSERT(!NS_IsMainThread()); + + if (_icon.status & ICON_STATUS_CACHED) { + // The icon data has already been set by ReplaceFaviconData. + return NS_OK; + } + + nsCOMPtr stmt = aDB->GetStatement( + "/* do not warn (bug no: not worth having a compound index) */ " + "SELECT id, expire_ms, data, width, root " + "FROM moz_icons " + "WHERE fixed_icon_url_hash = hash(fixup_url(:url)) " + "AND icon_url = :url " + "ORDER BY width DESC "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + DebugOnly rv = URIBinder::Bind(stmt, "url"_ns, _icon.spec); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + bool hasResult = false; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + IconPayload payload; + rv = stmt->GetInt64(0, &payload.id); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + // Expiration can be nullptr. + bool isNull; + rv = stmt->GetIsNull(1, &isNull); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + if (!isNull) { + int64_t expire_ms; + rv = stmt->GetInt64(1, &expire_ms); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + _icon.expiration = expire_ms * 1000; + } + + uint8_t* data; + uint32_t dataLen = 0; + rv = stmt->GetBlob(2, &dataLen, &data); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + payload.data.Adopt(TO_CHARBUFFER(data), dataLen); + + int32_t width; + rv = stmt->GetInt32(3, &width); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + payload.width = width; + if (payload.width == UINT16_MAX) { + payload.mimeType.AssignLiteral(SVG_MIME_TYPE); + } else { + payload.mimeType.AssignLiteral(PNG_MIME_TYPE); + } + + int32_t rootIcon; + rv = stmt->GetInt32(4, &rootIcon); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + _icon.rootIcon = rootIcon; + + if (aPreferredWidth == 0 || _icon.payloads.Length() == 0) { + _icon.payloads.AppendElement(payload); + } else if (payload.width >= aPreferredWidth) { + // Only retain the best matching payload. + _icon.payloads.ReplaceElementAt(0, payload); + } else { + break; + } + } + + return NS_OK; +} + +nsresult FetchIconPerSpec(const RefPtr& aDB, + const nsACString& aPageSpec, + const nsACString& aPageHost, IconData& aIconData, + uint16_t aPreferredWidth) { + MOZ_ASSERT(!aPageSpec.IsEmpty(), "Page spec must not be empty."); + MOZ_ASSERT(!NS_IsMainThread()); + + // This selects both associated and root domain icons, ordered by width, + // where an associated icon has priority over a root domain icon. + // Regardless, note that while this way we are far more efficient, we lost + // associations with root domain icons, so it's possible we'll return one + // for a specific size when an associated icon for that size doesn't exist. + nsCOMPtr stmt = aDB->GetStatement( + "/* do not warn (bug no: not worth having a compound index) */ " + "SELECT width, icon_url, root " + "FROM moz_icons i " + "JOIN moz_icons_to_pages ON i.id = icon_id " + "JOIN moz_pages_w_icons p ON p.id = page_id " + "WHERE page_url_hash = hash(:url) AND page_url = :url " + "OR (:hash_idx AND page_url_hash = hash(substr(:url, 0, :hash_idx)) " + "AND page_url = substr(:url, 0, :hash_idx)) " + "UNION ALL " + "SELECT width, icon_url, root " + "FROM moz_icons i " + "WHERE fixed_icon_url_hash = hash(fixup_url(:host) || '/favicon.ico') " + "ORDER BY width DESC, root ASC"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = URIBinder::Bind(stmt, "url"_ns, aPageSpec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("host"_ns, aPageHost); + NS_ENSURE_SUCCESS(rv, rv); + int32_t hashIdx = PromiseFlatCString(aPageSpec).RFind("#"); + rv = stmt->BindInt32ByName("hash_idx"_ns, hashIdx + 1); + NS_ENSURE_SUCCESS(rv, rv); + + // Return the biggest icon close to the preferred width. It may be bigger + // or smaller if the preferred width isn't found. + bool hasResult; + int32_t lastWidth = 0; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + int32_t width; + rv = stmt->GetInt32(0, &width); + if (lastWidth == width) { + // We already found an icon for this width. We always prefer the first + // icon found, because it's a non-root icon, per the root ASC ordering. + continue; + } + if (!aIconData.spec.IsEmpty() && width < aPreferredWidth) { + // We found the best match, or we already found a match so we don't need + // to fallback to the root domain icon. + break; + } + lastWidth = width; + rv = stmt->GetUTF8String(1, aIconData.spec); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +/** + * Tries to compute the expiration time for a icon from the channel. + * + * @param aChannel + * The network channel used to fetch the icon. + * @return a valid expiration value for the fetched icon. + */ +PRTime GetExpirationTimeFromChannel(nsIChannel* aChannel) { + MOZ_ASSERT(NS_IsMainThread()); + + // Attempt to get an expiration time from the cache. If this fails, we'll + // make one up. + PRTime now = PR_Now(); + PRTime expiration = -1; + nsCOMPtr cachingChannel = do_QueryInterface(aChannel); + if (cachingChannel) { + nsCOMPtr cacheToken; + nsresult rv = cachingChannel->GetCacheToken(getter_AddRefs(cacheToken)); + if (NS_SUCCEEDED(rv)) { + nsCOMPtr cacheEntry = do_QueryInterface(cacheToken); + uint32_t seconds; + rv = cacheEntry->GetExpirationTime(&seconds); + if (NS_SUCCEEDED(rv)) { + // Set the expiration, but make sure we honor our cap. + expiration = now + std::min((PRTime)seconds * PR_USEC_PER_SEC, + MAX_FAVICON_EXPIRATION); + } + } + } + // If we did not obtain a time from the cache, use the cap value. + return expiration < now + MIN_FAVICON_EXPIRATION + ? now + MAX_FAVICON_EXPIRATION + : expiration; +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncFetchAndSetIconForPage + +NS_IMPL_ISUPPORTS_INHERITED(AsyncFetchAndSetIconForPage, Runnable, + nsIStreamListener, nsIInterfaceRequestor, + nsIChannelEventSink, mozIPlacesPendingOperation) + +AsyncFetchAndSetIconForPage::AsyncFetchAndSetIconForPage( + IconData& aIcon, PageData& aPage, bool aFaviconLoadPrivate, + nsIFaviconDataCallback* aCallback, nsIPrincipal* aLoadingPrincipal, + uint64_t aRequestContextID) + : Runnable("places::AsyncFetchAndSetIconForPage"), + mCallback(new nsMainThreadPtrHolder( + "AsyncFetchAndSetIconForPage::mCallback", aCallback)), + mIcon(aIcon), + mPage(aPage), + mFaviconLoadPrivate(aFaviconLoadPrivate), + mLoadingPrincipal(new nsMainThreadPtrHolder( + "AsyncFetchAndSetIconForPage::mLoadingPrincipal", aLoadingPrincipal)), + mCanceled(false), + mRequestContextID(aRequestContextID) { + MOZ_ASSERT(NS_IsMainThread()); +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + + // Try to fetch the icon from the database. + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + nsresult rv = FetchIconInfo(DB, 0, mIcon); + NS_ENSURE_SUCCESS(rv, rv); + + bool isInvalidIcon = !mIcon.payloads.Length() || PR_Now() > mIcon.expiration; + bool fetchIconFromNetwork = + mIcon.fetchMode == FETCH_ALWAYS || + (mIcon.fetchMode == FETCH_IF_MISSING && isInvalidIcon); + + // Check if we can associate the icon to this page. + rv = FetchPageInfo(DB, mPage); + if (NS_FAILED(rv)) { + if (rv == NS_ERROR_NOT_AVAILABLE) { + // We have never seen this page. If we can add the page to history, + // we will try to do it later, otherwise just bail out. + if (!mPage.canAddToHistory) { + return NS_OK; + } + } + return rv; + } + + if (!fetchIconFromNetwork) { + // There is already a valid icon or we don't want to fetch a new one, + // directly proceed with association. + RefPtr event = + new AsyncAssociateIconToPage(mIcon, mPage, mCallback); + // We're already on the async thread. + return event->Run(); + } + + // Fetch the icon from the network, the request starts from the main-thread. + // When done this will associate the icon to the page and notify. + nsCOMPtr event = + NewRunnableMethod("places::AsyncFetchAndSetIconForPage::FetchFromNetwork", + this, &AsyncFetchAndSetIconForPage::FetchFromNetwork); + return NS_DispatchToMainThread(event); +} + +nsresult AsyncFetchAndSetIconForPage::FetchFromNetwork() { + MOZ_ASSERT(NS_IsMainThread()); + + if (mCanceled) { + return NS_OK; + } + + // Ensure data is cleared, since it's going to be overwritten. + mIcon.payloads.Clear(); + + IconPayload payload; + mIcon.payloads.AppendElement(payload); + + nsCOMPtr iconURI; + nsresult rv = NS_NewURI(getter_AddRefs(iconURI), mIcon.spec); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr channel; + rv = NS_NewChannel(getter_AddRefs(channel), iconURI, mLoadingPrincipal, + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT | + nsILoadInfo::SEC_ALLOW_CHROME | + nsILoadInfo::SEC_DISALLOW_SCRIPT, + nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON); + + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr listenerRequestor = + do_QueryInterface(reinterpret_cast(this)); + NS_ENSURE_STATE(listenerRequestor); + rv = channel->SetNotificationCallbacks(listenerRequestor); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr pbChannel = do_QueryInterface(channel); + if (pbChannel) { + rv = pbChannel->SetPrivate(mFaviconLoadPrivate); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsCOMPtr priorityChannel = do_QueryInterface(channel); + if (priorityChannel) { + priorityChannel->AdjustPriority(nsISupportsPriority::PRIORITY_LOWEST); + } + + if (StaticPrefs::network_http_tailing_enabled()) { + nsCOMPtr cos = do_QueryInterface(channel); + if (cos) { + cos->AddClassFlags(nsIClassOfService::Tail | + nsIClassOfService::Throttleable); + } + + nsCOMPtr httpChannel(do_QueryInterface(channel)); + if (httpChannel) { + Unused << httpChannel->SetRequestContextID(mRequestContextID); + } + } + + rv = channel->AsyncOpen(this); + if (NS_SUCCEEDED(rv)) { + mRequest = channel; + } + return rv; +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::Cancel() { + MOZ_ASSERT(NS_IsMainThread()); + if (mCanceled) { + return NS_ERROR_UNEXPECTED; + } + mCanceled = true; + if (mRequest) { + mRequest->CancelWithReason(NS_BINDING_ABORTED, + "AsyncFetchAndSetIconForPage::Cancel"_ns); + } + return NS_OK; +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::OnStartRequest(nsIRequest* aRequest) { + // mRequest should already be set from ::FetchFromNetwork, but in the case of + // a redirect we might get a new request, and we should make sure we keep a + // reference to the most current request. + mRequest = aRequest; + if (mCanceled) { + mRequest->Cancel(NS_BINDING_ABORTED); + } + // Don't store icons responding with Cache-Control: no-store, but always + // allow root domain icons. + nsCOMPtr httpChannel = do_QueryInterface(aRequest); + if (httpChannel) { + bool isNoStore; + nsAutoCString path; + nsCOMPtr uri; + if (NS_SUCCEEDED(httpChannel->GetURI(getter_AddRefs(uri))) && + NS_SUCCEEDED(uri->GetFilePath(path)) && + !path.EqualsLiteral("/favicon.ico") && + NS_SUCCEEDED(httpChannel->IsNoStoreResponse(&isNoStore)) && isNoStore) { + // Abandon the network fetch. + mRequest->CancelWithReason( + NS_BINDING_ABORTED, "AsyncFetchAndSetIconForPage::OnStartRequest"_ns); + } + } + return NS_OK; +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::OnDataAvailable(nsIRequest* aRequest, + nsIInputStream* aInputStream, + uint64_t aOffset, + uint32_t aCount) { + MOZ_ASSERT(mIcon.payloads.Length() == 1); + // Limit downloads to 500KB. + const size_t kMaxDownloadSize = 500 * 1024; + if (mIcon.payloads[0].data.Length() + aCount > kMaxDownloadSize) { + mIcon.payloads.Clear(); + return NS_ERROR_FILE_TOO_BIG; + } + + nsAutoCString buffer; + nsresult rv = NS_ConsumeStream(aInputStream, aCount, buffer); + if (rv != NS_BASE_STREAM_WOULD_BLOCK && NS_FAILED(rv)) { + return rv; + } + + if (!mIcon.payloads[0].data.Append(buffer, fallible)) { + mIcon.payloads.Clear(); + return NS_ERROR_OUT_OF_MEMORY; + } + + return NS_OK; +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::GetInterface(const nsIID& uuid, void** aResult) { + return QueryInterface(uuid, aResult); +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::AsyncOnChannelRedirect( + nsIChannel* oldChannel, nsIChannel* newChannel, uint32_t flags, + nsIAsyncVerifyRedirectCallback* cb) { + // If we've been canceled, stop the redirect with NS_BINDING_ABORTED, and + // handle the cancel on the original channel. + (void)cb->OnRedirectVerifyCallback(mCanceled ? NS_BINDING_ABORTED : NS_OK); + return NS_OK; +} + +NS_IMETHODIMP +AsyncFetchAndSetIconForPage::OnStopRequest(nsIRequest* aRequest, + nsresult aStatusCode) { + MOZ_ASSERT(NS_IsMainThread()); + + // Don't need to track this anymore. + mRequest = nullptr; + if (mCanceled) { + return NS_OK; + } + + nsFaviconService* favicons = nsFaviconService::GetFaviconService(); + NS_ENSURE_STATE(favicons); + + nsresult rv; + + // If fetching the icon failed, bail out. + if (NS_FAILED(aStatusCode) || mIcon.payloads.Length() == 0) { + return NS_OK; + } + + nsCOMPtr channel = do_QueryInterface(aRequest); + // aRequest should always QI to nsIChannel. + MOZ_ASSERT(channel); + + MOZ_ASSERT(mIcon.payloads.Length() == 1); + IconPayload& payload = mIcon.payloads[0]; + + nsAutoCString contentType; + channel->GetContentType(contentType); + // Bug 366324 - We don't want to sniff for SVG, so rely on server-specified + // type. + if (contentType.EqualsLiteral(SVG_MIME_TYPE)) { + payload.mimeType.AssignLiteral(SVG_MIME_TYPE); + payload.width = UINT16_MAX; + } else { + NS_SniffContent(NS_DATA_SNIFFER_CATEGORY, aRequest, + TO_INTBUFFER(payload.data), payload.data.Length(), + payload.mimeType); + } + + // If the icon does not have a valid MIME type, bail out. + if (payload.mimeType.IsEmpty()) { + return NS_OK; + } + + mIcon.expiration = GetExpirationTimeFromChannel(channel); + + // Telemetry probes to measure the favicon file sizes for each different file + // type. This allow us to measure common file sizes while also observing each + // type popularity. + if (payload.mimeType.EqualsLiteral(PNG_MIME_TYPE)) { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_PNG_SIZES, + payload.data.Length()); + } else if (payload.mimeType.EqualsLiteral("image/x-icon") || + payload.mimeType.EqualsLiteral("image/vnd.microsoft.icon")) { + Telemetry::Accumulate(mozilla::Telemetry::PLACES_FAVICON_ICO_SIZES, + payload.data.Length()); + } else if (payload.mimeType.EqualsLiteral("image/jpeg") || + payload.mimeType.EqualsLiteral("image/pjpeg")) { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_JPEG_SIZES, + payload.data.Length()); + } else if (payload.mimeType.EqualsLiteral("image/gif")) { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_GIF_SIZES, + payload.data.Length()); + } else if (payload.mimeType.EqualsLiteral("image/bmp") || + payload.mimeType.EqualsLiteral("image/x-windows-bmp")) { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_BMP_SIZES, + payload.data.Length()); + } else if (payload.mimeType.EqualsLiteral(SVG_MIME_TYPE)) { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_SVG_SIZES, + payload.data.Length()); + } else { + Telemetry::Accumulate(Telemetry::PLACES_FAVICON_OTHER_SIZES, + payload.data.Length()); + } + + rv = favicons->OptimizeIconSizes(mIcon); + NS_ENSURE_SUCCESS(rv, rv); + + // If there's not valid payload, don't store the icon into to the database. + if (mIcon.payloads.Length() == 0) { + return NS_OK; + } + + mIcon.status = ICON_STATUS_CHANGED; + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + RefPtr event = + new AsyncAssociateIconToPage(mIcon, mPage, mCallback); + DB->DispatchToAsyncThread(event); + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncAssociateIconToPage + +AsyncAssociateIconToPage::AsyncAssociateIconToPage( + const IconData& aIcon, const PageData& aPage, + const nsMainThreadPtrHandle& aCallback) + : Runnable("places::AsyncAssociateIconToPage"), + mCallback(aCallback), + mIcon(aIcon), + mPage(aPage) { + // May be created in both threads. +} + +NS_IMETHODIMP +AsyncAssociateIconToPage::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(!mPage.guid.IsEmpty(), + "Page info should have been fetched already"); + MOZ_ASSERT(mPage.canAddToHistory || !mPage.bookmarkedSpec.IsEmpty(), + "The page should be addable to history or a bookmark"); + + bool shouldUpdateIcon = mIcon.status & ICON_STATUS_CHANGED; + if (!shouldUpdateIcon) { + for (const auto& payload : mIcon.payloads) { + // If the entry is missing from the database, we should add it. + if (payload.id == 0) { + shouldUpdateIcon = true; + break; + } + } + } + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + + mozStorageTransaction transaction( + DB->MainConn(), false, mozIStorageConnection::TRANSACTION_IMMEDIATE); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + nsresult rv; + if (shouldUpdateIcon) { + rv = SetIconInfo(DB, mIcon); + if (NS_FAILED(rv)) { + (void)transaction.Commit(); + return rv; + } + + mIcon.status = (mIcon.status & ~(ICON_STATUS_CACHED)) | ICON_STATUS_SAVED; + } + + // If the page does not have an id, don't try to insert a new one, cause we + // don't know where the page comes from. Not doing so we may end adding + // a page that otherwise we'd explicitly ignore, like a POST or an error page. + if (mPage.placeId == 0) { + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; + } + + // Expire old favicons to keep up with website changes. Associated icons must + // be expired also when storing a root favicon, because a page may change to + // only have a root favicon. + // Note that here we could also be in the process of adding further payloads + // to a page, and we don't want to expire just added payloads. For this + // reason we only remove expired payloads. + // Oprhan icons are not removed at this time because it'd be expensive. The + // privacy implications are limited, since history removal methods also expire + // orphan icons. + if (mPage.id > 0) { + nsCOMPtr stmt; + stmt = DB->GetStatement( + "DELETE FROM moz_icons_to_pages " + "WHERE page_id = :page_id " + "AND expire_ms < strftime('%s','now','localtime','utc') * 1000 "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = stmt->BindInt64ByName("page_id"_ns, mPage.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Don't associate pages to root domain icons, since those will be returned + // regardless. This saves a lot of work and database space since we don't + // need to store urls and relations. + // Though, this is possible only if both the page and the icon have the same + // host, otherwise we couldn't relate them. + if (!mIcon.rootIcon || !mIcon.host.Equals(mPage.host)) { + // The page may have associated payloads already, and those could have to be + // expired. For example at a certain point a page could decide to stop + // serving its usual 16px and 32px pngs, and use an svg instead. On the + // other side, we could also be in the process of adding more payloads to + // this page, and we should not expire the payloads we just added. For this, + // we use the expiration field as an indicator and remove relations based on + // it being elapsed. We don't remove orphan icons at this time since it + // would have a cost. The privacy hit is limited since history removal + // methods already expire orphan icons. + if (mPage.id == 0) { + // We need to create the page entry. + nsCOMPtr stmt; + stmt = DB->GetStatement( + "INSERT OR IGNORE INTO moz_pages_w_icons (page_url, page_url_hash) " + "VALUES (:page_url, hash(:page_url)) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = URIBinder::Bind(stmt, "page_url"_ns, mPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Then we can create the relations. + nsCOMPtr stmt; + stmt = DB->GetStatement( + "INSERT INTO moz_icons_to_pages (page_id, icon_id, expire_ms) " + "VALUES ((SELECT id from moz_pages_w_icons WHERE page_url_hash = " + "hash(:page_url) AND page_url = :page_url), " + ":icon_id, :expire) " + "ON CONFLICT(page_id, icon_id) DO " + "UPDATE SET expire_ms = :expire "); + NS_ENSURE_STATE(stmt); + + // For some reason using BindingParamsArray here fails execution, so we must + // execute the statements one by one. + // In the future we may want to investigate the reasons, sounds like related + // to contraints. + for (const auto& payload : mIcon.payloads) { + mozStorageStatementScoper scoper(stmt); + nsCOMPtr params; + rv = URIBinder::Bind(stmt, "page_url"_ns, mPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("icon_id"_ns, payload.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("expire"_ns, mIcon.expiration / 1000); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + mIcon.status |= ICON_STATUS_ASSOCIATED; + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + // Finally, dispatch an event to the main thread to notify observers. + nsCOMPtr event = + new NotifyIconObservers(mIcon, mPage, mCallback); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + + // If there is a bookmarked page that redirects to this one, try to update its + // icon as well. + if (!mPage.bookmarkedSpec.IsEmpty() && + !mPage.bookmarkedSpec.Equals(mPage.spec)) { + // Create a new page struct to avoid polluting it with old data. + PageData bookmarkedPage; + bookmarkedPage.spec = mPage.bookmarkedSpec; + RefPtr DB = Database::GetDatabase(); + if (DB && NS_SUCCEEDED(FetchPageInfo(DB, bookmarkedPage))) { + // This will be silent, so be sure to not pass in the current callback. + nsMainThreadPtrHandle nullCallback; + RefPtr event = + new AsyncAssociateIconToPage(mIcon, bookmarkedPage, nullCallback); + Unused << event->Run(); + } + } + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncGetFaviconURLForPage + +AsyncGetFaviconURLForPage::AsyncGetFaviconURLForPage( + const nsACString& aPageSpec, const nsACString& aPageHost, + uint16_t aPreferredWidth, nsIFaviconDataCallback* aCallback) + : Runnable("places::AsyncGetFaviconURLForPage"), + mPreferredWidth(aPreferredWidth == 0 ? UINT16_MAX : aPreferredWidth), + mCallback(new nsMainThreadPtrHolder( + "AsyncGetFaviconURLForPage::mCallback", aCallback)) { + MOZ_ASSERT(NS_IsMainThread()); + mPageSpec.Assign(aPageSpec); + mPageHost.Assign(aPageHost); +} + +NS_IMETHODIMP +AsyncGetFaviconURLForPage::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + IconData iconData; + nsresult rv = + FetchIconPerSpec(DB, mPageSpec, mPageHost, iconData, mPreferredWidth); + NS_ENSURE_SUCCESS(rv, rv); + + // Now notify our callback of the icon spec we retrieved, even if empty. + PageData pageData; + pageData.spec.Assign(mPageSpec); + + nsCOMPtr event = + new NotifyIconObservers(iconData, pageData, mCallback); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncGetFaviconDataForPage + +AsyncGetFaviconDataForPage::AsyncGetFaviconDataForPage( + const nsACString& aPageSpec, const nsACString& aPageHost, + uint16_t aPreferredWidth, nsIFaviconDataCallback* aCallback) + : Runnable("places::AsyncGetFaviconDataForPage"), + mPreferredWidth(aPreferredWidth == 0 ? UINT16_MAX : aPreferredWidth), + mCallback(new nsMainThreadPtrHolder( + "AsyncGetFaviconDataForPage::mCallback", aCallback)) { + MOZ_ASSERT(NS_IsMainThread()); + mPageSpec.Assign(aPageSpec); + mPageHost.Assign(aPageHost); +} + +NS_IMETHODIMP +AsyncGetFaviconDataForPage::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + IconData iconData; + nsresult rv = + FetchIconPerSpec(DB, mPageSpec, mPageHost, iconData, mPreferredWidth); + NS_ENSURE_SUCCESS(rv, rv); + + if (!iconData.spec.IsEmpty()) { + rv = FetchIconInfo(DB, mPreferredWidth, iconData); + if (NS_FAILED(rv)) { + iconData.spec.Truncate(); + } + } + + PageData pageData; + pageData.spec.Assign(mPageSpec); + + nsCOMPtr event = + new NotifyIconObservers(iconData, pageData, mCallback); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncReplaceFaviconData + +AsyncReplaceFaviconData::AsyncReplaceFaviconData(const IconData& aIcon) + : Runnable("places::AsyncReplaceFaviconData"), mIcon(aIcon) { + MOZ_ASSERT(NS_IsMainThread()); +} + +NS_IMETHODIMP +AsyncReplaceFaviconData::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + + mozStorageTransaction transaction( + DB->MainConn(), false, mozIStorageConnection::TRANSACTION_IMMEDIATE); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + nsresult rv = SetIconInfo(DB, mIcon, true); + if (rv == NS_ERROR_NOT_AVAILABLE) { + // There's no previous icon to replace, we don't need to do anything. + (void)transaction.Commit(); + return NS_OK; + } + NS_ENSURE_SUCCESS(rv, rv); + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + // We can invalidate the cache version since we now persist the icon. + nsCOMPtr event = NewRunnableMethod( + "places::AsyncReplaceFaviconData::RemoveIconDataCacheEntry", this, + &AsyncReplaceFaviconData::RemoveIconDataCacheEntry); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult AsyncReplaceFaviconData::RemoveIconDataCacheEntry() { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr iconURI; + nsresult rv = NS_NewURI(getter_AddRefs(iconURI), mIcon.spec); + NS_ENSURE_SUCCESS(rv, rv); + + nsFaviconService* favicons = nsFaviconService::GetFaviconService(); + NS_ENSURE_STATE(favicons); + favicons->mUnassociatedIcons.RemoveEntry(iconURI); + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// NotifyIconObservers + +NotifyIconObservers::NotifyIconObservers( + const IconData& aIcon, const PageData& aPage, + const nsMainThreadPtrHandle& aCallback) + : Runnable("places::NotifyIconObservers"), + mCallback(aCallback), + mIcon(aIcon), + mPage(aPage) {} + +// MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is marked +// MOZ_CAN_RUN_SCRIPT. See bug 1535398. +MOZ_CAN_RUN_SCRIPT_BOUNDARY +NS_IMETHODIMP +NotifyIconObservers::Run() { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr iconURI; + if (!mIcon.spec.IsEmpty()) { + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(iconURI), mIcon.spec)); + if (iconURI) { + // Notify observers only if something changed. + if (mIcon.status & ICON_STATUS_SAVED || + mIcon.status & ICON_STATUS_ASSOCIATED) { + nsCOMPtr pageURI; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(pageURI), mPage.spec)); + if (pageURI) { + // Invalide page-icon image cache, since the icon is about to change. + nsFaviconService* favicons = nsFaviconService::GetFaviconService(); + MOZ_ASSERT(favicons); + if (favicons) { + nsCString pageIconSpec("page-icon:"); + pageIconSpec.Append(mPage.spec); + nsCOMPtr pageIconURI; + if (NS_SUCCEEDED( + NS_NewURI(getter_AddRefs(pageIconURI), pageIconSpec))) { + favicons->ClearImageCache(pageIconURI); + } + } + + // Notify about the favicon change. + dom::Sequence> events; + RefPtr faviconEvent = new dom::PlacesFavicon(); + AppendUTF8toUTF16(mPage.spec, faviconEvent->mUrl); + AppendUTF8toUTF16(mIcon.spec, faviconEvent->mFaviconUrl); + faviconEvent->mPageGuid.Assign(mPage.guid); + bool success = + !!events.AppendElement(faviconEvent.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + dom::PlacesObservers::NotifyListeners(events); + } + } + } + } + + if (!mCallback) { + return NS_OK; + } + + if (mIcon.payloads.Length() > 0) { + IconPayload& payload = mIcon.payloads[0]; + return mCallback->OnComplete(iconURI, payload.data.Length(), + TO_INTBUFFER(payload.data), payload.mimeType, + payload.width); + } + return mCallback->OnComplete(iconURI, 0, TO_INTBUFFER(EmptyCString()), ""_ns, + 0); +} + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncCopyFavicons + +AsyncCopyFavicons::AsyncCopyFavicons(PageData& aFromPage, PageData& aToPage, + nsIFaviconDataCallback* aCallback) + : Runnable("places::AsyncCopyFavicons"), + mFromPage(aFromPage), + mToPage(aToPage), + mCallback(new nsMainThreadPtrHolder( + "AsyncCopyFavicons::mCallback", aCallback)) { + MOZ_ASSERT(NS_IsMainThread()); +} + +NS_IMETHODIMP +AsyncCopyFavicons::Run() { + MOZ_ASSERT(!NS_IsMainThread()); + + IconData icon; + + // Ensure we'll callback and dispatch notifications to the main-thread. + auto cleanup = MakeScopeExit([&]() { + // If we bailed out early, just return a null icon uri, since we didn't + // copy anything. + if (!(icon.status & ICON_STATUS_ASSOCIATED)) { + icon.spec.Truncate(); + } + nsCOMPtr event = + new NotifyIconObservers(icon, mToPage, mCallback); + NS_DispatchToMainThread(event); + }); + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + + nsresult rv = FetchPageInfo(DB, mToPage); + if (rv == NS_ERROR_NOT_AVAILABLE || !mToPage.placeId) { + // We have never seen this page, or we can't add this page to history and + // and it's not a bookmark. We won't add the page. + return NS_OK; + } + NS_ENSURE_SUCCESS(rv, rv); + + // Get just one icon, to check whether the page has any, and to notify later. + rv = FetchIconPerSpec(DB, mFromPage.spec, ""_ns, icon, UINT16_MAX); + NS_ENSURE_SUCCESS(rv, rv); + + if (icon.spec.IsEmpty()) { + // There's nothing to copy. + return NS_OK; + } + + // Insert an entry in moz_pages_w_icons if needed. + if (!mToPage.id) { + // We need to create the page entry. + nsCOMPtr stmt; + stmt = DB->GetStatement( + "INSERT OR IGNORE INTO moz_pages_w_icons (page_url, page_url_hash) " + "VALUES (:page_url, hash(:page_url)) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = URIBinder::Bind(stmt, "page_url"_ns, mToPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + // Required to to fetch the id and the guid. + rv = FetchPageInfo(DB, mToPage); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Create the relations. + nsCOMPtr stmt = DB->GetStatement( + "INSERT OR IGNORE INTO moz_icons_to_pages (page_id, icon_id, expire_ms) " + "SELECT :id, icon_id, expire_ms " + "FROM moz_icons_to_pages " + "WHERE page_id = (SELECT id FROM moz_pages_w_icons WHERE page_url_hash = " + "hash(:url) AND page_url = :url) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = stmt->BindInt64ByName("id"_ns, mToPage.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = URIBinder::Bind(stmt, "url"_ns, mFromPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Setting this will make us send pageChanged notifications. + // The scope exit will take care of the callback and notifications. + icon.status |= ICON_STATUS_ASSOCIATED; + + return NS_OK; +} + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/FaviconHelpers.h b/toolkit/components/places/FaviconHelpers.h new file mode 100644 index 0000000000..ba3407722c --- /dev/null +++ b/toolkit/components/places/FaviconHelpers.h @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#pragma once + +#include "nsIFaviconService.h" +#include "nsIChannelEventSink.h" +#include "nsIInterfaceRequestor.h" +#include "nsIStreamListener.h" +#include "mozIPlacesPendingOperation.h" +#include "nsThreadUtils.h" +#include "nsProxyRelease.h" +#include "imgLoader.h" + +class nsIPrincipal; + +#include "Database.h" +#include "mozilla/storage.h" + +#define ICON_STATUS_UNKNOWN 0 +#define ICON_STATUS_CHANGED 1 << 0 +#define ICON_STATUS_SAVED 1 << 1 +#define ICON_STATUS_ASSOCIATED 1 << 2 +#define ICON_STATUS_CACHED 1 << 3 + +#define TO_CHARBUFFER(_buffer) \ + reinterpret_cast(const_cast(_buffer)) +#define TO_INTBUFFER(_string) \ + reinterpret_cast(const_cast(_string.get())) + +#define PNG_MIME_TYPE "image/png" +#define SVG_MIME_TYPE "image/svg+xml" + +// Always ensure a minimum expiration time, so icons are not already expired +// on addition. +#define MIN_FAVICON_EXPIRATION ((PRTime)1 * 24 * 60 * 60 * PR_USEC_PER_SEC) +// The maximum time we will keep a favicon around. We always ask the cache +// first and default to this value if we can't get a time, or the time we get +// is far in the future. +#define MAX_FAVICON_EXPIRATION ((PRTime)7 * 24 * 60 * 60 * PR_USEC_PER_SEC) + +namespace mozilla { +namespace places { + +/** + * Indicates when a icon should be fetched from network. + */ +enum AsyncFaviconFetchMode { FETCH_NEVER = 0, FETCH_IF_MISSING, FETCH_ALWAYS }; + +/** + * Represents one of the payloads (frames) of an icon entry. + */ +struct IconPayload { + IconPayload() : id(0), width(0) { + data.SetIsVoid(true); + mimeType.SetIsVoid(true); + } + + int64_t id; + uint16_t width; + nsCString data; + nsCString mimeType; +}; + +/** + * Represents an icon entry. + */ +struct IconData { + IconData() + : expiration(0), + fetchMode(FETCH_NEVER), + status(ICON_STATUS_UNKNOWN), + rootIcon(0) {} + + nsCString spec; + nsCString host; + PRTime expiration; + enum AsyncFaviconFetchMode fetchMode; + uint16_t status; // This is a bitset, see ICON_STATUS_* defines above. + uint8_t rootIcon; + CopyableTArray payloads; +}; + +/** + * Data cache for a page entry. + */ +struct PageData { + PageData() : id(0), placeId(0), canAddToHistory(true) { + guid.SetIsVoid(true); + } + + int64_t id; // This is the moz_pages_w_icons id. + int64_t placeId; // This is the moz_places page id. + nsCString spec; + nsCString host; + nsCString bookmarkedSpec; + bool canAddToHistory; // False for disabled history and unsupported schemas. + nsCString guid; +}; + +/** + * Info for a frame. + */ +struct FrameData { + FrameData(uint16_t aIndex, uint16_t aWidth) : index(aIndex), width(aWidth) {} + + uint16_t index; + uint16_t width; +}; + +/** + * Async fetches icon from database or network, associates it with the required + * page and finally notifies the change. + */ +class AsyncFetchAndSetIconForPage final : public Runnable, + public nsIStreamListener, + public nsIInterfaceRequestor, + public nsIChannelEventSink, + public mozIPlacesPendingOperation { + public: + NS_DECL_NSIRUNNABLE + NS_DECL_NSISTREAMLISTENER + NS_DECL_NSIINTERFACEREQUESTOR + NS_DECL_NSICHANNELEVENTSINK + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_MOZIPLACESPENDINGOPERATION + NS_DECL_ISUPPORTS_INHERITED + + /** + * Constructor. + * + * @param aIcon + * Icon to be fetched and associated. + * @param aPage + * Page to which associate the icon. + * @param aFaviconLoadPrivate + * Whether this favicon load is in private browsing. + * @param aCallback + * Function to be called when the fetch-and-associate process finishes. + * @param aLoadingPrincipal + * LoadingPrincipal of the icon to be fetched. + */ + AsyncFetchAndSetIconForPage(IconData& aIcon, PageData& aPage, + bool aFaviconLoadPrivate, + nsIFaviconDataCallback* aCallback, + nsIPrincipal* aLoadingPrincipal, + uint64_t aRequestContextID); + + private: + nsresult FetchFromNetwork(); + virtual ~AsyncFetchAndSetIconForPage() = default; + + nsMainThreadPtrHandle mCallback; + IconData mIcon; + PageData mPage; + const bool mFaviconLoadPrivate; + nsMainThreadPtrHandle mLoadingPrincipal; + bool mCanceled; + nsCOMPtr mRequest; + uint64_t mRequestContextID; +}; + +/** + * Associates the icon to the required page, finally dispatches an event to the + * main thread to notify the change to observers. + */ +class AsyncAssociateIconToPage final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + /** + * Constructor. + * + * @param aIcon + * Icon to be associated. + * @param aPage + * Page to which associate the icon. + * @param aCallback + * Function to be called when the associate process finishes. + */ + AsyncAssociateIconToPage( + const IconData& aIcon, const PageData& aPage, + const nsMainThreadPtrHandle& aCallback); + + private: + nsMainThreadPtrHandle mCallback; + IconData mIcon; + PageData mPage; +}; + +/** + * Asynchronously tries to get the URL of a page's favicon, then notifies the + * given observer. + */ +class AsyncGetFaviconURLForPage final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + /** + * Constructor. + * + * @param aPageSpec + * URL of the page whose favicon's URL we're fetching + * @param aPageHost + * Host of the page whose favicon's URL we're fetching + * @param aCallback + * function to be called once finished + * @param aPreferredWidth + * The preferred size for the icon + */ + AsyncGetFaviconURLForPage(const nsACString& aPageSpec, + const nsACString& aPageHost, + uint16_t aPreferredWidth, + nsIFaviconDataCallback* aCallback); + + private: + uint16_t mPreferredWidth; + nsMainThreadPtrHandle mCallback; + nsCString mPageSpec; + nsCString mPageHost; +}; + +/** + * Asynchronously tries to get the URL and data of a page's favicon, then + * notifies the given observer. + */ +class AsyncGetFaviconDataForPage final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + /** + * Constructor. + * + * @param aPageSpec + * URL of the page whose favicon URL and data we're fetching + * @param aPageHost + * Host of the page whose favicon's URL we're fetching + * @param aPreferredWidth + * The preferred size of the icon. We will try to return an icon close + * to this size. + * @param aCallback + * function to be called once finished + */ + AsyncGetFaviconDataForPage(const nsACString& aPageSpec, + const nsACString& aPageHost, + uint16_t aPreferredWidth, + nsIFaviconDataCallback* aCallback); + + private: + uint16_t mPreferredWidth; + nsMainThreadPtrHandle mCallback; + nsCString mPageSpec; + nsCString mPageHost; +}; + +class AsyncReplaceFaviconData final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + explicit AsyncReplaceFaviconData(const IconData& aIcon); + + private: + nsresult RemoveIconDataCacheEntry(); + + IconData mIcon; +}; + +/** + * Notifies the icon change to favicon observers. + */ +class NotifyIconObservers final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + /** + * Constructor. + * + * @param aIcon + * Icon information. Can be empty if no icon is associated to the page. + * @param aPage + * Page to which the icon information applies. + * @param aCallback + * Function to be notified in all cases. + */ + NotifyIconObservers( + const IconData& aIcon, const PageData& aPage, + const nsMainThreadPtrHandle& aCallback); + + private: + nsMainThreadPtrHandle mCallback; + IconData mIcon; + PageData mPage; +}; + +/** + * Copies Favicons from one page to another one. + */ +class AsyncCopyFavicons final : public Runnable { + public: + NS_DECL_NSIRUNNABLE + + /** + * Constructor. + * + * @param aFromPage + * The originating page. + * @param aToPage + * The destination page. + * @param aFaviconLoadPrivate + * Whether this favicon load is in private browsing. + * @param aCallback + * An optional callback to invoke when done. + */ + AsyncCopyFavicons(PageData& aFromPage, PageData& aToPage, + nsIFaviconDataCallback* aCallback); + + private: + PageData mFromPage; + PageData mToPage; + nsMainThreadPtrHandle mCallback; +}; + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/Helpers.cpp b/toolkit/components/places/Helpers.cpp new file mode 100644 index 0000000000..090743df06 --- /dev/null +++ b/toolkit/components/places/Helpers.cpp @@ -0,0 +1,353 @@ +/* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Helpers.h" +#include "mozIStorageError.h" +#include "prio.h" +#include "nsReadableUtils.h" +#include "nsString.h" +#include "nsNavHistory.h" +#include "mozilla/Base64.h" +#include "mozilla/HashFunctions.h" +#include "mozilla/RandomNum.h" +#include +#include "mozilla/Services.h" + +// The length of guids that are used by history and bookmarks. +#define GUID_LENGTH 12 + +// Maximum number of chars to use for calculating hashes. This value has been +// picked to ensure low hash collisions on a real world common places.sqlite. +// While collisions are not a big deal for functionality, a low ratio allows +// for slightly more efficient SELECTs. +#define MAX_CHARS_TO_HASH 1500U + +extern "C" { + +// Generates a new Places GUID. This function uses C linkage because it's +// called from the Rust synced bookmarks mirror, on the storage thread. +nsresult NS_GeneratePlacesGUID(nsACString* _guid) { + return mozilla::places::GenerateGUID(*_guid); +} + +} // extern "C" + +namespace mozilla { +namespace places { + +//////////////////////////////////////////////////////////////////////////////// +//// AsyncStatementCallback + +NS_IMPL_ISUPPORTS(AsyncStatementCallback, mozIStorageStatementCallback) + +NS_IMETHODIMP +WeakAsyncStatementCallback::HandleResult(mozIStorageResultSet* aResultSet) { + MOZ_ASSERT(false, "Was not expecting a resultset, but got it."); + return NS_OK; +} + +NS_IMETHODIMP +WeakAsyncStatementCallback::HandleCompletion(uint16_t aReason) { return NS_OK; } + +NS_IMETHODIMP +WeakAsyncStatementCallback::HandleError(mozIStorageError* aError) { +#ifdef DEBUG + int32_t result; + nsresult rv = aError->GetResult(&result); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString message; + rv = aError->GetMessage(message); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString warnMsg; + warnMsg.AppendLiteral( + "An error occurred while executing an async statement: "); + warnMsg.AppendInt(result); + warnMsg.Append(' '); + warnMsg.Append(message); + NS_WARNING(warnMsg.get()); +#endif + + return NS_OK; +} + +#define URI_TO_URLCSTRING(uri, spec) \ + nsAutoCString spec; \ + if (NS_FAILED(aURI->GetSpec(spec))) { \ + return NS_ERROR_UNEXPECTED; \ + } + +// Bind URI to statement by index. +nsresult // static +URIBinder::Bind(mozIStorageStatement* aStatement, int32_t aIndex, + nsIURI* aURI) { + NS_ASSERTION(aStatement, "Must have non-null statement"); + NS_ASSERTION(aURI, "Must have non-null uri"); + + URI_TO_URLCSTRING(aURI, spec); + return URIBinder::Bind(aStatement, aIndex, spec); +} + +// Statement URLCString to statement by index. +nsresult // static +URIBinder::Bind(mozIStorageStatement* aStatement, int32_t index, + const nsACString& aURLString) { + NS_ASSERTION(aStatement, "Must have non-null statement"); + return aStatement->BindUTF8StringByIndex( + index, StringHead(aURLString, URI_LENGTH_MAX)); +} + +// Bind URI to statement by name. +nsresult // static +URIBinder::Bind(mozIStorageStatement* aStatement, const nsACString& aName, + nsIURI* aURI) { + NS_ASSERTION(aStatement, "Must have non-null statement"); + NS_ASSERTION(aURI, "Must have non-null uri"); + + URI_TO_URLCSTRING(aURI, spec); + return URIBinder::Bind(aStatement, aName, spec); +} + +// Bind URLCString to statement by name. +nsresult // static +URIBinder::Bind(mozIStorageStatement* aStatement, const nsACString& aName, + const nsACString& aURLString) { + NS_ASSERTION(aStatement, "Must have non-null statement"); + return aStatement->BindUTF8StringByName( + aName, StringHead(aURLString, URI_LENGTH_MAX)); +} + +// Bind URI to params by index. +nsresult // static +URIBinder::Bind(mozIStorageBindingParams* aParams, int32_t aIndex, + nsIURI* aURI) { + NS_ASSERTION(aParams, "Must have non-null statement"); + NS_ASSERTION(aURI, "Must have non-null uri"); + + URI_TO_URLCSTRING(aURI, spec); + return URIBinder::Bind(aParams, aIndex, spec); +} + +// Bind URLCString to params by index. +nsresult // static +URIBinder::Bind(mozIStorageBindingParams* aParams, int32_t index, + const nsACString& aURLString) { + NS_ASSERTION(aParams, "Must have non-null statement"); + return aParams->BindUTF8StringByIndex(index, + StringHead(aURLString, URI_LENGTH_MAX)); +} + +// Bind URI to params by name. +nsresult // static +URIBinder::Bind(mozIStorageBindingParams* aParams, const nsACString& aName, + nsIURI* aURI) { + NS_ASSERTION(aParams, "Must have non-null params array"); + NS_ASSERTION(aURI, "Must have non-null uri"); + + URI_TO_URLCSTRING(aURI, spec); + return URIBinder::Bind(aParams, aName, spec); +} + +// Bind URLCString to params by name. +nsresult // static +URIBinder::Bind(mozIStorageBindingParams* aParams, const nsACString& aName, + const nsACString& aURLString) { + NS_ASSERTION(aParams, "Must have non-null params array"); + + nsresult rv = aParams->BindUTF8StringByName( + aName, StringHead(aURLString, URI_LENGTH_MAX)); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +#undef URI_TO_URLCSTRING + +nsresult GetReversedHostname(nsIURI* aURI, nsString& aRevHost) { + nsAutoCString forward8; + nsresult rv = aURI->GetHost(forward8); + // Not all URIs have a host. + if (NS_FAILED(rv)) return rv; + + // can't do reversing in UTF8, better use 16-bit chars + GetReversedHostname(NS_ConvertUTF8toUTF16(forward8), aRevHost); + return NS_OK; +} + +void GetReversedHostname(const nsString& aForward, nsString& aRevHost) { + ReverseString(aForward, aRevHost); + aRevHost.Append(char16_t('.')); +} + +void ReverseString(const nsString& aInput, nsString& aReversed) { + aReversed.Truncate(0); + for (int32_t i = aInput.Length() - 1; i >= 0; i--) { + aReversed.Append(aInput[i]); + } +} + +nsresult GenerateGUID(nsACString& _guid) { + _guid.Truncate(); + + // Request raw random bytes and base64url encode them. For each set of three + // bytes, we get one character. + const uint32_t kRequiredBytesLength = + static_cast(GUID_LENGTH / 4 * 3); + + uint8_t buffer[kRequiredBytesLength]; + if (!mozilla::GenerateRandomBytesFromOS(buffer, kRequiredBytesLength)) { + return NS_ERROR_FAILURE; + } + + nsresult rv = Base64URLEncode(kRequiredBytesLength, buffer, + Base64URLEncodePaddingPolicy::Omit, _guid); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(_guid.Length() == GUID_LENGTH, "GUID is not the right size!"); + return NS_OK; +} + +bool IsValidGUID(const nsACString& aGUID) { + nsCString::size_type len = aGUID.Length(); + if (len != GUID_LENGTH) { + return false; + } + + for (nsCString::size_type i = 0; i < len; i++) { + char c = aGUID[i]; + if ((c >= 'a' && c <= 'z') || // a-z + (c >= 'A' && c <= 'Z') || // A-Z + (c >= '0' && c <= '9') || // 0-9 + c == '-' || c == '_') { // - or _ + continue; + } + return false; + } + return true; +} + +void TruncateTitle(const nsACString& aTitle, nsACString& aTrimmed) { + if (aTitle.IsVoid()) { + return; + } + aTrimmed = aTitle; + if (aTitle.Length() > TITLE_LENGTH_MAX) { + aTrimmed = StringHead(aTitle, TITLE_LENGTH_MAX); + } +} + +PRTime RoundToMilliseconds(PRTime aTime) { + return aTime - (aTime % PR_USEC_PER_MSEC); +} + +PRTime RoundedPRNow() { return RoundToMilliseconds(PR_Now()); } + +nsresult HashURL(const nsACString& aSpec, const nsACString& aMode, + uint64_t* _hash) { + NS_ENSURE_ARG_POINTER(_hash); + + // HashString doesn't stop at the string boundaries if a length is passed to + // it, so ensure to pass a proper value. + const uint32_t maxLenToHash = + std::min(static_cast(aSpec.Length()), MAX_CHARS_TO_HASH); + + if (aMode.IsEmpty()) { + // URI-like strings (having a prefix before a colon), are handled specially, + // as a 48 bit hash, where first 16 bits are the prefix hash, while the + // other 32 are the string hash. + // The 16 bits have been decided based on the fact hashing all of the IANA + // known schemes, plus "places", does not generate collisions. + // Since we only care about schemes, we just search in the first 50 chars. + // The longest known IANA scheme, at this time, is 30 chars. + const nsDependentCSubstring& strHead = StringHead(aSpec, 50); + nsACString::const_iterator start, tip, end; + strHead.BeginReading(tip); + start = tip; + strHead.EndReading(end); + uint32_t strHash = HashString(aSpec.BeginReading(), maxLenToHash); + if (FindCharInReadable(':', tip, end)) { + const nsDependentCSubstring& prefix = Substring(start, tip); + uint64_t prefixHash = + static_cast(HashString(prefix) & 0x0000FFFF); + // The second half of the url is more likely to be unique, so we add it. + *_hash = (prefixHash << 32) + strHash; + } else { + *_hash = strHash; + } + } else if (aMode.EqualsLiteral("prefix_lo")) { + // Keep only 16 bits. + *_hash = static_cast( + HashString(aSpec.BeginReading(), maxLenToHash) & 0x0000FFFF) + << 32; + } else if (aMode.EqualsLiteral("prefix_hi")) { + // Keep only 16 bits. + *_hash = static_cast( + HashString(aSpec.BeginReading(), maxLenToHash) & 0x0000FFFF) + << 32; + // Make this a prefix upper bound by filling the lowest 32 bits. + *_hash += 0xFFFFFFFF; + } else { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +bool GetHiddenState(bool aIsRedirect, uint32_t aTransitionType) { + return aTransitionType == nsINavHistoryService::TRANSITION_FRAMED_LINK || + aTransitionType == nsINavHistoryService::TRANSITION_EMBED || + aIsRedirect; +} + +nsresult TokenizeQueryString(const nsACString& aQuery, + nsTArray* aTokens) { + // Strip off the "place:" prefix + const uint32_t prefixlen = 6; // = strlen("place:"); + nsCString query; + if (aQuery.Length() >= prefixlen && + Substring(aQuery, 0, prefixlen).EqualsLiteral("place:")) + query = Substring(aQuery, prefixlen); + else + query = aQuery; + + int32_t keyFirstIndex = 0; + int32_t equalsIndex = 0; + for (uint32_t i = 0; i < query.Length(); i++) { + if (query[i] == '&') { + // new clause, save last one + if (i - keyFirstIndex > 1) { + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier, or change the return type to void. + aTokens->AppendElement( + QueryKeyValuePair(query, keyFirstIndex, equalsIndex, i)); + } + keyFirstIndex = equalsIndex = i + 1; + } else if (query[i] == '=') { + equalsIndex = i; + } + } + + // handle last pair, if any + if (query.Length() - keyFirstIndex > 1) { + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier, or change the return type to void. + aTokens->AppendElement( + QueryKeyValuePair(query, keyFirstIndex, equalsIndex, query.Length())); + } + return NS_OK; +} + +void TokensToQueryString(const nsTArray& aTokens, + nsACString& aQuery) { + aQuery = "place:"_ns; + StringJoinAppend(aQuery, "&"_ns, aTokens, + [](nsACString& dst, const QueryKeyValuePair& token) { + dst.Append(token.key); + dst.AppendLiteral("="); + dst.Append(token.value); + }); +} + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/Helpers.h b/toolkit/components/places/Helpers.h new file mode 100644 index 0000000000..fe7ed82b91 --- /dev/null +++ b/toolkit/components/places/Helpers.h @@ -0,0 +1,281 @@ +/* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_Helpers_h_ +#define mozilla_places_Helpers_h_ + +/** + * This file contains helper classes used by various bits of Places code. + */ + +#include "mozilla/storage.h" +#include "nsIURI.h" +#include "nsThreadUtils.h" +#include "nsProxyRelease.h" +#include "prtime.h" +#include "mozilla/Telemetry.h" +#include "mozIStorageStatementCallback.h" + +namespace mozilla { +namespace places { + +//////////////////////////////////////////////////////////////////////////////// +//// Asynchronous Statement Callback Helper + +class WeakAsyncStatementCallback : public mozIStorageStatementCallback { + public: + NS_DECL_MOZISTORAGESTATEMENTCALLBACK + WeakAsyncStatementCallback() = default; + + protected: + virtual ~WeakAsyncStatementCallback() = default; +}; + +class AsyncStatementCallback : public WeakAsyncStatementCallback { + public: + NS_DECL_ISUPPORTS + AsyncStatementCallback() = default; + + protected: + virtual ~AsyncStatementCallback() = default; +}; + +/** + * Macros to use in place of NS_DECL_MOZISTORAGESTATEMENTCALLBACK to declare the + * methods this class assumes silent or notreached. + */ +#define NS_DECL_ASYNCSTATEMENTCALLBACK \ + NS_IMETHOD HandleResult(mozIStorageResultSet*) override; \ + NS_IMETHOD HandleCompletion(uint16_t) override; + +/** + * Utils to bind a specified URI (or URL) to a statement or binding params, at + * the specified index or name. + * @note URIs are always bound as UTF8. + */ +class URIBinder // static +{ + public: + // Bind URI to statement by index. + static nsresult Bind(mozIStorageStatement* statement, int32_t index, + nsIURI* aURI); + // Statement URLCString to statement by index. + static nsresult Bind(mozIStorageStatement* statement, int32_t index, + const nsACString& aURLString); + // Bind URI to statement by name. + static nsresult Bind(mozIStorageStatement* statement, const nsACString& aName, + nsIURI* aURI); + // Bind URLCString to statement by name. + static nsresult Bind(mozIStorageStatement* statement, const nsACString& aName, + const nsACString& aURLString); + // Bind URI to params by index. + static nsresult Bind(mozIStorageBindingParams* aParams, int32_t index, + nsIURI* aURI); + // Bind URLCString to params by index. + static nsresult Bind(mozIStorageBindingParams* aParams, int32_t index, + const nsACString& aURLString); + // Bind URI to params by name. + static nsresult Bind(mozIStorageBindingParams* aParams, + const nsACString& aName, nsIURI* aURI); + // Bind URLCString to params by name. + static nsresult Bind(mozIStorageBindingParams* aParams, + const nsACString& aName, const nsACString& aURLString); +}; + +/** + * This extracts the hostname from the URI and reverses it in the + * form that we use (always ending with a "."). So + * "http://microsoft.com/" becomes "moc.tfosorcim." + * + * The idea behind this is that we can create an index over the items in + * the reversed host name column, and then query for as much or as little + * of the host name as we feel like. + * + * For example, the query "host >= 'gro.allizom.' AND host < 'gro.allizom/' + * Matches all host names ending in '.mozilla.org', including + * 'developer.mozilla.org' and just 'mozilla.org' (since we define all + * reversed host names to end in a period, even 'mozilla.org' matches). + * The important thing is that this operation uses the index. Any substring + * calls in a select statement (even if it's for the beginning of a string) + * will bypass any indices and will be slow). + * + * @param aURI + * URI that contains spec to reverse + * @param aRevHost + * Out parameter + */ +nsresult GetReversedHostname(nsIURI* aURI, nsString& aRevHost); + +/** + * Similar method to GetReversedHostName but for strings + */ +void GetReversedHostname(const nsString& aForward, nsString& aRevHost); + +/** + * Reverses a string. + * + * @param aInput + * The string to be reversed + * @param aReversed + * Output parameter will contain the reversed string + */ +void ReverseString(const nsString& aInput, nsString& aReversed); + +/** + * Generates an 12 character guid to be used by bookmark and history entries. + * + * @note This guid uses the characters a-z, A-Z, 0-9, '-', and '_'. + */ +nsresult GenerateGUID(nsACString& _guid); + +/** + * Determines if the string is a valid guid or not. + * + * @param aGUID + * The guid to test. + * @return true if it is a valid guid, false otherwise. + */ +bool IsValidGUID(const nsACString& aGUID); + +/** + * Truncates the title if it's longer than TITLE_LENGTH_MAX. + * + * @param aTitle + * The title to truncate (if necessary) + * @param aTrimmed + * Output parameter to return the trimmed string + */ +void TruncateTitle(const nsACString& aTitle, nsACString& aTrimmed); + +/** + * Round down a PRTime value to milliseconds precision (...000). + * + * @param aTime + * a PRTime value. + * @return aTime rounded down to milliseconds precision. + */ +PRTime RoundToMilliseconds(PRTime aTime); + +/** + * Round down PR_Now() to milliseconds precision. + * + * @return @see PR_Now, RoundToMilliseconds. + */ +PRTime RoundedPRNow(); + +nsresult HashURL(const nsACString& aSpec, const nsACString& aMode, + uint64_t* _hash); + +class QueryKeyValuePair final { + public: + QueryKeyValuePair(const nsACString& aKey, const nsACString& aValue) { + key = aKey; + value = aValue; + }; + + // QueryKeyValuePair + // + // 01234567890 + // input : qwerty&key=value&qwerty + // ^ ^ ^ + // aKeyBegin | aPastEnd (may point to null terminator) + // aEquals + // + // Special case: if aKeyBegin == aEquals, then there is only one string + // and no equal sign, so we treat the entire thing as a key with no value + + QueryKeyValuePair(const nsACString& aSource, int32_t aKeyBegin, + int32_t aEquals, int32_t aPastEnd) { + if (aEquals == aKeyBegin) aEquals = aPastEnd; + key = Substring(aSource, aKeyBegin, aEquals - aKeyBegin); + if (aPastEnd - aEquals > 0) + value = Substring(aSource, aEquals + 1, aPastEnd - aEquals - 1); + } + nsCString key; + nsCString value; +}; + +/** + * Tokenizes a QueryString. + * + * @param aQuery The string to tokenize. + * @param aTokens The tokenized result. + */ +nsresult TokenizeQueryString(const nsACString& aQuery, + nsTArray* aTokens); + +void TokensToQueryString(const nsTArray& aTokens, + nsACString& aQuery); + +/** + * Used to finalize a statementCache on a specified thread. + */ +template +class FinalizeStatementCacheProxy : public Runnable { + public: + /** + * Constructor. + * + * @param aStatementCache + * The statementCache that should be finalized. + * @param aOwner + * The object that owns the statement cache. This runnable will hold + * a strong reference to it so aStatementCache will not disappear from + * under us. + */ + FinalizeStatementCacheProxy( + mozilla::storage::StatementCache& aStatementCache, + nsISupports* aOwner) + : Runnable("places::FinalizeStatementCacheProxy"), + mStatementCache(aStatementCache), + mOwner(aOwner), + mCallingThread(do_GetCurrentThread()) {} + + NS_IMETHOD Run() override { + mStatementCache.FinalizeStatements(); + // Release the owner back on the calling thread. + NS_ProxyRelease("FinalizeStatementCacheProxy::mOwner", mCallingThread, + mOwner.forget()); + return NS_OK; + } + + protected: + mozilla::storage::StatementCache& mStatementCache; + nsCOMPtr mOwner; + nsCOMPtr mCallingThread; +}; + +/** + * Determines if a visit should be marked as hidden given its transition type + * and whether or not it was a redirect. + * + * @param aIsRedirect + * True if this visit was a redirect, false otherwise. + * @param aTransitionType + * The transition type of the visit. + * @return true if this visit should be hidden. + */ +bool GetHiddenState(bool aIsRedirect, uint32_t aTransitionType); + +/** + * Used to notify a topic to system observers on async execute completion. + */ +class AsyncStatementTelemetryTimer : public AsyncStatementCallback { + public: + explicit AsyncStatementTelemetryTimer(Telemetry::HistogramID aHistogramId, + TimeStamp aStart = TimeStamp::Now()) + : mHistogramId(aHistogramId), mStart(aStart) {} + + NS_IMETHOD HandleCompletion(uint16_t aReason) override; + + private: + const Telemetry::HistogramID mHistogramId; + const TimeStamp mStart; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_Helpers_h_ diff --git a/toolkit/components/places/History.cpp b/toolkit/components/places/History.cpp new file mode 100644 index 0000000000..f0da2392ff --- /dev/null +++ b/toolkit/components/places/History.cpp @@ -0,0 +1,2371 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Attributes.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/MemoryReporting.h" + +#include "mozilla/dom/ContentChild.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/BrowserChild.h" +#include "nsXULAppAPI.h" + +#include "History.h" +#include "nsNavHistory.h" +#include "nsNavBookmarks.h" +#include "Helpers.h" +#include "PlaceInfo.h" +#include "VisitInfo.h" +#include "nsPlacesMacros.h" +#include "NotifyRankingChanged.h" + +#include "mozilla/storage.h" +#include "mozilla/dom/Link.h" +#include "nsDocShellCID.h" +#include "mozilla/Components.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsThreadUtils.h" +#include "nsNetUtil.h" +#include "nsIWidget.h" +#include "nsIXPConnect.h" +#include "nsIXULRuntime.h" +#include "mozilla/Unused.h" +#include "nsContentUtils.h" // for nsAutoScriptBlocker +#include "nsJSUtils.h" +#include "nsStandardURL.h" +#include "mozilla/ipc/URIUtils.h" +#include "nsPrintfCString.h" +#include "nsTHashtable.h" +#include "jsapi.h" +#include "js/Array.h" // JS::GetArrayLength, JS::IsArrayObject, JS::NewArrayObject +#include "js/PropertyAndElement.h" // JS_DefineElement, JS_GetElement, JS_GetProperty +#include "mozilla/StaticPrefs_layout.h" +#include "mozilla/StaticPrefs_places.h" +#include "mozilla/dom/ContentProcessMessageManager.h" +#include "mozilla/dom/Element.h" +#include "mozilla/dom/PlacesObservers.h" +#include "mozilla/dom/PlacesVisit.h" +#include "mozilla/dom/PlacesVisitTitle.h" +#include "mozilla/dom/ScriptSettings.h" + +#include "nsIBrowserWindowTracker.h" +#include "nsImportModule.h" +#include "mozilla/StaticPrefs_browser.h" + +using namespace mozilla::dom; +using namespace mozilla::ipc; + +namespace mozilla::places { + +//////////////////////////////////////////////////////////////////////////////// +//// Global Defines + +// Observer event fired after a visit has been registered in the DB. +#define URI_VISIT_SAVED "uri-visit-saved" + +#define DESTINATIONFILEURI_ANNO "downloads/destinationFileURI"_ns + +//////////////////////////////////////////////////////////////////////////////// +//// VisitData + +struct VisitData { + VisitData() + : placeId(0), + visitId(0), + hidden(true), + shouldUpdateHidden(true), + typed(false), + transitionType(UINT32_MAX), + visitTime(0), + frecency(-1), + lastVisitId(0), + lastVisitTime(0), + visitCount(0), + referrerVisitId(0), + titleChanged(false), + shouldUpdateFrecency(true), + useFrecencyRedirectBonus(false), + source(nsINavHistoryService::VISIT_SOURCE_ORGANIC), + triggeringPlaceId(0), + triggeringSponsoredURLVisitTimeMS(0), + bookmarked(false) { + guid.SetIsVoid(true); + title.SetIsVoid(true); + baseDomain.SetIsVoid(true); + triggeringSearchEngine.SetIsVoid(true); + triggeringSponsoredURL.SetIsVoid(true); + triggeringSponsoredURLBaseDomain.SetIsVoid(true); + } + + explicit VisitData(nsIURI* aURI, nsIURI* aReferrer = nullptr) + : placeId(0), + visitId(0), + hidden(true), + shouldUpdateHidden(true), + typed(false), + transitionType(UINT32_MAX), + visitTime(0), + frecency(-1), + lastVisitId(0), + lastVisitTime(0), + visitCount(0), + referrerVisitId(0), + titleChanged(false), + shouldUpdateFrecency(true), + useFrecencyRedirectBonus(false), + source(nsINavHistoryService::VISIT_SOURCE_ORGANIC), + triggeringPlaceId(0), + triggeringSponsoredURLVisitTimeMS(0), + bookmarked(false) { + MOZ_ASSERT(aURI); + if (aURI) { + (void)aURI->GetSpec(spec); + (void)GetReversedHostname(aURI, revHost); + } + if (aReferrer) { + (void)aReferrer->GetSpec(referrerSpec); + } + guid.SetIsVoid(true); + title.SetIsVoid(true); + baseDomain.SetIsVoid(true); + triggeringSearchEngine.SetIsVoid(true); + triggeringSponsoredURL.SetIsVoid(true); + triggeringSponsoredURLBaseDomain.SetIsVoid(true); + } + + /** + * Sets the transition type of the visit, as well as if it was typed. + * + * @param aTransitionType + * The transition type constant to set. Must be one of the + * TRANSITION_ constants on nsINavHistoryService. + */ + void SetTransitionType(uint32_t aTransitionType) { + typed = aTransitionType == nsINavHistoryService::TRANSITION_TYPED; + transitionType = aTransitionType; + } + + int64_t placeId; + nsCString guid; + int64_t visitId; + nsCString spec; + nsCString baseDomain; + nsString revHost; + bool hidden; + bool shouldUpdateHidden; + bool typed; + uint32_t transitionType; + PRTime visitTime; + int32_t frecency; + int64_t lastVisitId; + PRTime lastVisitTime; + uint32_t visitCount; + + /** + * Stores the title. If this is empty (IsEmpty() returns true), then the + * title should be removed from the Place. If the title is void (IsVoid() + * returns true), then no title has been set on this object, and titleChanged + * should remain false. + */ + nsString title; + + nsCString referrerSpec; + int64_t referrerVisitId; + + // TODO bug 626836 hook up hidden and typed change tracking too! + bool titleChanged; + + // Indicates whether frecency should be updated for this visit. + bool shouldUpdateFrecency; + + // Whether to override the visit type bonus with a redirect bonus when + // calculating frecency on the most recent visit. + bool useFrecencyRedirectBonus; + + uint16_t source; + nsCString triggeringSearchEngine; + int64_t triggeringPlaceId; + nsCString triggeringSponsoredURL; + nsCString triggeringSponsoredURLBaseDomain; + int64_t triggeringSponsoredURLVisitTimeMS; + bool bookmarked; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Anonymous Helpers + +namespace { + +/** + * Convert the given js value to a js array. + * + * @param [in] aValue + * the JS value to convert. + * @param [in] aCtx + * The JSContext for aValue. + * @param [out] _array + * the JS array. + * @param [out] _arrayLength + * _array's length. + */ +nsresult GetJSArrayFromJSValue(JS::Handle aValue, JSContext* aCtx, + JS::MutableHandle _array, + uint32_t* _arrayLength) { + if (aValue.isObjectOrNull()) { + JS::Rooted val(aCtx, aValue.toObjectOrNull()); + bool isArray; + if (!JS::IsArrayObject(aCtx, val, &isArray)) { + return NS_ERROR_UNEXPECTED; + } + if (isArray) { + _array.set(val); + (void)JS::GetArrayLength(aCtx, _array, _arrayLength); + NS_ENSURE_ARG(*_arrayLength > 0); + return NS_OK; + } + } + + // Build a temporary array to store this one item so the code below can + // just loop. + *_arrayLength = 1; + _array.set(JS::NewArrayObject(aCtx, 0)); + NS_ENSURE_TRUE(_array, NS_ERROR_OUT_OF_MEMORY); + + bool rc = JS_DefineElement(aCtx, _array, 0, aValue, 0); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + return NS_OK; +} + +/** + * Attemps to convert a given js value to a nsIURI object. + * @param aCtx + * The JSContext for aValue. + * @param aValue + * The JS value to convert. + * @return the nsIURI object, or null if aValue is not a nsIURI object. + */ +already_AddRefed GetJSValueAsURI(JSContext* aCtx, + const JS::Value& aValue) { + if (!aValue.isPrimitive()) { + nsCOMPtr xpc = nsIXPConnect::XPConnect(); + + nsCOMPtr wrappedObj; + JS::Rooted obj(aCtx, aValue.toObjectOrNull()); + nsresult rv = + xpc->GetWrappedNativeOfJSObject(aCtx, obj, getter_AddRefs(wrappedObj)); + NS_ENSURE_SUCCESS(rv, nullptr); + nsCOMPtr uri = do_QueryInterface(wrappedObj->Native()); + return uri.forget(); + } + return nullptr; +} + +/** + * Obtains an nsIURI from the "uri" property of a JSObject. + * + * @param aCtx + * The JSContext for aObject. + * @param aObject + * The JSObject to get the URI from. + * @param aProperty + * The name of the property to get the URI from. + * @return the URI if it exists. + */ +already_AddRefed GetURIFromJSObject(JSContext* aCtx, + JS::Handle aObject, + const char* aProperty) { + JS::Rooted uriVal(aCtx); + bool rc = JS_GetProperty(aCtx, aObject, aProperty, &uriVal); + NS_ENSURE_TRUE(rc, nullptr); + return GetJSValueAsURI(aCtx, uriVal); +} + +/** + * Attemps to convert a JS value to a string. + * @param aCtx + * The JSContext for aObject. + * @param aValue + * The JS value to convert. + * @param _string + * The string to populate with the value, or set it to void. + */ +void GetJSValueAsString(JSContext* aCtx, const JS::Value& aValue, + nsString& _string) { + if (aValue.isUndefined() || !(aValue.isNull() || aValue.isString())) { + _string.SetIsVoid(true); + return; + } + + // |null| in JS maps to the empty string. + if (aValue.isNull()) { + _string.Truncate(); + return; + } + + if (!AssignJSString(aCtx, _string, aValue.toString())) { + _string.SetIsVoid(true); + } +} + +/** + * Obtains the specified property of a JSObject. + * + * @param aCtx + * The JSContext for aObject. + * @param aObject + * The JSObject to get the string from. + * @param aProperty + * The property to get the value from. + * @param _string + * The string to populate with the value, or set it to void. + */ +void GetStringFromJSObject(JSContext* aCtx, JS::Handle aObject, + const char* aProperty, nsString& _string) { + JS::Rooted val(aCtx); + bool rc = JS_GetProperty(aCtx, aObject, aProperty, &val); + if (!rc) { + _string.SetIsVoid(true); + return; + } + GetJSValueAsString(aCtx, val, _string); +} + +/** + * Obtains the specified property of a JSObject. + * + * @param aCtx + * The JSContext for aObject. + * @param aObject + * The JSObject to get the int from. + * @param aProperty + * The property to get the value from. + * @param _int + * The integer to populate with the value on success. + */ +template +nsresult GetIntFromJSObject(JSContext* aCtx, JS::Handle aObject, + const char* aProperty, IntType* _int) { + JS::Rooted value(aCtx); + bool rc = JS_GetProperty(aCtx, aObject, aProperty, &value); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + if (value.isUndefined()) { + return NS_ERROR_INVALID_ARG; + } + NS_ENSURE_ARG(value.isPrimitive()); + NS_ENSURE_ARG(value.isNumber()); + + double num; + rc = JS::ToNumber(aCtx, value, &num); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + NS_ENSURE_ARG(IntType(num) == num); + + *_int = IntType(num); + return NS_OK; +} + +/** + * Obtains the specified property of a JSObject. + * + * @pre aArray must be an Array object. + * + * @param aCtx + * The JSContext for aArray. + * @param aArray + * The JSObject to get the object from. + * @param aIndex + * The index to get the object from. + * @param objOut + * Set to the JSObject pointer on success. + */ +nsresult GetJSObjectFromArray(JSContext* aCtx, JS::Handle aArray, + uint32_t aIndex, + JS::MutableHandle objOut) { + JS::Rooted value(aCtx); + bool rc = JS_GetElement(aCtx, aArray, aIndex, &value); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + NS_ENSURE_ARG(!value.isPrimitive()); + objOut.set(&value.toObject()); + return NS_OK; +} + +} // namespace + +class VisitedQuery final : public AsyncStatementCallback { + public: + NS_DECL_ISUPPORTS_INHERITED + + static nsresult Start(nsIURI* aURI, + History::ContentParentSet&& aContentProcessesToNotify) { + MOZ_ASSERT(aURI, "Null URI"); + MOZ_ASSERT(XRE_IsParentProcess()); + + History* history = History::GetService(); + NS_ENSURE_STATE(history); + RefPtr query = + new VisitedQuery(aURI, std::move(aContentProcessesToNotify)); + return history->QueueVisitedStatement(std::move(query)); + } + + static nsresult Start(nsIURI* aURI, + mozIVisitedStatusCallback* aCallback = nullptr) { + MOZ_ASSERT(aURI, "Null URI"); + MOZ_ASSERT(XRE_IsParentProcess()); + + nsMainThreadPtrHandle callback( + new nsMainThreadPtrHolder( + "mozIVisitedStatusCallback", aCallback)); + + History* history = History::GetService(); + NS_ENSURE_STATE(history); + RefPtr query = new VisitedQuery(aURI, callback); + return history->QueueVisitedStatement(std::move(query)); + } + + void Execute(mozIStorageAsyncStatement& aStatement) { + // Bind by index for performance. + nsresult rv = URIBinder::Bind(&aStatement, 0, mURI); + if (NS_WARN_IF(NS_FAILED(rv))) { + return; + } + + nsCOMPtr handle; + rv = aStatement.ExecuteAsync(this, getter_AddRefs(handle)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } + + NS_IMETHOD HandleResult(mozIStorageResultSet* aResults) override { + // If this method is called, we've gotten results, which means we have a + // visit. + mIsVisited = true; + return NS_OK; + } + + NS_IMETHOD HandleError(mozIStorageError* aError) override { + // mIsVisited is already set to false, and that's the assumption we will + // make if an error occurred. + return NS_OK; + } + + NS_IMETHOD HandleCompletion(uint16_t aReason) override { + if (aReason == mozIStorageStatementCallback::REASON_FINISHED) { + NotifyVisitedStatus(); + } + return NS_OK; + } + + void NotifyVisitedStatus() { + // If an external handling callback is provided, just notify through it. + if (mCallback) { + mCallback->IsVisited(mURI, mIsVisited); + return; + } + + if (History* history = History::GetService()) { + auto status = mIsVisited ? IHistory::VisitedStatus::Visited + : IHistory::VisitedStatus::Unvisited; + history->NotifyVisited(mURI, status, &mContentProcessesToNotify); + } + } + + private: + explicit VisitedQuery( + nsIURI* aURI, + const nsMainThreadPtrHandle& aCallback) + : mURI(aURI), mCallback(aCallback) {} + + explicit VisitedQuery(nsIURI* aURI, + History::ContentParentSet&& aContentProcessesToNotify) + : mURI(aURI), + mContentProcessesToNotify(std::move(aContentProcessesToNotify)) {} + + ~VisitedQuery() = default; + + nsCOMPtr mURI; + nsMainThreadPtrHandle mCallback; + History::ContentParentSet mContentProcessesToNotify; + bool mIsVisited = false; +}; + +NS_IMPL_ISUPPORTS_INHERITED0(VisitedQuery, AsyncStatementCallback) + +/** + * Notifies observers about a visit or an array of visits. + */ +class NotifyManyVisitsObservers : public Runnable { + public: + explicit NotifyManyVisitsObservers(const VisitData& aPlace) + : Runnable("places::NotifyManyVisitsObservers"), + mPlaces({aPlace}), + mHistory(History::GetService()) {} + + explicit NotifyManyVisitsObservers(nsTArray&& aPlaces) + : Runnable("places::NotifyManyVisitsObservers"), + mPlaces(std::move(aPlaces)), + mHistory(History::GetService()) {} + + nsresult NotifyVisit(nsNavHistory* aNavHistory, + nsCOMPtr& aObsService, PRTime aNow, + nsIURI* aURI, const VisitData& aPlace) { + if (aObsService) { + DebugOnly rv = + aObsService->NotifyObservers(aURI, URI_VISIT_SAVED, nullptr); + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "Could not notify observers"); + } + + if (aNow - aPlace.visitTime < RECENTLY_VISITED_URIS_MAX_AGE) { + mHistory->AppendToRecentlyVisitedURIs(aURI, aPlace.hidden); + } + mHistory->NotifyVisited(aURI, IHistory::VisitedStatus::Visited); + + aNavHistory->UpdateDaysOfHistory(aPlace.visitTime); + + return NS_OK; + } + + void AddPlaceForNotify(const VisitData& aPlace, + Sequence>& aEvents) { + if (aPlace.transitionType == nsINavHistoryService::TRANSITION_EMBED) { + return; + } + + RefPtr visitEvent = new PlacesVisit(); + visitEvent->mVisitId = aPlace.visitId; + visitEvent->mUrl.Assign(NS_ConvertUTF8toUTF16(aPlace.spec)); + visitEvent->mVisitTime = aPlace.visitTime / 1000; + visitEvent->mReferringVisitId = aPlace.referrerVisitId; + visitEvent->mTransitionType = aPlace.transitionType; + visitEvent->mPageGuid.Assign(aPlace.guid); + visitEvent->mHidden = aPlace.hidden; + visitEvent->mVisitCount = aPlace.visitCount + 1; // Add current visit + visitEvent->mTypedCount = static_cast(aPlace.typed); + visitEvent->mLastKnownTitle.Assign(aPlace.title); + bool success = !!aEvents.AppendElement(visitEvent.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + if (aPlace.titleChanged) { + RefPtr titleEvent = new PlacesVisitTitle(); + titleEvent->mUrl.Assign(NS_ConvertUTF8toUTF16(aPlace.spec)); + titleEvent->mPageGuid.Assign(aPlace.guid); + titleEvent->mTitle.Assign(aPlace.title); + bool success = !!aEvents.AppendElement(titleEvent.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + } + } + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is marked + // MOZ_CAN_RUN_SCRIPT. See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY + NS_IMETHOD Run() override { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + + // We are in the main thread, no need to lock. + if (mHistory->IsShuttingDown()) { + // If we are shutting down, we cannot notify the observers. + return NS_OK; + } + + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + if (!navHistory) { + NS_WARNING( + "Trying to notify visits observers but cannot get the history " + "service!"); + return NS_OK; + } + + nsCOMPtr obsService = + mozilla::services::GetObserverService(); + + Sequence> events; + PRTime now = PR_Now(); + for (uint32_t i = 0; i < mPlaces.Length(); ++i) { + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), mPlaces[i].spec)); + if (!uri) { + return NS_ERROR_UNEXPECTED; + } + AddPlaceForNotify(mPlaces[i], events); + + nsresult rv = NotifyVisit(navHistory, obsService, now, uri, mPlaces[i]); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (events.Length() > 0) { + PlacesObservers::NotifyListeners(events); + } + + return NS_OK; + } + + private: + AutoTArray mPlaces; + RefPtr mHistory; +}; + +/** + * Notifies observers about a pages title changing. + */ +class NotifyTitleObservers : public Runnable { + public: + /** + * Notifies observers on the main thread. + * + * @param aSpec + * The spec of the URI to notify about. + * @param aTitle + * The new title to notify about. + */ + NotifyTitleObservers(const nsCString& aSpec, const nsString& aTitle, + const nsCString& aGUID) + : Runnable("places::NotifyTitleObservers"), + mSpec(aSpec), + mTitle(aTitle), + mGUID(aGUID) {} + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is marked + // MOZ_CAN_RUN_SCRIPT. See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY + NS_IMETHOD Run() override { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + + RefPtr titleEvent = new PlacesVisitTitle(); + titleEvent->mUrl.Assign(NS_ConvertUTF8toUTF16(mSpec)); + titleEvent->mPageGuid.Assign(mGUID); + titleEvent->mTitle.Assign(mTitle); + + Sequence> events; + bool success = !!events.AppendElement(titleEvent.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + PlacesObservers::NotifyListeners(events); + + return NS_OK; + } + + private: + const nsCString mSpec; + const nsString mTitle; + const nsCString mGUID; +}; + +/** + * Helper class for methods which notify their callers through the + * mozIVisitInfoCallback interface. + */ +class NotifyPlaceInfoCallback : public Runnable { + public: + NotifyPlaceInfoCallback( + const nsMainThreadPtrHandle& aCallback, + const VisitData& aPlace, bool aIsSingleVisit, nsresult aResult) + : Runnable("places::NotifyPlaceInfoCallback"), + mCallback(aCallback), + mPlace(aPlace), + mResult(aResult), + mIsSingleVisit(aIsSingleVisit) { + MOZ_ASSERT(aCallback, "Must pass a non-null callback!"); + } + + NS_IMETHOD Run() override { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + + bool hasValidURIs = true; + nsCOMPtr referrerURI; + if (!mPlace.referrerSpec.IsEmpty()) { + MOZ_ALWAYS_SUCCEEDS( + NS_NewURI(getter_AddRefs(referrerURI), mPlace.referrerSpec)); + hasValidURIs = !!referrerURI; + } + + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), mPlace.spec)); + hasValidURIs = hasValidURIs && !!uri; + + nsCOMPtr place; + if (mIsSingleVisit) { + nsCOMPtr visit = + new VisitInfo(mPlace.visitId, mPlace.visitTime, mPlace.transitionType, + referrerURI.forget()); + PlaceInfo::VisitsArray visits; + (void)visits.AppendElement(visit); + + // The frecency isn't exposed because it may not reflect the updated value + // in the case of InsertVisitedURIs. + place = new PlaceInfo(mPlace.placeId, mPlace.guid, uri.forget(), + mPlace.title, -1, visits); + } else { + // Same as above. + place = new PlaceInfo(mPlace.placeId, mPlace.guid, uri.forget(), + mPlace.title, -1); + } + + if (NS_SUCCEEDED(mResult) && hasValidURIs) { + (void)mCallback->HandleResult(place); + } else { + (void)mCallback->HandleError(mResult, place); + } + + return NS_OK; + } + + private: + nsMainThreadPtrHandle mCallback; + VisitData mPlace; + const nsresult mResult; + bool mIsSingleVisit; +}; + +/** + * Notifies a callback object when the operation is complete. + */ +class NotifyCompletion : public Runnable { + public: + explicit NotifyCompletion( + const nsMainThreadPtrHandle& aCallback, + uint32_t aUpdatedCount = 0) + : Runnable("places::NotifyCompletion"), + mCallback(aCallback), + mUpdatedCount(aUpdatedCount) { + MOZ_ASSERT(aCallback, "Must pass a non-null callback!"); + } + + NS_IMETHOD Run() override { + if (NS_IsMainThread()) { + (void)mCallback->HandleCompletion(mUpdatedCount); + } else { + (void)NS_DispatchToMainThread(this); + } + return NS_OK; + } + + private: + nsMainThreadPtrHandle mCallback; + uint32_t mUpdatedCount; +}; + +/** + * Checks to see if we can add aURI to history, and dispatches an error to + * aCallback (if provided) if we cannot. + * + * @param aURI + * The URI to check. + * @param [optional] aGUID + * The guid of the URI to check. This is passed back to the callback. + * @param [optional] aCallback + * The callback to notify if the URI cannot be added to history. + * @return true if the URI can be added to history, false otherwise. + */ +bool CanAddURI(nsIURI* aURI, const nsCString& aGUID = ""_ns, + mozIVisitInfoCallback* aCallback = nullptr) { + MOZ_ASSERT(NS_IsMainThread()); + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(navHistory, false); + + bool canAdd; + nsresult rv = navHistory->CanAddURI(aURI, &canAdd); + if (NS_SUCCEEDED(rv) && canAdd) { + return true; + }; + + // We cannot add the URI. Notify the callback, if we were given one. + if (aCallback) { + VisitData place(aURI); + place.guid = aGUID; + nsMainThreadPtrHandle callback( + new nsMainThreadPtrHolder( + "mozIVisitInfoCallback", aCallback)); + nsCOMPtr event = new NotifyPlaceInfoCallback( + callback, place, true, NS_ERROR_INVALID_ARG); + (void)NS_DispatchToMainThread(event); + } + + return false; +} + +/** + * Adds a visit to the database. + */ +class InsertVisitedURIs final : public Runnable { + public: + /** + * Adds a visit to the database asynchronously. + * + * @param aConnection + * The database connection to use for these operations. + * @param aPlaces + * The locations to record visits. + * @param [optional] aCallback + * The callback to notify about the visit. + */ + static nsresult Start(mozIStorageConnection* aConnection, + nsTArray&& aPlaces, + mozIVisitInfoCallback* aCallback = nullptr, + uint32_t aInitialUpdatedCount = 0) { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + MOZ_ASSERT(aPlaces.Length() > 0, "Must pass a non-empty array!"); + + // Make sure nsNavHistory service is up before proceeding: + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + MOZ_ASSERT(navHistory, "Could not get nsNavHistory?!"); + if (!navHistory) { + return NS_ERROR_FAILURE; + } + + nsMainThreadPtrHandle callback( + new nsMainThreadPtrHolder( + "mozIVisitInfoCallback", aCallback)); + bool ignoreErrors = false, ignoreResults = false; + if (aCallback) { + // We ignore errors from either of these methods in case old JS consumers + // don't implement them (in which case they will get error/result + // notifications as normal). + Unused << aCallback->GetIgnoreErrors(&ignoreErrors); + Unused << aCallback->GetIgnoreResults(&ignoreResults); + } + RefPtr event = new InsertVisitedURIs( + aConnection, std::move(aPlaces), callback, ignoreErrors, ignoreResults, + aInitialUpdatedCount); + + // Get the target thread, and then start the work! + nsCOMPtr target = do_GetInterface(aConnection); + NS_ENSURE_TRUE(target, NS_ERROR_UNEXPECTED); + nsresult rv = target->Dispatch(event, NS_DISPATCH_NORMAL); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + NS_IMETHOD Run() override { + MOZ_ASSERT(!NS_IsMainThread(), + "This should not be called on the main thread"); + + // The inner run method may bail out at any point, so we ensure we do + // whatever we can and then notify the main thread we're done. + nsresult rv = InnerRun(); + + if (mSuccessfulUpdatedCount > 0) { + NS_DispatchToMainThread(new NotifyRankingChanged()); + } + if (!!mCallback) { + NS_DispatchToMainThread( + new NotifyCompletion(mCallback, mSuccessfulUpdatedCount)); + } + return rv; + } + + nsresult InnerRun() { + MOZ_ASSERT(!NS_IsMainThread()); + // Prevent Shutdown() from proceeding while this is running. + MutexAutoLock lockedScope(mHistory->mBlockShutdownMutex); + // Check if we were already shutting down. + if (mHistory->IsShuttingDown()) { + return NS_OK; + } + + mozStorageTransaction transaction( + mDBConn, false, mozIStorageConnection::TRANSACTION_IMMEDIATE); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + const VisitData* lastFetchedPlace = nullptr; + uint32_t lastFetchedVisitCount = 0; + bool shouldChunkNotifications = mPlaces.Length() > NOTIFY_VISITS_CHUNK_SIZE; + nsTArray notificationChunk; + if (shouldChunkNotifications) { + notificationChunk.SetCapacity(NOTIFY_VISITS_CHUNK_SIZE); + } + for (nsTArray::size_type i = 0; i < mPlaces.Length(); i++) { + VisitData& place = mPlaces.ElementAt(i); + + // Fetching from the database can overwrite this information, so save it + // apart. + bool typed = place.typed; + bool hidden = place.hidden; + + // We can avoid a database lookup if it's the same place as the last + // visit we added. + bool known = + lastFetchedPlace && lastFetchedPlace->spec.Equals(place.spec); + if (!known) { + nsresult rv = mHistory->FetchPageInfo(place, &known); + if (NS_FAILED(rv)) { + if (!!mCallback && !mIgnoreErrors) { + nsCOMPtr event = + new NotifyPlaceInfoCallback(mCallback, place, true, rv); + return NS_DispatchToMainThread(event); + } + return NS_OK; + } + lastFetchedPlace = &mPlaces.ElementAt(i); + lastFetchedVisitCount = lastFetchedPlace->visitCount; + } else { + // Copy over the data from the already known place. + place.placeId = lastFetchedPlace->placeId; + place.guid = lastFetchedPlace->guid; + place.lastVisitId = lastFetchedPlace->visitId; + place.lastVisitTime = lastFetchedPlace->visitTime; + if (!place.title.IsVoid()) { + place.titleChanged = !lastFetchedPlace->title.Equals(place.title); + } + place.frecency = lastFetchedPlace->frecency; + // Add one visit for the previous loop. + place.visitCount = ++lastFetchedVisitCount; + } + + // If any transition is typed, ensure the page is marked as typed. + if (typed != lastFetchedPlace->typed) { + place.typed = true; + } + + // If any transition is visible, ensure the page is marked as visible. + if (hidden != lastFetchedPlace->hidden) { + place.hidden = false; + } + + // If this is a new page, or the existing page was already visible, + // there's no need to try to unhide it. + if (!known || !lastFetchedPlace->hidden) { + place.shouldUpdateHidden = false; + } + + FetchReferrerInfo(place); + UpdateVisitSource(place, mHistory); + + nsresult rv = DoDatabaseInserts(known, place); + if (!!mCallback) { + // Check if consumers wanted to be notified about success/failure, + // depending on whether this action succeeded or not. + if ((NS_SUCCEEDED(rv) && !mIgnoreResults) || + (NS_FAILED(rv) && !mIgnoreErrors)) { + nsCOMPtr event = + new NotifyPlaceInfoCallback(mCallback, place, true, rv); + nsresult rv2 = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv2, rv2); + } + } + NS_ENSURE_SUCCESS(rv, rv); + + if (shouldChunkNotifications) { + int32_t numRemaining = (int32_t)(mPlaces.Length() - (i + 1)); + notificationChunk.AppendElement(place); + if (notificationChunk.Length() == NOTIFY_VISITS_CHUNK_SIZE || + numRemaining == 0) { + nsCOMPtr event = + new NotifyManyVisitsObservers(std::move(notificationChunk)); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t nextCapacity = + std::min(NOTIFY_VISITS_CHUNK_SIZE, numRemaining); + notificationChunk.SetCapacity(nextCapacity); + } + } + + // If we get here, we must have been successful adding/updating this + // visit/place, so update the count: + mSuccessfulUpdatedCount++; + } + + { + // Trigger insertions for all the new origins of the places we inserted. + nsAutoCString query("DELETE FROM moz_updateoriginsinsert_temp"); + nsCOMPtr stmt = mHistory->GetStatement(query); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + nsresult rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + { + // Trigger frecency updates for all those origins. + nsAutoCString query("DELETE FROM moz_updateoriginsupdate_temp"); + nsCOMPtr stmt = mHistory->GetStatement(query); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + nsresult rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsresult rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + // If we don't need to chunk the notifications, just notify using the + // original mPlaces array. + if (!shouldChunkNotifications) { + nsCOMPtr event = + new NotifyManyVisitsObservers(std::move(mPlaces)); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; + } + + private: + InsertVisitedURIs( + mozIStorageConnection* aConnection, nsTArray&& aPlaces, + const nsMainThreadPtrHandle& aCallback, + bool aIgnoreErrors, bool aIgnoreResults, uint32_t aInitialUpdatedCount) + : Runnable("places::InsertVisitedURIs"), + mDBConn(aConnection), + mPlaces(std::move(aPlaces)), + mCallback(aCallback), + mIgnoreErrors(aIgnoreErrors), + mIgnoreResults(aIgnoreResults), + mSuccessfulUpdatedCount(aInitialUpdatedCount), + mHistory(History::GetService()) { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + +#ifdef DEBUG + for (nsTArray::size_type i = 0; i < mPlaces.Length(); i++) { + nsCOMPtr uri; + MOZ_ASSERT(NS_SUCCEEDED(NS_NewURI(getter_AddRefs(uri), mPlaces[i].spec))); + MOZ_ASSERT(CanAddURI(uri), + "Passed a VisitData with a URI we cannot add to history!"); + } +#endif + } + + /** + * Inserts or updates the entry in moz_places for this visit, adds the visit, + * and updates the frecency of the place. + * + * @param aKnown + * True if we already have an entry for this place in moz_places, false + * otherwise. + * @param aPlace + * The place we are adding a visit for. + */ + nsresult DoDatabaseInserts(bool aKnown, VisitData& aPlace) { + MOZ_ASSERT(!NS_IsMainThread(), + "This should not be called on the main thread"); + + // If the page was in moz_places, we need to update the entry. + nsresult rv; + if (aKnown) { + rv = mHistory->UpdatePlace(aPlace); + NS_ENSURE_SUCCESS(rv, rv); + } + // Otherwise, the page was not in moz_places, so now we have to add it. + else { + rv = mHistory->InsertPlace(aPlace); + NS_ENSURE_SUCCESS(rv, rv); + aPlace.placeId = nsNavHistory::sLastInsertedPlaceId; + } + MOZ_ASSERT(aPlace.placeId > 0); + + rv = AddVisit(aPlace); + NS_ENSURE_SUCCESS(rv, rv); + + // TODO (bug 623969) we shouldn't update this after each visit, but + // rather only for each unique place to save disk I/O. + + // Don't update frecency if the page should not appear in autocomplete. + if (aPlace.shouldUpdateFrecency) { + rv = UpdateFrecency(aPlace); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; + } + + /** + * Fetches information about a referrer for aPlace if it was a recent + * visit or not. + * + * @param aPlace + * The VisitData for the visit we will eventually add. + * + */ + void FetchReferrerInfo(VisitData& aPlace) { + if (aPlace.referrerSpec.IsEmpty()) { + return; + } + + VisitData referrer; + referrer.spec = aPlace.referrerSpec; + // If the referrer is the same as the page, we don't need to fetch it. + if (aPlace.referrerSpec.Equals(aPlace.spec)) { + referrer = aPlace; + // The page last visit id is also the referrer visit id. + aPlace.referrerVisitId = aPlace.lastVisitId; + } else { + bool exists = false; + if (NS_SUCCEEDED(mHistory->FetchPageInfo(referrer, &exists)) && exists) { + // Copy the referrer last visit id. + aPlace.referrerVisitId = referrer.lastVisitId; + } + } + + // Check if the page has effectively been visited recently, otherwise + // discard the referrer info. + if (!aPlace.referrerVisitId || !referrer.lastVisitTime || + aPlace.visitTime - referrer.lastVisitTime > RECENT_EVENT_THRESHOLD) { + // We will not be using the referrer data. + aPlace.referrerSpec.Truncate(); + aPlace.referrerVisitId = 0; + } + } + + /** + * Adds a visit for _place and updates it with the right visit id. + * + * @param _place + * The VisitData for the place we need to know visit information about. + */ + nsresult AddVisit(VisitData& _place) { + MOZ_ASSERT(_place.placeId > 0); + + nsresult rv; + nsCOMPtr stmt; + stmt = mHistory->GetStatement( + "INSERT INTO moz_historyvisits " + "(from_visit, place_id, visit_date, visit_type, session, source, " + "triggeringPlaceId) " + "VALUES (:from_visit, :page_id, :visit_date, :visit_type, 0, :source, " + ":triggeringPlaceId) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = stmt->BindInt64ByName("page_id"_ns, _place.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("from_visit"_ns, _place.referrerVisitId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("visit_date"_ns, _place.visitTime); + NS_ENSURE_SUCCESS(rv, rv); + uint32_t transitionType = _place.transitionType; + MOZ_ASSERT(transitionType >= nsINavHistoryService::TRANSITION_LINK && + transitionType <= nsINavHistoryService::TRANSITION_RELOAD, + "Invalid transition type!"); + rv = stmt->BindInt32ByName("visit_type"_ns, (int32_t)transitionType); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("source"_ns, _place.source); + NS_ENSURE_SUCCESS(rv, rv); + if (_place.triggeringPlaceId != 0) { + rv = stmt->BindInt64ByName("triggeringPlaceId"_ns, + _place.triggeringPlaceId); + } else { + rv = stmt->BindNullByName("triggeringPlaceId"_ns); + } + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + _place.visitId = nsNavHistory::sLastInsertedVisitId; + MOZ_ASSERT(_place.visitId > 0); + + return NS_OK; + } + + /** + * Updates the frecency, and possibly the hidden-ness of aPlace. + * + * @param aPlace + * The VisitData for the place we want to update. + */ + nsresult UpdateFrecency(const VisitData& aPlace) { + MOZ_ASSERT(aPlace.shouldUpdateFrecency); + MOZ_ASSERT(aPlace.placeId > 0); + + nsresult rv; + { // First, set our frecency to the proper value. + nsCOMPtr stmt = mHistory->GetStatement( + "UPDATE moz_places " + "SET frecency = CALCULATE_FRECENCY(:page_id, :redirect) " + "WHERE id = :page_id"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = stmt->BindInt64ByName("page_id"_ns, aPlace.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = + stmt->BindInt32ByName("redirect"_ns, aPlace.useFrecencyRedirectBonus); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (!aPlace.hidden && aPlace.shouldUpdateHidden) { + // Mark the page as not hidden if the frecency is now nonzero. + nsCOMPtr stmt; + stmt = mHistory->GetStatement( + "UPDATE moz_places " + "SET hidden = 0 " + "WHERE id = :page_id AND frecency <> 0"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = stmt->BindInt64ByName("page_id"_ns, aPlace.placeId); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (StaticPrefs:: + places_frecency_pages_alternative_featureGate_AtStartup()) { + nsCOMPtr stmt = mHistory->GetStatement( + "UPDATE moz_places " + "SET alt_frecency = CALCULATE_ALT_FRECENCY(id, :redirect), " + "recalc_alt_frecency = 0 " + "WHERE id = :page_id"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + rv = stmt->BindInt64ByName("page_id"_ns, aPlace.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = + stmt->BindInt32ByName("redirect"_ns, aPlace.useFrecencyRedirectBonus); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; + } + + nsresult UpdateVisitSource(VisitData& aPlace, History* aHistory) { + if (aPlace.bookmarked) { + aPlace.source = nsINavHistoryService::VISIT_SOURCE_BOOKMARKED; + } else if (!aPlace.triggeringSearchEngine.IsEmpty()) { + aPlace.source = nsINavHistoryService::VISIT_SOURCE_SEARCHED; + } else { + aPlace.source = nsINavHistoryService::VISIT_SOURCE_ORGANIC; + } + + if (aPlace.triggeringSponsoredURL.IsEmpty()) { + // No triggeringSponsoredURL. + return NS_OK; + } + + if ((aPlace.visitTime - + aPlace.triggeringSponsoredURLVisitTimeMS * PR_USEC_PER_MSEC) > + StaticPrefs::browser_places_sponsoredSession_timeoutSecs() * + PR_USEC_PER_SEC) { + // Sponsored session timeout. + return NS_OK; + } + + if (aPlace.spec.Equals(aPlace.triggeringSponsoredURL)) { + // This place is the triggeringSponsoredURL. + aPlace.source = nsINavHistoryService::VISIT_SOURCE_SPONSORED; + return NS_OK; + } + + if (!aPlace.baseDomain.Equals(aPlace.triggeringSponsoredURLBaseDomain)) { + // The base domain is not same. + return NS_OK; + } + + nsCOMPtr stmt; + stmt = aHistory->GetStatement( + "SELECT id FROM moz_places h " + "WHERE url_hash = hash(:url) AND url = :url"); + NS_ENSURE_STATE(stmt); + nsresult rv = + URIBinder::Bind(stmt, "url"_ns, aPlace.triggeringSponsoredURL); + NS_ENSURE_SUCCESS(rv, rv); + + mozStorageStatementScoper scoper(stmt); + + bool exists; + rv = stmt->ExecuteStep(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (exists) { + rv = stmt->GetInt64(0, &aPlace.triggeringPlaceId); + NS_ENSURE_SUCCESS(rv, rv); + } else { + Telemetry::ScalarAdd( + Telemetry::ScalarID::PLACES_SPONSORED_VISIT_NO_TRIGGERING_URL, 1); + } + + aPlace.source = nsINavHistoryService::VISIT_SOURCE_SPONSORED; + + return NS_OK; + } + + mozIStorageConnection* mDBConn; + + nsTArray mPlaces; + + nsMainThreadPtrHandle mCallback; + + bool mIgnoreErrors; + + bool mIgnoreResults; + + uint32_t mSuccessfulUpdatedCount; + + /** + * Strong reference to the History object because we do not want it to + * disappear out from under us. + */ + RefPtr mHistory; +}; + +/** + * Sets the page title for a page in moz_places (if necessary). + */ +class SetPageTitle : public Runnable { + public: + /** + * Sets a pages title in the database asynchronously. + * + * @param aConnection + * The database connection to use for this operation. + * @param aURI + * The URI to set the page title on. + * @param aTitle + * The title to set for the page, if the page exists. + */ + static nsresult Start(mozIStorageConnection* aConnection, nsIURI* aURI, + const nsAString& aTitle) { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + MOZ_ASSERT(aURI, "Must pass a non-null URI object!"); + + nsCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr event = new SetPageTitle(spec, aTitle); + + // Get the target thread, and then start the work! + nsCOMPtr target = do_GetInterface(aConnection); + NS_ENSURE_TRUE(target, NS_ERROR_UNEXPECTED); + rv = target->Dispatch(event, NS_DISPATCH_NORMAL); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + NS_IMETHOD Run() override { + MOZ_ASSERT(!NS_IsMainThread(), + "This should not be called on the main thread"); + + // First, see if the page exists in the database (we'll need its id later). + bool exists; + nsresult rv = mHistory->FetchPageInfo(mPlace, &exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!exists || !mPlace.titleChanged) { + // We have no record of this page, or we have no title change, so there + // is no need to do any further work. + return NS_OK; + } + + MOZ_ASSERT(mPlace.placeId > 0, "We somehow have an invalid place id here!"); + + // Now we can update our database record. + nsCOMPtr stmt = mHistory->GetStatement( + "UPDATE moz_places " + "SET title = :page_title " + "WHERE id = :page_id "); + NS_ENSURE_STATE(stmt); + + { + mozStorageStatementScoper scoper(stmt); + rv = stmt->BindInt64ByName("page_id"_ns, mPlace.placeId); + NS_ENSURE_SUCCESS(rv, rv); + // Empty strings should clear the title, just like + // nsNavHistory::SetPageTitle. + if (mPlace.title.IsEmpty()) { + rv = stmt->BindNullByName("page_title"_ns); + } else { + rv = stmt->BindStringByName("page_title"_ns, + StringHead(mPlace.title, TITLE_LENGTH_MAX)); + } + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsCOMPtr event = + new NotifyTitleObservers(mPlace.spec, mPlace.title, mPlace.guid); + rv = NS_DispatchToMainThread(event); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + private: + SetPageTitle(const nsCString& aSpec, const nsAString& aTitle) + : Runnable("places::SetPageTitle"), mHistory(History::GetService()) { + mPlace.spec = aSpec; + mPlace.title = aTitle; + } + + VisitData mPlace; + + /** + * Strong reference to the History object because we do not want it to + * disappear out from under us. + */ + RefPtr mHistory; +}; + +/** + * Stores an embed visit, and notifies observers. + * + * @param aPlace + * The VisitData of the visit to store as an embed visit. + * @param [optional] aCallback + * The mozIVisitInfoCallback to notify, if provided. + * + * FIXME(emilio, bug 1595484): We should get rid of EMBED visits completely. + */ +void NotifyEmbedVisit(VisitData& aPlace, + mozIVisitInfoCallback* aCallback = nullptr) { + MOZ_ASSERT(aPlace.transitionType == nsINavHistoryService::TRANSITION_EMBED, + "Must only pass TRANSITION_EMBED visits to this!"); + MOZ_ASSERT(NS_IsMainThread(), "Must be called on the main thread!"); + + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), aPlace.spec)); + + if (!uri) { + return; + } + + if (!!aCallback) { + nsMainThreadPtrHandle callback( + new nsMainThreadPtrHolder( + "mozIVisitInfoCallback", aCallback)); + bool ignoreResults = false; + Unused << aCallback->GetIgnoreResults(&ignoreResults); + if (!ignoreResults) { + nsCOMPtr event = + new NotifyPlaceInfoCallback(callback, aPlace, true, NS_OK); + (void)NS_DispatchToMainThread(event); + } + } + + nsCOMPtr event = new NotifyManyVisitsObservers(aPlace); + (void)NS_DispatchToMainThread(event); +} + +//////////////////////////////////////////////////////////////////////////////// +//// History + +History* History::gService = nullptr; + +History::History() + : mShuttingDown(false), + mShuttingDownMutex("History::mShuttingDownMutex"), + mBlockShutdownMutex("History::mBlockShutdownMutex"), + mRecentlyVisitedURIs(RECENTLY_VISITED_URIS_SIZE) { + NS_ASSERTION(!gService, "Ruh-roh! This service has already been created!"); + if (XRE_IsParentProcess()) { + nsCOMPtr dirsvc = components::Directory::Service(); + bool haveProfile = false; + MOZ_RELEASE_ASSERT( + dirsvc && + NS_SUCCEEDED( + dirsvc->Has(NS_APP_USER_PROFILE_50_DIR, &haveProfile)) && + haveProfile, + "Can't construct history service if there is no profile."); + } + gService = this; + + nsCOMPtr os = services::GetObserverService(); + NS_WARNING_ASSERTION(os, "Observer service was not found!"); + if (os) { + (void)os->AddObserver(this, TOPIC_PLACES_SHUTDOWN, false); + } +} + +History::~History() { + UnregisterWeakMemoryReporter(this); + + MOZ_ASSERT(gService == this); + gService = nullptr; +} + +void History::InitMemoryReporter() { RegisterWeakMemoryReporter(this); } + +class ConcurrentStatementsHolder final : public mozIStorageCompletionCallback { + public: + NS_DECL_ISUPPORTS + + ConcurrentStatementsHolder() : mShutdownWasInvoked(false) {} + + static RefPtr Create( + mozIStorageConnection* aDBConn) { + RefPtr holder = + new ConcurrentStatementsHolder(); + nsresult rv = aDBConn->AsyncClone(true, holder); + if (NS_FAILED(rv)) { + return nullptr; + } + return holder; + } + + NS_IMETHOD Complete(nsresult aStatus, nsISupports* aConnection) override { + if (NS_FAILED(aStatus)) { + return NS_OK; + } + mReadOnlyDBConn = do_QueryInterface(aConnection); + // It's possible Shutdown was invoked before we were handed back the + // cloned connection handle. + if (mShutdownWasInvoked) { + Shutdown(); + return NS_OK; + } + + // Now we can create our cached statements. + + if (!mIsVisitedStatement) { + (void)mReadOnlyDBConn->CreateAsyncStatement( + nsLiteralCString("SELECT 1 FROM moz_places h " + "WHERE url_hash = hash(?1) AND url = ?1 AND " + "last_visit_date NOTNULL "), + getter_AddRefs(mIsVisitedStatement)); + MOZ_ASSERT(mIsVisitedStatement); + auto queries = std::move(mVisitedQueries); + if (mIsVisitedStatement) { + for (auto& query : queries) { + query->Execute(*mIsVisitedStatement); + } + } + } + + return NS_OK; + } + + void QueueVisitedStatement(RefPtr aCallback) { + if (mIsVisitedStatement) { + aCallback->Execute(*mIsVisitedStatement); + } else { + mVisitedQueries.AppendElement(std::move(aCallback)); + } + } + + void Shutdown() { + mShutdownWasInvoked = true; + if (mReadOnlyDBConn) { + mVisitedQueries.Clear(); + DebugOnly rv; + if (mIsVisitedStatement) { + rv = mIsVisitedStatement->Finalize(); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + rv = mReadOnlyDBConn->AsyncClose(nullptr); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + mReadOnlyDBConn = nullptr; + } + } + + private: + ~ConcurrentStatementsHolder() = default; + + nsCOMPtr mReadOnlyDBConn; + nsCOMPtr mIsVisitedStatement; + nsTArray> mVisitedQueries; + bool mShutdownWasInvoked; +}; + +NS_IMPL_ISUPPORTS(ConcurrentStatementsHolder, mozIStorageCompletionCallback) + +nsresult History::QueueVisitedStatement(RefPtr aQuery) { + MOZ_ASSERT(NS_IsMainThread()); + if (IsShuttingDown()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (!mConcurrentStatementsHolder) { + mozIStorageConnection* dbConn = GetDBConn(); + NS_ENSURE_STATE(dbConn); + mConcurrentStatementsHolder = ConcurrentStatementsHolder::Create(dbConn); + if (!mConcurrentStatementsHolder) { + return NS_ERROR_NOT_AVAILABLE; + } + } + mConcurrentStatementsHolder->QueueVisitedStatement(std::move(aQuery)); + return NS_OK; +} + +nsresult History::InsertPlace(VisitData& aPlace) { + MOZ_ASSERT(aPlace.placeId == 0, "should not have a valid place id!"); + MOZ_ASSERT(!aPlace.shouldUpdateHidden, "We should not need to update hidden"); + MOZ_ASSERT(!NS_IsMainThread(), "must be called off of the main thread!"); + + nsCOMPtr stmt = GetStatement( + "INSERT INTO moz_places " + "(url, url_hash, title, rev_host, hidden, typed, frecency, guid) " + "VALUES (:url, hash(:url), :title, :rev_host, :hidden, :typed, " + ":frecency, :guid) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindStringByName("rev_host"_ns, aPlace.revHost); + NS_ENSURE_SUCCESS(rv, rv); + rv = URIBinder::Bind(stmt, "url"_ns, aPlace.spec); + NS_ENSURE_SUCCESS(rv, rv); + nsString title = aPlace.title; + // Empty strings should have no title, just like nsNavHistory::SetPageTitle. + if (title.IsEmpty()) { + rv = stmt->BindNullByName("title"_ns); + } else { + title.Assign(StringHead(aPlace.title, TITLE_LENGTH_MAX)); + rv = stmt->BindStringByName("title"_ns, title); + } + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("typed"_ns, aPlace.typed); + NS_ENSURE_SUCCESS(rv, rv); + // When inserting a page for a first visit that should not appear in + // autocomplete, for example an error page, use a zero frecency. + int32_t frecency = aPlace.shouldUpdateFrecency ? aPlace.frecency : 0; + rv = stmt->BindInt32ByName("frecency"_ns, frecency); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("hidden"_ns, aPlace.hidden); + NS_ENSURE_SUCCESS(rv, rv); + if (aPlace.guid.IsVoid()) { + rv = GenerateGUID(aPlace.guid); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = stmt->BindUTF8StringByName("guid"_ns, aPlace.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult History::UpdatePlace(const VisitData& aPlace) { + MOZ_ASSERT(!NS_IsMainThread(), "must be called off of the main thread!"); + MOZ_ASSERT(aPlace.placeId > 0, "must have a valid place id!"); + MOZ_ASSERT(!aPlace.guid.IsVoid(), "must have a guid!"); + + nsCOMPtr stmt; + bool titleIsVoid = aPlace.title.IsVoid(); + if (titleIsVoid) { + // Don't change the title. + stmt = GetStatement( + "UPDATE moz_places " + "SET hidden = :hidden, " + "typed = :typed, " + "guid = :guid " + "WHERE id = :page_id "); + } else { + stmt = GetStatement( + "UPDATE moz_places " + "SET title = :title, " + "hidden = :hidden, " + "typed = :typed, " + "guid = :guid " + "WHERE id = :page_id "); + } + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv; + if (!titleIsVoid) { + // An empty string clears the title. + if (aPlace.title.IsEmpty()) { + rv = stmt->BindNullByName("title"_ns); + } else { + rv = stmt->BindStringByName("title"_ns, + StringHead(aPlace.title, TITLE_LENGTH_MAX)); + } + NS_ENSURE_SUCCESS(rv, rv); + } + rv = stmt->BindInt32ByName("typed"_ns, aPlace.typed); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("hidden"_ns, aPlace.hidden); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("guid"_ns, aPlace.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("page_id"_ns, aPlace.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult History::FetchPageInfo(VisitData& _place, bool* _exists) { + MOZ_ASSERT(!_place.spec.IsEmpty() || !_place.guid.IsEmpty(), + "must have either a non-empty spec or guid!"); + MOZ_ASSERT(!NS_IsMainThread(), "must be called off of the main thread!"); + + nsresult rv; + + // URI takes precedence. + nsCOMPtr stmt; + bool selectByURI = !_place.spec.IsEmpty(); + if (selectByURI) { + stmt = GetStatement( + "SELECT guid, id, title, hidden, typed, frecency, visit_count, " + "last_visit_date, " + "(SELECT id FROM moz_historyvisits " + "WHERE place_id = h.id AND visit_date = h.last_visit_date) AS " + "last_visit_id, " + "EXISTS(SELECT 1 FROM moz_bookmarks WHERE fk = h.id) AS bookmarked " + "FROM moz_places h " + "WHERE url_hash = hash(:page_url) AND url = :page_url "); + NS_ENSURE_STATE(stmt); + + rv = URIBinder::Bind(stmt, "page_url"_ns, _place.spec); + NS_ENSURE_SUCCESS(rv, rv); + } else { + stmt = GetStatement( + "SELECT url, id, title, hidden, typed, frecency, visit_count, " + "last_visit_date, " + "(SELECT id FROM moz_historyvisits " + "WHERE place_id = h.id AND visit_date = h.last_visit_date) AS " + "last_visit_id, " + "EXISTS(SELECT 1 FROM moz_bookmarks WHERE fk = h.id) AS bookmarked " + "FROM moz_places h " + "WHERE guid = :guid "); + NS_ENSURE_STATE(stmt); + + rv = stmt->BindUTF8StringByName("guid"_ns, _place.guid); + NS_ENSURE_SUCCESS(rv, rv); + } + + mozStorageStatementScoper scoper(stmt); + + rv = stmt->ExecuteStep(_exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!*_exists) { + return NS_OK; + } + + if (selectByURI) { + if (_place.guid.IsEmpty()) { + rv = stmt->GetUTF8String(0, _place.guid); + NS_ENSURE_SUCCESS(rv, rv); + } + } else { + nsAutoCString spec; + rv = stmt->GetUTF8String(0, spec); + NS_ENSURE_SUCCESS(rv, rv); + _place.spec = spec; + } + + rv = stmt->GetInt64(1, &_place.placeId); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoString title; + rv = stmt->GetString(2, title); + NS_ENSURE_SUCCESS(rv, rv); + + // If the title we were given was void, that means we did not bother to set + // it to anything. As a result, ignore the fact that we may have changed the + // title (because we don't want to, that would be empty), and set the title + // to what is currently stored in the datbase. + if (_place.title.IsVoid()) { + _place.title = title; + } + // Otherwise, just indicate if the title has changed. + else { + _place.titleChanged = !(_place.title.Equals(title)) && + !(_place.title.IsEmpty() && title.IsVoid()); + } + + int32_t hidden; + rv = stmt->GetInt32(3, &hidden); + NS_ENSURE_SUCCESS(rv, rv); + _place.hidden = !!hidden; + + int32_t typed; + rv = stmt->GetInt32(4, &typed); + NS_ENSURE_SUCCESS(rv, rv); + _place.typed = !!typed; + + rv = stmt->GetInt32(5, &_place.frecency); + NS_ENSURE_SUCCESS(rv, rv); + int32_t visitCount; + rv = stmt->GetInt32(6, &visitCount); + NS_ENSURE_SUCCESS(rv, rv); + _place.visitCount = visitCount; + rv = stmt->GetInt64(7, &_place.lastVisitTime); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(8, &_place.lastVisitId); + NS_ENSURE_SUCCESS(rv, rv); + int32_t bookmarked; + rv = stmt->GetInt32(9, &bookmarked); + NS_ENSURE_SUCCESS(rv, rv); + _place.bookmarked = bookmarked == 1; + + return NS_OK; +} + +MOZ_DEFINE_MALLOC_SIZE_OF(HistoryMallocSizeOf) + +NS_IMETHODIMP +History::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) { + MOZ_COLLECT_REPORT( + "explicit/history-links-hashtable", KIND_HEAP, UNITS_BYTES, + SizeOfIncludingThis(HistoryMallocSizeOf), + "Memory used by the hashtable that records changes to the visited state " + "of links."); + + return NS_OK; +} + +size_t History::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) { + size_t size = aMallocSizeOf(this); + size += mTrackedURIs.ShallowSizeOfExcludingThis(aMallocSizeOf); + for (const auto& entry : mTrackedURIs.Values()) { + size += entry.SizeOfExcludingThis(aMallocSizeOf); + } + return size; +} + +/* static */ +History* History::GetService() { + if (gService) { + return gService; + } + + nsCOMPtr service = components::History::Service(); + if (service) { + NS_ASSERTION(gService, "Our constructor was not run?!"); + } + + return gService; +} + +/* static */ +already_AddRefed History::GetSingleton() { + if (!gService) { + RefPtr svc = new History(); + MOZ_ASSERT(gService == svc.get()); + svc->InitMemoryReporter(); + return svc.forget(); + } + + return do_AddRef(gService); +} + +mozIStorageConnection* History::GetDBConn() { + MOZ_ASSERT(NS_IsMainThread()); + if (IsShuttingDown()) { + return nullptr; + } + if (!mDB) { + mDB = Database::GetDatabase(); + NS_ENSURE_TRUE(mDB, nullptr); + // This must happen on the main-thread, so when we try to use the connection + // later it's initialized. + mDB->EnsureConnection(); + NS_ENSURE_TRUE(mDB, nullptr); + } + return mDB->MainConn(); +} + +const mozIStorageConnection* History::GetConstDBConn() { + MOZ_ASSERT(!NS_IsMainThread()); + { + MOZ_ASSERT(mDB || IsShuttingDown()); + if (IsShuttingDown() || !mDB) { + return nullptr; + } + } + return mDB->MainConn(); +} + +void History::Shutdown() { + MOZ_ASSERT(NS_IsMainThread()); + MutexAutoLock lockedScope(mBlockShutdownMutex); + { + MutexAutoLock lockedScope(mShuttingDownMutex); + MOZ_ASSERT(!mShuttingDown && "Shutdown was called more than once!"); + mShuttingDown = true; + } + if (mConcurrentStatementsHolder) { + mConcurrentStatementsHolder->Shutdown(); + } +} + +void History::AppendToRecentlyVisitedURIs(nsIURI* aURI, bool aHidden) { + PRTime now = PR_Now(); + + mRecentlyVisitedURIs.InsertOrUpdate(aURI, RecentURIVisit{now, aHidden}); + + // Remove entries older than RECENTLY_VISITED_URIS_MAX_AGE. + for (auto iter = mRecentlyVisitedURIs.Iter(); !iter.Done(); iter.Next()) { + if ((now - iter.Data().mTime) > RECENTLY_VISITED_URIS_MAX_AGE) { + iter.Remove(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +//// IHistory + +NS_IMETHODIMP +History::VisitURI(nsIWidget* aWidget, nsIURI* aURI, nsIURI* aLastVisitedURI, + uint32_t aFlags, uint64_t aBrowserId) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aURI); + + if (IsShuttingDown()) { + return NS_OK; + } + + if (XRE_IsContentProcess()) { + if (!BaseHistory::CanStore(aURI)) { + return NS_OK; + } + + NS_ENSURE_ARG(aWidget); + BrowserChild* browserChild = aWidget->GetOwningBrowserChild(); + NS_ENSURE_TRUE(browserChild, NS_ERROR_FAILURE); + (void)browserChild->SendVisitURI(aURI, aLastVisitedURI, aFlags, aBrowserId); + return NS_OK; + } + + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(navHistory, NS_ERROR_OUT_OF_MEMORY); + + // Silently return if URI is something we shouldn't add to DB. + bool canAdd; + nsresult rv = navHistory->CanAddURI(aURI, &canAdd); + NS_ENSURE_SUCCESS(rv, rv); + if (!canAdd) { + return NS_OK; + } + + bool reload = false; + if (aLastVisitedURI) { + rv = aURI->Equals(aLastVisitedURI, &reload); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsTArray placeArray(1); + placeArray.AppendElement(VisitData(aURI, aLastVisitedURI)); + VisitData& place = placeArray.ElementAt(0); + NS_ENSURE_FALSE(place.spec.IsEmpty(), NS_ERROR_INVALID_ARG); + + place.visitTime = PR_Now(); + + // Assigns a type to the edge in the visit linked list. Each type will be + // considered differently when weighting the frecency of a location. + uint32_t recentFlags = navHistory->GetRecentFlags(aURI); + bool isFollowedLink = recentFlags & nsNavHistory::RECENT_ACTIVATED; + + // Embed visits should never be added to the database, and the same is valid + // for redirects across frames. + // For the above reasoning non-toplevel transitions are handled at first. + // if the visit is toplevel or a non-toplevel followed link, then it can be + // handled as usual and stored on disk. + + uint32_t transitionType = nsINavHistoryService::TRANSITION_LINK; + + if (!(aFlags & IHistory::TOP_LEVEL) && !isFollowedLink) { + // A frame redirected to a new site without user interaction. + transitionType = nsINavHistoryService::TRANSITION_EMBED; + } else if (aFlags & IHistory::REDIRECT_TEMPORARY) { + transitionType = nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY; + } else if (aFlags & IHistory::REDIRECT_PERMANENT) { + transitionType = nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT; + } else if (reload) { + transitionType = nsINavHistoryService::TRANSITION_RELOAD; + } else if ((recentFlags & nsNavHistory::RECENT_TYPED) && + !(aFlags & IHistory::UNRECOVERABLE_ERROR)) { + // Don't mark error pages as typed, even if they were actually typed by + // the user. This is useful to limit their score in autocomplete. + transitionType = nsINavHistoryService::TRANSITION_TYPED; + } else if (recentFlags & nsNavHistory::RECENT_BOOKMARKED) { + transitionType = nsINavHistoryService::TRANSITION_BOOKMARK; + } else if (!(aFlags & IHistory::TOP_LEVEL) && isFollowedLink) { + // User activated a link in a frame. + transitionType = nsINavHistoryService::TRANSITION_FRAMED_LINK; + } + + place.SetTransitionType(transitionType); + bool isRedirect = aFlags & IHistory::REDIRECT_SOURCE; + if (isRedirect) { + place.useFrecencyRedirectBonus = + (aFlags & IHistory::REDIRECT_SOURCE_PERMANENT) || + transitionType != nsINavHistoryService::TRANSITION_TYPED; + } + place.hidden = GetHiddenState(isRedirect, place.transitionType); + + // Error pages should never be autocompleted. + if (aFlags & IHistory::UNRECOVERABLE_ERROR) { + place.shouldUpdateFrecency = false; + } + + // Do not save a reloaded uri if we have visited the same URI recently. + if (reload) { + auto entry = mRecentlyVisitedURIs.Lookup(aURI); + // Check if the entry exists and is younger than + // RECENTLY_VISITED_URIS_MAX_AGE. + if (entry && (PR_Now() - entry->mTime) < RECENTLY_VISITED_URIS_MAX_AGE) { + bool wasHidden = entry->mHidden; + // Regardless of whether we store the visit or not, we must update the + // stored visit time. + AppendToRecentlyVisitedURIs(aURI, place.hidden); + // We always want to store an unhidden visit, if the previous visits were + // hidden, because otherwise the page may not appear in the history UI. + // This can happen for example at a page redirecting to itself. + if (!wasHidden || place.hidden) { + // We can skip this visit. + return NS_OK; + } + } + } + + nsCOMPtr bwt = + do_ImportModule("resource:///modules/BrowserWindowTracker.jsm", + "BrowserWindowTracker", &rv); + if (NS_SUCCEEDED(rv)) { + // Only if it is running on Firefox, continue to process the followings. + nsCOMPtr browser; + rv = bwt->GetBrowserById(aBrowserId, getter_AddRefs(browser)); + NS_ENSURE_SUCCESS(rv, rv); + if (browser) { + RefPtr browserElement = static_cast(browser.get()); + + nsAutoString triggeringSearchEngineURL; + browserElement->GetAttribute(u"triggeringSearchEngineURL"_ns, + triggeringSearchEngineURL); + if (!triggeringSearchEngineURL.IsEmpty() && + place.spec.Equals(NS_ConvertUTF16toUTF8(triggeringSearchEngineURL))) { + nsAutoString triggeringSearchEngine; + browserElement->GetAttribute(u"triggeringSearchEngine"_ns, + triggeringSearchEngine); + place.triggeringSearchEngine.Assign( + NS_ConvertUTF16toUTF8(triggeringSearchEngine)); + } + + nsAutoString triggeringSponsoredURL; + browserElement->GetAttribute(u"triggeringSponsoredURL"_ns, + triggeringSponsoredURL); + if (!triggeringSponsoredURL.IsEmpty()) { + place.triggeringSponsoredURL.Assign( + NS_ConvertUTF16toUTF8(triggeringSponsoredURL)); + + nsAutoString triggeringSponsoredURLVisitTimeMS; + browserElement->GetAttribute(u"triggeringSponsoredURLVisitTimeMS"_ns, + triggeringSponsoredURLVisitTimeMS); + place.triggeringSponsoredURLVisitTimeMS = + triggeringSponsoredURLVisitTimeMS.ToInteger64(&rv); + NS_ENSURE_SUCCESS(rv, rv); + + // Get base domain. We need to get it here since nsIEffectiveTLDService + // referred in DomainNameFromURI should access on main thread. + nsCOMPtr currentURL; + rv = NS_MutateURI(new net::nsStandardURL::Mutator()) + .SetSpec(place.spec) + .Finalize(currentURL); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr sponsoredURL; + rv = NS_MutateURI(new net::nsStandardURL::Mutator()) + .SetSpec(place.triggeringSponsoredURL) + .Finalize(sponsoredURL); + NS_ENSURE_SUCCESS(rv, rv); + navHistory->DomainNameFromURI(currentURL, place.baseDomain); + navHistory->DomainNameFromURI(sponsoredURL, + place.triggeringSponsoredURLBaseDomain); + } + } + } + + // EMBED visits should not go through the database. + // They exist only to keep track of isVisited status during the session. + if (place.transitionType == nsINavHistoryService::TRANSITION_EMBED) { + NotifyEmbedVisit(place); + } else { + mozIStorageConnection* dbConn = GetDBConn(); + NS_ENSURE_STATE(dbConn); + + rv = InsertVisitedURIs::Start(dbConn, std::move(placeArray)); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +NS_IMETHODIMP +History::SetURITitle(nsIURI* aURI, const nsAString& aTitle) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aURI); + + if (IsShuttingDown()) { + return NS_OK; + } + + if (XRE_IsContentProcess()) { + auto* cpc = dom::ContentChild::GetSingleton(); + MOZ_ASSERT(cpc, "Content Protocol is NULL!"); + Unused << cpc->SendSetURITitle(aURI, PromiseFlatString(aTitle)); + return NS_OK; + } + + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + + // At first, it seems like nav history should always be available here, no + // matter what. + // + // nsNavHistory fails to register as a service if there is no profile in + // place (for instance, if user is choosing a profile). + // + // Maybe the correct thing to do is to not register this service if no + // profile has been selected? + // + NS_ENSURE_TRUE(navHistory, NS_ERROR_FAILURE); + + bool canAdd; + nsresult rv = navHistory->CanAddURI(aURI, &canAdd); + NS_ENSURE_SUCCESS(rv, rv); + if (!canAdd) { + return NS_OK; + } + + mozIStorageConnection* dbConn = GetDBConn(); + NS_ENSURE_STATE(dbConn); + + return SetPageTitle::Start(dbConn, aURI, aTitle); +} + +//////////////////////////////////////////////////////////////////////////////// +//// mozIAsyncHistory + +NS_IMETHODIMP +History::UpdatePlaces(JS::Handle aPlaceInfos, + mozIVisitInfoCallback* aCallback, JSContext* aCtx) { + NS_ENSURE_TRUE(NS_IsMainThread(), NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(!aPlaceInfos.isPrimitive(), NS_ERROR_INVALID_ARG); + + uint32_t infosLength; + JS::Rooted infos(aCtx); + nsresult rv = GetJSArrayFromJSValue(aPlaceInfos, aCtx, &infos, &infosLength); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t initialUpdatedCount = 0; + + nsTArray visitData; + for (uint32_t i = 0; i < infosLength; i++) { + JS::Rooted info(aCtx); + nsresult rv = GetJSObjectFromArray(aCtx, infos, i, &info); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr uri = GetURIFromJSObject(aCtx, info, "uri"); + nsCString guid; + { + nsString fatGUID; + GetStringFromJSObject(aCtx, info, "guid", fatGUID); + if (fatGUID.IsVoid()) { + guid.SetIsVoid(true); + } else { + CopyUTF16toUTF8(fatGUID, guid); + } + } + + // Make sure that any uri we are given can be added to history, and if not, + // skip it (CanAddURI will notify our callback for us). + if (uri && !CanAddURI(uri, guid, aCallback)) { + continue; + } + + // We must have at least one of uri or guid. + NS_ENSURE_ARG(uri || !guid.IsVoid()); + + // If we were given a guid, make sure it is valid. + bool isValidGUID = IsValidGUID(guid); + NS_ENSURE_ARG(guid.IsVoid() || isValidGUID); + + nsString title; + GetStringFromJSObject(aCtx, info, "title", title); + + JS::Rooted visits(aCtx, nullptr); + { + JS::Rooted visitsVal(aCtx); + bool rc = JS_GetProperty(aCtx, info, "visits", &visitsVal); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + if (!visitsVal.isPrimitive()) { + visits = visitsVal.toObjectOrNull(); + bool isArray; + if (!JS::IsArrayObject(aCtx, visits, &isArray)) { + return NS_ERROR_UNEXPECTED; + } + if (!isArray) { + return NS_ERROR_INVALID_ARG; + } + } + } + NS_ENSURE_ARG(visits); + + uint32_t visitsLength = 0; + if (visits) { + (void)JS::GetArrayLength(aCtx, visits, &visitsLength); + } + NS_ENSURE_ARG(visitsLength > 0); + + // Check each visit, and build our array of VisitData objects. + visitData.SetCapacity(visitData.Length() + visitsLength); + for (uint32_t j = 0; j < visitsLength; j++) { + JS::Rooted visit(aCtx); + rv = GetJSObjectFromArray(aCtx, visits, j, &visit); + NS_ENSURE_SUCCESS(rv, rv); + + VisitData& data = *visitData.AppendElement(VisitData(uri)); + if (!title.IsEmpty()) { + data.title = title; + } else if (!title.IsVoid()) { + // Setting data.title to an empty string wouldn't make it non-void. + data.title.SetIsVoid(false); + } + data.guid = guid; + + // We must have a date and a transaction type! + rv = GetIntFromJSObject(aCtx, visit, "visitDate", &data.visitTime); + NS_ENSURE_SUCCESS(rv, rv); + // visitDate should be in microseconds. It's easy to do the wrong thing + // and pass milliseconds to updatePlaces, so we lazily check for that. + // While it's not easily distinguishable, since both are integers, we can + // check if the value is very far in the past, and assume it's probably + // a mistake. + if (data.visitTime < (PR_Now() / 1000)) { +#ifdef DEBUG + nsCOMPtr xpc = nsIXPConnect::XPConnect(); + Unused << xpc->DebugDumpJSStack(false, false, false); + MOZ_CRASH("invalid time format passed to updatePlaces"); +#endif + return NS_ERROR_INVALID_ARG; + } + uint32_t transitionType = 0; + rv = GetIntFromJSObject(aCtx, visit, "transitionType", &transitionType); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_ARG_RANGE(transitionType, nsINavHistoryService::TRANSITION_LINK, + nsINavHistoryService::TRANSITION_RELOAD); + data.SetTransitionType(transitionType); + data.hidden = GetHiddenState(false, transitionType); + + // If the visit is an embed visit, we do not actually add it to the + // database. + if (transitionType == nsINavHistoryService::TRANSITION_EMBED) { + NotifyEmbedVisit(data, aCallback); + visitData.RemoveLastElement(); + initialUpdatedCount++; + continue; + } + + // The referrer is optional. + nsCOMPtr referrer = + GetURIFromJSObject(aCtx, visit, "referrerURI"); + if (referrer) { + (void)referrer->GetSpec(data.referrerSpec); + } + } + } + + mozIStorageConnection* dbConn = GetDBConn(); + NS_ENSURE_STATE(dbConn); + + nsMainThreadPtrHandle callback( + new nsMainThreadPtrHolder("mozIVisitInfoCallback", + aCallback)); + + // It is possible that all of the visits we were passed were dissallowed by + // CanAddURI, which isn't an error. If we have no visits to add, however, + // we should not call InsertVisitedURIs::Start. + if (visitData.Length()) { + nsresult rv = InsertVisitedURIs::Start(dbConn, std::move(visitData), + callback, initialUpdatedCount); + NS_ENSURE_SUCCESS(rv, rv); + } else if (aCallback) { + // Be sure to notify that all of our operations are complete. This + // is dispatched to the background thread first and redirected to the + // main thread from there to make sure that all database notifications + // and all embed or canAddURI notifications have finished. + + // Note: if we're inserting anything, it's the responsibility of + // InsertVisitedURIs to call the completion callback, as here we won't + // know how yet many items we will successfully insert/update. + nsCOMPtr backgroundThread = do_GetInterface(dbConn); + NS_ENSURE_TRUE(backgroundThread, NS_ERROR_UNEXPECTED); + nsCOMPtr event = + new NotifyCompletion(callback, initialUpdatedCount); + return backgroundThread->Dispatch(event, NS_DISPATCH_NORMAL); + } + + return NS_OK; +} + +NS_IMETHODIMP +History::IsURIVisited(nsIURI* aURI, mozIVisitedStatusCallback* aCallback) { + NS_ENSURE_STATE(NS_IsMainThread()); + NS_ENSURE_ARG(aURI); + NS_ENSURE_ARG(aCallback); + + return VisitedQuery::Start(aURI, aCallback); +} + +NS_IMETHODIMP +History::ClearCache() { + mRecentlyVisitedURIs.Clear(); + return NS_OK; +} + +void History::StartPendingVisitedQueries(PendingVisitedQueries&& aQueries) { + if (XRE_IsContentProcess()) { + auto* cpc = dom::ContentChild::GetSingleton(); + MOZ_ASSERT(cpc, "Content Protocol is NULL!"); + + // Fairly arbitrary limit on the number of URLs we send at a time, to avoid + // going over the IPC message size limit... Note that this is imperfect (we + // could have very long URIs), so this is a best-effort kind of thing. See + // bug 1775265. + constexpr size_t kBatchLimit = 4000; + + nsTArray> uris(aQueries.Count()); + for (const auto& entry : aQueries) { + uris.AppendElement(entry.GetKey()); + MOZ_ASSERT(entry.GetData().IsEmpty(), + "Child process shouldn't have parent requests"); + if (uris.Length() == kBatchLimit) { + Unused << cpc->SendStartVisitedQueries(uris); + uris.ClearAndRetainStorage(); + } + } + + if (!uris.IsEmpty()) { + Unused << cpc->SendStartVisitedQueries(uris); + } + } else { + // TODO(bug 1594368): We could do a single query, as long as we can + // then notify each URI individually. + for (auto& entry : aQueries) { + nsresult queryStatus = VisitedQuery::Start( + entry.GetKey(), std::move(*entry.GetModifiableData())); + Unused << NS_WARN_IF(NS_FAILED(queryStatus)); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsIObserver + +NS_IMETHODIMP +History::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + if (strcmp(aTopic, TOPIC_PLACES_SHUTDOWN) == 0) { + Shutdown(); + + nsCOMPtr os = mozilla::services::GetObserverService(); + if (os) { + (void)os->RemoveObserver(this, TOPIC_PLACES_SHUTDOWN); + } + } + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsISupports + +NS_IMPL_ISUPPORTS(History, IHistory, mozIAsyncHistory, nsIObserver, + nsIMemoryReporter) + +} // namespace mozilla::places diff --git a/toolkit/components/places/History.h b/toolkit/components/places/History.h new file mode 100644 index 0000000000..03a31186fd --- /dev/null +++ b/toolkit/components/places/History.h @@ -0,0 +1,203 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_History_h_ +#define mozilla_places_History_h_ + +#include + +#include "Database.h" +#include "mozIAsyncHistory.h" +#include "mozIStorageConnection.h" +#include "mozilla/BaseHistory.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Mutex.h" +#include "mozilla/dom/Link.h" +#include "mozilla/dom/PContentChild.h" +#include "nsTHashMap.h" +#include "nsIMemoryReporter.h" +#include "nsIObserver.h" +#include "nsString.h" +#include "nsTHashtable.h" +#include "nsTObserverArray.h" +#include "nsURIHashKey.h" + +namespace mozilla::places { + +struct VisitData; +class ConcurrentStatementsHolder; +class VisitedQuery; + +// Initial size of mRecentlyVisitedURIs. +#define RECENTLY_VISITED_URIS_SIZE 64 +// Microseconds after which a visit can be expired from mRecentlyVisitedURIs. +// When an URI is reloaded we only take into account the first visit to it, and +// ignore any subsequent visits, if they happen before this time has elapsed. +// A commonly found case is to reload a page every 5 minutes, so we pick a time +// larger than that. +#define RECENTLY_VISITED_URIS_MAX_AGE (6 * 60 * PR_USEC_PER_SEC) +// When notifying the main thread after inserting visits, we chunk the visits +// into medium-sized groups so that we can amortize the cost of the runnable +// without janking the main thread by expecting it to process hundreds at once. +#define NOTIFY_VISITS_CHUNK_SIZE 100 + +class History final : public BaseHistory, + public mozIAsyncHistory, + public nsIObserver, + public nsIMemoryReporter { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZIASYNCHISTORY + NS_DECL_NSIOBSERVER + NS_DECL_NSIMEMORYREPORTER + + // IHistory + NS_IMETHOD VisitURI(nsIWidget*, nsIURI*, nsIURI* aLastVisitedURI, + uint32_t aFlags, uint64_t aBrowserId) final; + NS_IMETHOD SetURITitle(nsIURI*, const nsAString&) final; + + // BaseHistory + void StartPendingVisitedQueries(PendingVisitedQueries&&) final; + + History(); + + nsresult QueueVisitedStatement(RefPtr); + + /** + * Adds an entry in moz_places with the data in aVisitData. + * + * @param aPlace + * The visit data to use to populate a new row in moz_places. + */ + nsresult InsertPlace(VisitData& aPlace); + + /** + * Updates an entry in moz_places with the data in aVisitData. + * + * @param aPlace + * The visit data to use to update the existing row in moz_places. + */ + nsresult UpdatePlace(const VisitData& aPlace); + + /** + * Loads information about the page into _place from moz_places. + * + * @param _place + * The VisitData for the place we need to know information about. + * @param [out] _exists + * Whether or the page was recorded in moz_places, false otherwise. + */ + nsresult FetchPageInfo(VisitData& _place, bool* _exists); + + /** + * Get the number of bytes of memory this History object is using, + * including sizeof(*this)) + */ + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf); + + /** + * Obtains a pointer to this service. + */ + static History* GetService(); + + /** + * Used by the service manager only. + */ + static already_AddRefed GetSingleton(); + + template + already_AddRefed GetStatement(const char (&aQuery)[N]) { + // May be invoked on both threads. + const mozIStorageConnection* dbConn = GetConstDBConn(); + NS_ENSURE_TRUE(dbConn, nullptr); + return mDB->GetStatement(aQuery); + } + + already_AddRefed GetStatement( + const nsACString& aQuery) { + // May be invoked on both threads. + const mozIStorageConnection* dbConn = GetConstDBConn(); + NS_ENSURE_TRUE(dbConn, nullptr); + return mDB->GetStatement(aQuery); + } + + bool IsShuttingDown() { + MutexAutoLock lockedScope(mShuttingDownMutex); + return mShuttingDown; + } + + /** + * Helper function to append a new URI to mRecentlyVisitedURIs. See + * mRecentlyVisitedURIs. + * @param {nsIURI} aURI The URI to append + * @param {bool} aHidden The hidden status of the visit being appended. + */ + void AppendToRecentlyVisitedURIs(nsIURI* aURI, bool aHidden); + + private: + virtual ~History(); + + void InitMemoryReporter(); + + /** + * Obtains a read-write database connection, initializing the connection + * if needed. Must be invoked on the main thread. + */ + mozIStorageConnection* GetDBConn(); + + /** + * Obtains a read-write database connection, but won't try to initialize it. + * May be invoked on both threads, but first one must invoke GetDBConn() on + * the main-thread at least once. + */ + const mozIStorageConnection* GetConstDBConn(); + + /** + * The database handle. This is initialized lazily by the first call to + * GetDBConn(), so never use it directly, or, if you really need, always + * invoke GetDBConn() before. + */ + RefPtr mDB; + + RefPtr mConcurrentStatementsHolder; + + /** + * Remove any memory references to tasks and do not take on any more. + */ + void Shutdown(); + + static History* gService; + + // Ensures new tasks aren't started on destruction. Should only be changed on + // the main thread. + bool mShuttingDown; + // This mutex guards mShuttingDown and should be acquired on the helper + // thread. + Mutex mShuttingDownMutex MOZ_UNANNOTATED; + // Code running in the helper thread can acquire this mutex to block shutdown + // from proceeding until done, otherwise it may be impossible to get + // statements to execute and an insert operation could be interrupted in the + // middle. + Mutex mBlockShutdownMutex MOZ_UNANNOTATED; + + // Allow private access from the helper thread to acquire mutexes. + friend class InsertVisitedURIs; + + /** + * mRecentlyVisitedURIs remembers URIs which have been recently added to + * history, to avoid saving these locations repeatedly in a short period. + */ + struct RecentURIVisit { + PRTime mTime; + bool mHidden; + }; + + nsTHashMap mRecentlyVisitedURIs; +}; + +} // namespace mozilla::places + +#endif // mozilla_places_History_h_ diff --git a/toolkit/components/places/History.sys.mjs b/toolkit/components/places/History.sys.mjs new file mode 100644 index 0000000000..dde455465e --- /dev/null +++ b/toolkit/components/places/History.sys.mjs @@ -0,0 +1,1738 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Asynchronous API for managing history. + * + * + * The API makes use of `PageInfo` and `VisitInfo` objects, defined as follows. + * + * A `PageInfo` object is any object that contains A SUBSET of the + * following properties: + * - guid: (string) + * The globally unique id of the page. + * - url: (URL) + * or (nsIURI) + * or (string) + * The full URI of the page. Note that `PageInfo` values passed as + * argument may hold `nsIURI` or `string` values for property `url`, + * but `PageInfo` objects returned by this module always hold `URL` + * values. + * - title: (string) + * The title associated with the page, if any. + * - description: (string) + * The description of the page, if any. + * - previewImageURL: (URL) + * or (nsIURI) + * or (string) + * The preview image URL of the page, if any. + * - frecency: (number) + * The frecency of the page, if any. + * See https://developer.mozilla.org/en-US/docs/Mozilla/Tech/Places/Frecency_algorithm + * Note that this property may not be used to change the actualy frecency + * score of a page, only to retrieve it. In other words, any `frecency` field + * passed as argument to a function of this API will be ignored. + * - visits: (Array) + * All the visits for this page, if any. + * - annotations: (Map) + * A map containing key/value pairs of the annotations for this page, if any. + * + * See the documentation of individual methods to find out which properties + * are required for `PageInfo` arguments or returned for `PageInfo` results. + * + * A `VisitInfo` object is any object that contains A SUBSET of the following + * properties: + * - date: (Date) + * The time the visit occurred. + * - transition: (number) + * How the user reached the page. See constants `TRANSITIONS.*` + * for the possible transition types. + * - referrer: (URL) + * or (nsIURI) + * or (string) + * The referring URI of this visit. Note that `VisitInfo` passed + * as argument may hold `nsIURI` or `string` values for property `referrer`, + * but `VisitInfo` objects returned by this module always hold `URL` + * values. + * See the documentation of individual methods to find out which properties + * are required for `VisitInfo` arguments or returned for `VisitInfo` results. + * + * + * + * Each successful operation notifies through the PlacesObservers. To listen to such + * notifications you must register using + * PlacesObservers `addListener` and `removeListener` methods. + * @see PlacesObservers + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "asyncHistory", + "@mozilla.org/browser/history;1", + "mozIAsyncHistory" +); + +/** + * Whenever we update numerous pages, it is preferable to yield time to the main + * thread every so often to avoid janking. + * These constants determine the maximal number of notifications we + * may emit before we yield. + */ +const ONRESULT_CHUNK_SIZE = 300; + +// This constant determines the maximum number of remove pages before we cycle. +const REMOVE_PAGES_CHUNKLEN = 300; + +export var History = Object.freeze({ + ANNOTATION_EXPIRE_NEVER: 4, + // Constants for the type of annotation. + ANNOTATION_TYPE_STRING: 3, + ANNOTATION_TYPE_INT64: 5, + + /** + * Fetch the available information for one page. + * + * @param {URL|nsIURI|string} guidOrURI: (string) or (URL, nsIURI or href) + * Either the full URI of the page or the GUID of the page. + * @param {object} [options] + * An optional object whose properties describe options: + * - `includeVisits` (boolean) set this to true if `visits` in the + * PageInfo needs to contain VisitInfo in a reverse chronological order. + * By default, `visits` is undefined inside the returned `PageInfo`. + * - `includeMeta` (boolean) set this to true to fetch page meta fields, + * i.e. `description`, `site_name` and `preview_image_url`. + * - `includeAnnotations` (boolean) set this to true to fetch any + * annotations that are associated with the page. + * + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolves (PageInfo | null) If the page could be found, the information + * on that page. + * @note the VisitInfo objects returned while fetching visits do not + * contain the property `referrer`. + * TODO: Add `referrer` to VisitInfo. See Bug #1365913. + * @note the visits returned will not contain `TRANSITION_EMBED` visits. + * + * @throws (Error) + * If `guidOrURI` does not have the expected type or if it is a string + * that may be parsed neither as a valid URL nor as a valid GUID. + */ + fetch(guidOrURI, options = {}) { + // First, normalize to guid or string, and throw if not possible + guidOrURI = lazy.PlacesUtils.normalizeToURLOrGUID(guidOrURI); + + // See if options exists and make sense + if (!options || typeof options !== "object") { + throw new TypeError("options should be an object and not null"); + } + + let hasIncludeVisits = "includeVisits" in options; + if (hasIncludeVisits && typeof options.includeVisits !== "boolean") { + throw new TypeError("includeVisits should be a boolean if exists"); + } + + let hasIncludeMeta = "includeMeta" in options; + if (hasIncludeMeta && typeof options.includeMeta !== "boolean") { + throw new TypeError("includeMeta should be a boolean if exists"); + } + + let hasIncludeAnnotations = "includeAnnotations" in options; + if ( + hasIncludeAnnotations && + typeof options.includeAnnotations !== "boolean" + ) { + throw new TypeError("includeAnnotations should be a boolean if exists"); + } + + return lazy.PlacesUtils.promiseDBConnection().then(db => + fetch(db, guidOrURI, options) + ); + }, + + /** + * Fetches all pages which have one or more of the specified annotations. + * + * @param annotations: An array of strings containing the annotation names to + * find. + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolves (Map) + * A Map containing the annotations, pages and their contents, e.g. + * Map("anno1" => [{page, content}, {page, content}]), "anno2" => ....); + * @rejects (Error) XXX + * Rejects if the insert was unsuccessful. + */ + fetchAnnotatedPages(annotations) { + // See if options exists and make sense + if (!annotations || !Array.isArray(annotations)) { + throw new TypeError("annotations should be an Array and not null"); + } + if (annotations.some(name => typeof name !== "string")) { + throw new TypeError("all annotation values should be strings"); + } + + return lazy.PlacesUtils.promiseDBConnection().then(db => + fetchAnnotatedPages(db, annotations) + ); + }, + + /** + * Fetch multiple pages. + * + * @param {string[]|nsIURI[]|URL[]} guidOrURIs: Array of href or URLs to fetch. + * @returns {Promise} + * A promise resolved once the operation is complete. + * @resolves {Map} Map of PageInfos, keyed by the input info, + * either guid or href. We don't use nsIURI or URL as keys to avoid + * complexity, in all the cases the caller knows which objects is handling, + * and can unwrap them. Unknown input pages will have no entry in the Map. + * @throws (Error) + * If input is invalid, for example not a valid GUID or URL. + */ + fetchMany(guidOrURIs) { + if (!Array.isArray(guidOrURIs)) { + throw new TypeError("Input is not an array"); + } + // First, normalize to guid or URL, throw if not possible + guidOrURIs = guidOrURIs.map(v => lazy.PlacesUtils.normalizeToURLOrGUID(v)); + return lazy.PlacesUtils.promiseDBConnection().then(db => + fetchMany(db, guidOrURIs) + ); + }, + + /** + * Adds a number of visits for a single page. + * + * Any change may be observed through PlacesObservers. + * + * @param pageInfo: (PageInfo) + * Information on a page. This `PageInfo` MUST contain + * - a property `url`, as specified by the definition of `PageInfo`. + * - a property `visits`, as specified by the definition of + * `PageInfo`, which MUST contain at least one visit. + * If a property `title` is provided, the title of the page + * is updated. + * If the `date` of a visit is not provided, it defaults + * to now. + * If the `transition` of a visit is not provided, it defaults to + * TRANSITION_LINK. + * + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolves (PageInfo) + * A PageInfo object populated with data after the insert is complete. + * @rejects (Error) + * Rejects if the insert was unsuccessful. + * + * @throws (Error) + * If the `url` specified was for a protocol that should not be + * stored (@see nsNavHistory::CanAddURI). + * @throws (Error) + * If `pageInfo` has an unexpected type. + * @throws (Error) + * If `pageInfo` does not have a `url`. + * @throws (Error) + * If `pageInfo` does not have a `visits` property or if the + * value of `visits` is ill-typed or is an empty array. + * @throws (Error) + * If an element of `visits` has an invalid `date`. + * @throws (Error) + * If an element of `visits` has an invalid `transition`. + */ + insert(pageInfo) { + let info = lazy.PlacesUtils.validatePageInfo(pageInfo); + + return lazy.PlacesUtils.withConnectionWrapper("History.jsm: insert", db => + insert(db, info) + ); + }, + + /** + * Adds a number of visits for a number of pages. + * + * Any change may be observed through PlacesObservers. + * + * @param pageInfos: (Array) + * Information on a page. This `PageInfo` MUST contain + * - a property `url`, as specified by the definition of `PageInfo`. + * - a property `visits`, as specified by the definition of + * `PageInfo`, which MUST contain at least one visit. + * If a property `title` is provided, the title of the page + * is updated. + * If the `date` of a visit is not provided, it defaults + * to now. + * If the `transition` of a visit is not provided, it defaults to + * TRANSITION_LINK. + * @param onResult: (function(PageInfo)) + * A callback invoked for each page inserted. + * @param onError: (function(PageInfo)) + * A callback invoked for each page which generated an error + * when an insert was attempted. + * + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolves (null) + * @rejects (Error) + * Rejects if all of the inserts were unsuccessful. + * + * @throws (Error) + * If the `url` specified was for a protocol that should not be + * stored (@see nsNavHistory::CanAddURI). + * @throws (Error) + * If `pageInfos` has an unexpected type. + * @throws (Error) + * If a `pageInfo` does not have a `url`. + * @throws (Error) + * If a `PageInfo` does not have a `visits` property or if the + * value of `visits` is ill-typed or is an empty array. + * @throws (Error) + * If an element of `visits` has an invalid `date`. + * @throws (Error) + * If an element of `visits` has an invalid `transition`. + */ + insertMany(pageInfos, onResult, onError) { + let infos = []; + + if (!Array.isArray(pageInfos)) { + throw new TypeError("pageInfos must be an array"); + } + if (!pageInfos.length) { + throw new TypeError("pageInfos may not be an empty array"); + } + + if (onResult && typeof onResult != "function") { + throw new TypeError(`onResult: ${onResult} is not a valid function`); + } + if (onError && typeof onError != "function") { + throw new TypeError(`onError: ${onError} is not a valid function`); + } + + for (let pageInfo of pageInfos) { + let info = lazy.PlacesUtils.validatePageInfo(pageInfo); + infos.push(info); + } + + return lazy.PlacesUtils.withConnectionWrapper( + "History.jsm: insertMany", + db => insertMany(db, infos, onResult, onError) + ); + }, + + /** + * Remove pages from the database. + * + * Any change may be observed through PlacesObservers. + * + * + * @param page: (URL or nsIURI) + * The full URI of the page. + * or (string) + * Either the full URI of the page or the GUID of the page. + * or (Array) + * An array of the above, to batch requests. + * @param onResult: (function(PageInfo)) + * A callback invoked for each page found. + * + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolve (bool) + * `true` if at least one page was removed, `false` otherwise. + * @throws (TypeError) + * If `pages` has an unexpected type or if a string provided + * is neither a valid GUID nor a valid URI or if `pages` + * is an empty array. + */ + remove(pages, onResult = null) { + // Normalize and type-check arguments + if (Array.isArray(pages)) { + if (!pages.length) { + throw new TypeError("Expected at least one page"); + } + } else { + pages = [pages]; + } + + let guids = []; + let urls = []; + for (let page of pages) { + // Normalize to URL or GUID, or throw if `page` cannot + // be normalized. + let normalized = lazy.PlacesUtils.normalizeToURLOrGUID(page); + if (typeof normalized === "string") { + guids.push(normalized); + } else { + urls.push(normalized.href); + } + } + + // At this stage, we know that either `guids` is not-empty + // or `urls` is not-empty. + + if (onResult && typeof onResult != "function") { + throw new TypeError("Invalid function: " + onResult); + } + + return (async function () { + let removedPages = false; + let count = 0; + while (guids.length || urls.length) { + if (count && count % 2 == 0) { + // Every few cycles, yield time back to the main + // thread to avoid jank. + await Promise.resolve(); + } + count++; + let guidsSlice = guids.splice(0, REMOVE_PAGES_CHUNKLEN); + let urlsSlice = []; + if (guidsSlice.length < REMOVE_PAGES_CHUNKLEN) { + urlsSlice = urls.splice(0, REMOVE_PAGES_CHUNKLEN - guidsSlice.length); + } + + let pages = { guids: guidsSlice, urls: urlsSlice }; + + let result = await lazy.PlacesUtils.withConnectionWrapper( + "History.jsm: remove", + db => remove(db, pages, onResult) + ); + + removedPages = removedPages || result; + } + return removedPages; + })(); + }, + + /** + * Remove visits matching specific characteristics. + * + * Any change may be observed through PlacesObservers. + * + * @param filter: (object) + * The `object` may contain some of the following + * properties: + * - beginDate: (Date) Remove visits that have + * been added since this date (inclusive). + * - endDate: (Date) Remove visits that have + * been added before this date (inclusive). + * - limit: (Number) Limit the number of visits + * we remove to this number + * - url: (URL) Only remove visits to this URL + * - transition: (Integer) + * The type of the transition (see TRANSITIONS below) + * If both `beginDate` and `endDate` are specified, + * visits between `beginDate` (inclusive) and `end` + * (inclusive) are removed. + * + * @param onResult: (function(VisitInfo), [optional]) + * A callback invoked for each visit found and removed. + * Note that the referrer property of `VisitInfo` + * is NOT populated. + * + * @return (Promise) + * @resolve (bool) + * `true` if at least one visit was removed, `false` + * otherwise. + * @throws (TypeError) + * If `filter` does not have the expected type, in + * particular if the `object` is empty. + */ + removeVisitsByFilter(filter, onResult = null) { + if (!filter || typeof filter != "object") { + throw new TypeError("Expected a filter"); + } + + let hasBeginDate = "beginDate" in filter; + let hasEndDate = "endDate" in filter; + let hasURL = "url" in filter; + let hasLimit = "limit" in filter; + let hasTransition = "transition" in filter; + if (hasBeginDate) { + this.ensureDate(filter.beginDate); + } + if (hasEndDate) { + this.ensureDate(filter.endDate); + } + if (hasBeginDate && hasEndDate && filter.beginDate > filter.endDate) { + throw new TypeError("`beginDate` should be at least as old as `endDate`"); + } + if (hasTransition && !this.isValidTransition(filter.transition)) { + throw new TypeError("`transition` should be valid"); + } + if ( + !hasBeginDate && + !hasEndDate && + !hasURL && + !hasLimit && + !hasTransition + ) { + throw new TypeError("Expected a non-empty filter"); + } + + if ( + hasURL && + !URL.isInstance(filter.url) && + typeof filter.url != "string" && + !(filter.url instanceof Ci.nsIURI) + ) { + throw new TypeError("Expected a valid URL for `url`"); + } + + if ( + hasLimit && + (typeof filter.limit != "number" || + filter.limit <= 0 || + !Number.isInteger(filter.limit)) + ) { + throw new TypeError("Expected a non-zero positive integer as a limit"); + } + + if (onResult && typeof onResult != "function") { + throw new TypeError("Invalid function: " + onResult); + } + + return lazy.PlacesUtils.withConnectionWrapper( + "History.jsm: removeVisitsByFilter", + db => removeVisitsByFilter(db, filter, onResult) + ); + }, + + /** + * Remove pages from the database based on a filter. + * + * Any change may be observed through PlacesObservers + * + * + * @param filter: An object containing a non empty subset of the following + * properties: + * - host: (string) + * Hostname with or without subhost. Examples: + * "mozilla.org" removes pages from mozilla.org but not its subdomains + * ".mozilla.org" removes pages from mozilla.org and its subdomains + * "." removes local files + * - beginDate: (Date) + * The first time the page was visited (inclusive) + * - endDate: (Date) + * The last time the page was visited (inclusive) + * @param [optional] onResult: (function(PageInfo)) + * A callback invoked for each page found. + * + * @note This removes pages with at least one visit inside the timeframe. + * Any visits outside the timeframe will also be removed with the page. + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolve (bool) + * `true` if at least one page was removed, `false` otherwise. + * @throws (TypeError) + * if `filter` does not have the expected type, in particular + * if the `object` is empty, or its components do not satisfy the + * criteria given above + */ + removeByFilter(filter, onResult) { + if (!filter || typeof filter !== "object") { + throw new TypeError("Expected a filter object"); + } + + let hasHost = filter.host; + if (hasHost) { + if (typeof filter.host !== "string") { + throw new TypeError("`host` should be a string"); + } + filter.host = filter.host.toLowerCase(); + if (filter.host.length > 1 && filter.host.lastIndexOf(".") == 0) { + // The input contains only an initial period, thus it may be a + // wildcarded local host, like ".localhost". Ideally the consumer should + // pass just "localhost", because there is no concept of subhosts for + // it, but we are being more lenient to allow for simpler input. + // Anyway, in this case we remove the wildcard to avoid clearing too + // much if the consumer wrongly passes in things like ".com". + filter.host = filter.host.slice(1); + } + } + + let hasBeginDate = "beginDate" in filter; + if (hasBeginDate) { + this.ensureDate(filter.beginDate); + } + + let hasEndDate = "endDate" in filter; + if (hasEndDate) { + this.ensureDate(filter.endDate); + } + + if (hasBeginDate && hasEndDate && filter.beginDate > filter.endDate) { + throw new TypeError("`beginDate` should be at least as old as `endDate`"); + } + + if (!hasBeginDate && !hasEndDate && !hasHost) { + throw new TypeError("Expected a non-empty filter"); + } + + // Check the host format. + // Either it has no dots, or has multiple dots, or it's a single dot char. + if ( + hasHost && + (!/^(\.?([.a-z0-9-]+\.[a-z0-9-]+)?|[a-z0-9-]+)$/.test(filter.host) || + filter.host.includes("..")) + ) { + throw new TypeError( + "Expected well formed hostname string for `host` with atmost 1 wildcard." + ); + } + + if (onResult && typeof onResult != "function") { + throw new TypeError("Invalid function: " + onResult); + } + + return lazy.PlacesUtils.withConnectionWrapper( + "History.jsm: removeByFilter", + db => removeByFilter(db, filter, onResult) + ); + }, + + /** + * Determine if a page has been visited. + * + * @param guidOrURI: (string) or (URL, nsIURI or href) + * Either the full URI of the page or the GUID of the page. + * @return (Promise) + * A promise resolved once the operation is complete. + * @resolve (bool) + * `true` if the page has been visited, `false` otherwise. + * @throws (Error) + * If `guidOrURI` has an unexpected type or if a string provided + * is neither not a valid GUID nor a valid URI. + */ + hasVisits(guidOrURI) { + // Quick fallback to the cpp version. + if (guidOrURI instanceof Ci.nsIURI) { + return new Promise(resolve => { + lazy.asyncHistory.isURIVisited(guidOrURI, (aURI, aIsVisited) => { + resolve(aIsVisited); + }); + }); + } + + guidOrURI = lazy.PlacesUtils.normalizeToURLOrGUID(guidOrURI); + let isGuid = typeof guidOrURI == "string"; + let sqlFragment = isGuid + ? "guid = :val" + : "url_hash = hash(:val) AND url = :val "; + + return lazy.PlacesUtils.promiseDBConnection().then(async db => { + let rows = await db.executeCached( + `SELECT 1 FROM moz_places + WHERE ${sqlFragment} + AND last_visit_date NOTNULL`, + { val: isGuid ? guidOrURI : guidOrURI.href } + ); + return !!rows.length; + }); + }, + + /** + * Clear all history. + * + * @return (Promise) + * A promise resolved once the operation is complete. + */ + clear() { + return lazy.PlacesUtils.withConnectionWrapper("History.jsm: clear", clear); + }, + + /** + * Is a value a valid transition type? + * + * @param transition: (String) + * @return (Boolean) + */ + isValidTransition(transition) { + return Object.values(History.TRANSITIONS).includes(transition); + }, + + /** + * Throw if an object is not a Date object. + */ + ensureDate(arg) { + if ( + !arg || + typeof arg != "object" || + arg.constructor.name != "Date" || + isNaN(arg) + ) { + throw new TypeError("Expected a valid Date, got " + arg); + } + }, + + /** + * Update information for a page. + * + * Currently, it supports updating the description, preview image URL and annotations + * for a page, any other fields will be ignored. + * + * Note that this function will ignore the update if the target page has not + * yet been stored in the database. `History.fetch` could be used to check + * whether the page and its meta information exist or not. Beware that + * fetch&update might fail as they are not executed in a single transaction. + * + * @param pageInfo: (PageInfo) + * pageInfo must contain a URL of the target page. It will be ignored + * if a valid page `guid` is also provided. + * + * If a property `description` is provided, the description of the + * page is updated. Note that: + * 1). An empty string or null `description` will clear the existing + * value in the database. + * 2). Descriptions longer than DB_DESCRIPTION_LENGTH_MAX will be + * truncated. + * + * If a property `siteName` is provided, the site name of the + * page is updated. Note that: + * 1). An empty string or null `siteName` will clear the existing + * value in the database. + * 2). Descriptions longer than DB_SITENAME_LENGTH_MAX will be + * truncated. + * + * If a property `previewImageURL` is provided, the preview image + * URL of the page is updated. Note that: + * 1). A null `previewImageURL` will clear the existing value in the + * database. + * 2). It throws if its length is greater than DB_URL_LENGTH_MAX + * defined in PlacesUtils.jsm. + * + * If a property `annotations` is provided, the annotations will be + * updated. Note that: + * 1). It should be a Map containing key/value pairs to be updated. + * 2). If the value is falsy, the annotation will be removed. + * 3). If the value is non-falsy, the annotation will be added or updated. + * For `annotations` the keys must all be strings, the values should be + * Boolean, Number or Strings. null and undefined are supported as falsy values. + * + * @return (Promise) + * A promise resolved once the update is complete. + * @rejects (Error) + * Rejects if the update was unsuccessful. + * + * @throws (Error) + * If `pageInfo` has an unexpected type. + * @throws (Error) + * If `pageInfo` has an invalid `url` or an invalid `guid`. + * @throws (Error) + * If `pageInfo` has neither `description` nor `previewImageURL`. + * @throws (Error) + * If the length of `pageInfo.previewImageURL` is greater than + * DB_URL_LENGTH_MAX defined in PlacesUtils.jsm. + */ + update(pageInfo) { + let info = lazy.PlacesUtils.validatePageInfo(pageInfo, false); + + if ( + info.description === undefined && + info.siteName === undefined && + info.previewImageURL === undefined && + info.annotations === undefined + ) { + throw new TypeError( + "pageInfo object must at least have either a description, siteName, previewImageURL or annotations property." + ); + } + + return lazy.PlacesUtils.withConnectionWrapper("History.jsm: update", db => + update(db, info) + ); + }, + + /** + * Possible values for the `transition` property of `VisitInfo` + * objects. + */ + + TRANSITIONS: { + /** + * The user followed a link and got a new toplevel window. + */ + LINK: Ci.nsINavHistoryService.TRANSITION_LINK, + + /** + * The user typed the page's URL in the URL bar or selected it from + * URL bar autocomplete results, clicked on it from a history query + * (from the History sidebar, History menu, or history query in the + * personal toolbar or Places organizer. + */ + TYPED: Ci.nsINavHistoryService.TRANSITION_TYPED, + + /** + * The user followed a bookmark to get to the page. + */ + BOOKMARK: Ci.nsINavHistoryService.TRANSITION_BOOKMARK, + + /** + * Some inner content is loaded. This is true of all images on a + * page, and the contents of the iframe. It is also true of any + * content in a frame if the user did not explicitly follow a link + * to get there. + */ + EMBED: Ci.nsINavHistoryService.TRANSITION_EMBED, + + /** + * Set when the transition was a permanent redirect. + */ + REDIRECT_PERMANENT: Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT, + + /** + * Set when the transition was a temporary redirect. + */ + REDIRECT_TEMPORARY: Ci.nsINavHistoryService.TRANSITION_REDIRECT_TEMPORARY, + + /** + * Set when the transition is a download. + */ + DOWNLOAD: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + + /** + * The user followed a link and got a visit in a frame. + */ + FRAMED_LINK: Ci.nsINavHistoryService.TRANSITION_FRAMED_LINK, + + /** + * The user reloaded a page. + */ + RELOAD: Ci.nsINavHistoryService.TRANSITION_RELOAD, + }, +}); + +/** + * Convert a PageInfo object into the format expected by updatePlaces. + * + * Note: this assumes that the PageInfo object has already been validated + * via PlacesUtils.validatePageInfo. + * + * @param pageInfo: (PageInfo) + * @return (info) + */ +function convertForUpdatePlaces(pageInfo) { + let info = { + guid: pageInfo.guid, + uri: lazy.PlacesUtils.toURI(pageInfo.url), + title: pageInfo.title, + visits: [], + }; + + for (let inVisit of pageInfo.visits) { + let visit = { + visitDate: lazy.PlacesUtils.toPRTime(inVisit.date), + transitionType: inVisit.transition, + referrerURI: inVisit.referrer + ? lazy.PlacesUtils.toURI(inVisit.referrer) + : undefined, + }; + info.visits.push(visit); + } + return info; +} + +// Inner implementation of History.clear(). +var clear = async function (db) { + await db.executeTransaction(async function () { + // Remove all non-bookmarked places entries first, this will speed up the + // triggers work. + await db.execute(`DELETE FROM moz_places WHERE foreign_count = 0`); + await db.execute(`DELETE FROM moz_updateoriginsdelete_temp`); + + // Expire orphan icons. + await db.executeCached(`DELETE FROM moz_pages_w_icons + WHERE page_url_hash NOT IN (SELECT url_hash FROM moz_places)`); + await removeOrphanIcons(db); + + // Expire annotations. + await db.execute(`DELETE FROM moz_annos WHERE NOT EXISTS ( + SELECT 1 FROM moz_places WHERE id = place_id + )`); + + // Expire inputhistory. + await db.execute(`DELETE FROM moz_inputhistory WHERE place_id IN ( + SELECT i.place_id FROM moz_inputhistory i + LEFT JOIN moz_places h ON h.id = i.place_id + WHERE h.id IS NULL)`); + + // Remove all history. + await db.execute("DELETE FROM moz_historyvisits"); + }); + + PlacesObservers.notifyListeners([new PlacesHistoryCleared()]); + + // Trigger frecency updates for all affected origins. + await db.execute(`DELETE FROM moz_updateoriginsupdate_temp`); +}; + +/** + * Clean up pages whose history has been modified, by either + * removing them entirely (if they are marked for removal, + * typically because all visits have been removed and there + * are no more foreign keys such as bookmarks) or updating + * their frecency (otherwise). + * + * @param db: (Sqlite connection) + * The database. + * @param pages: (Array of objects) + * Pages that have been touched and that need cleaning up. + * Each object should have the following properties: + * - id: (number) The `moz_places` identifier for the place. + * - hasVisits: (boolean) If `true`, there remains at least one + * visit to this page, so the page should be kept and its + * frecency updated. + * - hasForeign: (boolean) If `true`, the page has at least + * one foreign reference (i.e. a bookmark), so the page should + * be kept and its frecency updated. + * @return (Promise) + */ +var cleanupPages = async function (db, pages) { + let pagesToRemove = pages.filter(p => !p.hasForeign && !p.hasVisits); + if (!pagesToRemove.length) { + return; + } + + // Note, we are already in a transaction, since callers create it. + // Check relations regardless, to avoid creating orphans in case of + // async race conditions. + for (let chunk of lazy.PlacesUtils.chunkArray( + pagesToRemove, + db.variableLimit + )) { + let idsToRemove = chunk.map(p => p.id); + await db.execute( + `DELETE FROM moz_places + WHERE id IN ( ${lazy.PlacesUtils.sqlBindPlaceholders(idsToRemove)} ) + AND foreign_count = 0 AND last_visit_date ISNULL`, + idsToRemove + ); + + // Expire orphans. + let hashesToRemove = chunk.map(p => p.hash); + await db.executeCached( + `DELETE FROM moz_pages_w_icons + WHERE page_url_hash IN (${lazy.PlacesUtils.sqlBindPlaceholders( + hashesToRemove + )})`, + hashesToRemove + ); + + await db.execute( + `DELETE FROM moz_annos + WHERE place_id IN ( ${lazy.PlacesUtils.sqlBindPlaceholders( + idsToRemove + )} )`, + idsToRemove + ); + await db.execute( + `DELETE FROM moz_inputhistory + WHERE place_id IN ( ${lazy.PlacesUtils.sqlBindPlaceholders( + idsToRemove + )} )`, + idsToRemove + ); + } + // Hosts accumulated during the places delete are updated through a trigger + // (see nsPlacesTriggers.h). + await db.executeCached(`DELETE FROM moz_updateoriginsdelete_temp`); + + await removeOrphanIcons(db); +}; + +/** + * Remove icons whose origin is not in moz_origins, unless referenced. + * @param db: (Sqlite connection) + * The database. + */ +function removeOrphanIcons(db) { + return db.executeCached(` + DELETE FROM moz_icons WHERE id IN ( + SELECT id FROM moz_icons WHERE root = 0 + UNION ALL + SELECT id FROM moz_icons + WHERE root = 1 + AND get_host_and_port(icon_url) NOT IN (SELECT host FROM moz_origins) + AND fixup_url(get_host_and_port(icon_url)) NOT IN (SELECT host FROM moz_origins) + EXCEPT + SELECT icon_id FROM moz_icons_to_pages + )`); +} + +/** + * Notify observers that pages have been removed/updated. + * + * @param db: (Sqlite connection) + * The database. + * @param pages: (Array of objects) + * Pages that have been touched and that need cleaning up. + * Each object should have the following properties: + * - id: (number) The `moz_places` identifier for the place. + * - hasVisits: (boolean) If `true`, there remains at least one + * visit to this page, so the page should be kept and its + * frecency updated. + * - hasForeign: (boolean) If `true`, the page has at least + * one foreign reference (i.e. a bookmark), so the page should + * be kept and its frecency updated. + * @param transitionType: (Number) + * Set to a valid TRANSITIONS value to indicate all transitions of a + * certain type have been removed, otherwise defaults to 0 (unknown value). + * @return (Promise) + */ +var notifyCleanup = async function (db, pages, transitionType = 0) { + const notifications = []; + + for (let page of pages) { + const isRemovedFromStore = !page.hasVisits && !page.hasForeign; + notifications.push( + new PlacesVisitRemoved({ + url: page.url.href, + pageGuid: page.guid, + reason: PlacesVisitRemoved.REASON_DELETED, + transitionType, + isRemovedFromStore, + isPartialVisistsRemoval: !isRemovedFromStore && page.hasVisits > 0, + }) + ); + } + + PlacesObservers.notifyListeners(notifications); +}; + +/** + * Notify an `onResult` callback of a set of operations + * that just took place. + * + * @param data: (Array) + * The data to send to the callback. + * @param onResult: (function [optional]) + * If provided, call `onResult` with `data[0]`, `data[1]`, etc. + * Otherwise, do nothing. + */ +var notifyOnResult = async function (data, onResult) { + if (!onResult) { + return; + } + let notifiedCount = 0; + for (let info of data) { + try { + onResult(info); + } catch (ex) { + // Errors should be reported but should not stop the operation. + Promise.reject(ex); + } + if (++notifiedCount % ONRESULT_CHUNK_SIZE == 0) { + // Every few notifications, yield time back to the main + // thread to avoid jank. + await Promise.resolve(); + } + } +}; + +// Inner implementation of History.fetch. +var fetch = async function (db, guidOrURL, options) { + let whereClauseFragment = ""; + let params = {}; + if (URL.isInstance(guidOrURL)) { + whereClauseFragment = "WHERE h.url_hash = hash(:url) AND h.url = :url"; + params.url = guidOrURL.href; + } else { + whereClauseFragment = "WHERE h.guid = :guid"; + params.guid = guidOrURL; + } + + let visitSelectionFragment = ""; + let joinFragment = ""; + let visitOrderFragment = ""; + if (options.includeVisits) { + visitSelectionFragment = ", v.visit_date, v.visit_type"; + joinFragment = "JOIN moz_historyvisits v ON h.id = v.place_id"; + visitOrderFragment = "ORDER BY v.visit_date DESC"; + } + + let pageMetaSelectionFragment = ""; + if (options.includeMeta) { + pageMetaSelectionFragment = ", description, site_name, preview_image_url"; + } + + let query = `SELECT h.id, guid, url, title, frecency + ${pageMetaSelectionFragment} ${visitSelectionFragment} + FROM moz_places h ${joinFragment} + ${whereClauseFragment} + ${visitOrderFragment}`; + let pageInfo = null; + let placeId = null; + await db.executeCached(query, params, row => { + if (pageInfo === null) { + // This means we're on the first row, so we need to get all the page info. + pageInfo = { + guid: row.getResultByName("guid"), + url: new URL(row.getResultByName("url")), + frecency: row.getResultByName("frecency"), + title: row.getResultByName("title") || "", + }; + placeId = row.getResultByName("id"); + } + if (options.includeMeta) { + pageInfo.description = row.getResultByName("description") || ""; + pageInfo.siteName = row.getResultByName("site_name") || ""; + let previewImageURL = row.getResultByName("preview_image_url"); + pageInfo.previewImageURL = previewImageURL + ? new URL(previewImageURL) + : null; + } + if (options.includeVisits) { + // On every row (not just the first), we need to collect visit data. + if (!("visits" in pageInfo)) { + pageInfo.visits = []; + } + let date = lazy.PlacesUtils.toDate(row.getResultByName("visit_date")); + let transition = row.getResultByName("visit_type"); + + // TODO: Bug #1365913 add referrer URL to the `VisitInfo` data as well. + pageInfo.visits.push({ date, transition }); + } + }); + + // Only try to get annotations if requested, and if there's an actual page found. + if (pageInfo && options.includeAnnotations) { + let rows = await db.executeCached( + ` + SELECT n.name, a.content FROM moz_anno_attributes n + JOIN moz_annos a ON n.id = a.anno_attribute_id + WHERE a.place_id = :placeId + `, + { placeId } + ); + + pageInfo.annotations = new Map( + rows.map(row => [ + row.getResultByName("name"), + row.getResultByName("content"), + ]) + ); + } + return pageInfo; +}; + +// Inner implementation of History.fetchAnnotatedPages. +var fetchAnnotatedPages = async function (db, annotations) { + let result = new Map(); + let rows = await db.execute( + ` + SELECT n.name, h.url, a.content FROM moz_anno_attributes n + JOIN moz_annos a ON n.id = a.anno_attribute_id + JOIN moz_places h ON h.id = a.place_id + WHERE n.name IN (${new Array(annotations.length).fill("?").join(",")}) + `, + annotations + ); + + for (let row of rows) { + let uri; + try { + uri = new URL(row.getResultByName("url")); + } catch (ex) { + console.error("Invalid URL read from database in fetchAnnotatedPages"); + continue; + } + + let anno = { + uri, + content: row.getResultByName("content"), + }; + let annoName = row.getResultByName("name"); + let pageAnnos = result.get(annoName); + if (!pageAnnos) { + pageAnnos = []; + result.set(annoName, pageAnnos); + } + pageAnnos.push(anno); + } + + return result; +}; + +// Inner implementation of History.fetchMany. +var fetchMany = async function (db, guidOrURLs) { + let resultsMap = new Map(); + for (let chunk of lazy.PlacesUtils.chunkArray(guidOrURLs, db.variableLimit)) { + let urls = []; + let guids = []; + for (let v of chunk) { + if (URL.isInstance(v)) { + urls.push(v); + } else { + guids.push(v); + } + } + let wheres = []; + let params = []; + if (urls.length) { + wheres.push(` + ( + url_hash IN(${lazy.PlacesUtils.sqlBindPlaceholders( + urls, + "hash(", + ")" + )}) AND + url IN(${lazy.PlacesUtils.sqlBindPlaceholders(urls)}) + )`); + let hrefs = urls.map(u => u.href); + params = [...params, ...hrefs, ...hrefs]; + } + if (guids.length) { + wheres.push(`guid IN(${lazy.PlacesUtils.sqlBindPlaceholders(guids)})`); + params = [...params, ...guids]; + } + + let rows = await db.executeCached( + ` + SELECT h.id, guid, url, title, frecency + FROM moz_places h + WHERE ${wheres.join(" OR ")} + `, + params + ); + for (let row of rows) { + let pageInfo = { + guid: row.getResultByName("guid"), + url: new URL(row.getResultByName("url")), + frecency: row.getResultByName("frecency"), + title: row.getResultByName("title") || "", + }; + if (guidOrURLs.includes(pageInfo.guid)) { + resultsMap.set(pageInfo.guid, pageInfo); + } else { + resultsMap.set(pageInfo.url.href, pageInfo); + } + } + } + return resultsMap; +}; + +// Inner implementation of History.removeVisitsByFilter. +var removeVisitsByFilter = async function (db, filter, onResult = null) { + // 1. Determine visits that took place during the interval. Note + // that the database uses microseconds, while JS uses milliseconds, + // so we need to *1000 one way and /1000 the other way. + let conditions = []; + let args = {}; + let transition = -1; + if ("beginDate" in filter) { + conditions.push("v.visit_date >= :begin * 1000"); + args.begin = Number(filter.beginDate); + } + if ("endDate" in filter) { + conditions.push("v.visit_date <= :end * 1000"); + args.end = Number(filter.endDate); + } + if ("limit" in filter) { + args.limit = Number(filter.limit); + } + if ("transition" in filter) { + conditions.push("v.visit_type = :transition"); + args.transition = filter.transition; + transition = filter.transition; + } + + let optionalJoin = ""; + if ("url" in filter) { + let url = filter.url; + if (url instanceof Ci.nsIURI) { + url = filter.url.spec; + } else { + url = new URL(url).href; + } + optionalJoin = `JOIN moz_places h ON h.id = v.place_id`; + conditions.push("h.url_hash = hash(:url)", "h.url = :url"); + args.url = url; + } + + let visitsToRemove = []; + let pagesToInspect = new Set(); + let onResultData = onResult ? [] : null; + + await db.executeCached( + `SELECT v.id, place_id, visit_date / 1000 AS date, visit_type FROM moz_historyvisits v + ${optionalJoin} + WHERE ${conditions.join(" AND ")}${ + args.limit ? " LIMIT :limit" : "" + }`, + args, + row => { + let id = row.getResultByName("id"); + let place_id = row.getResultByName("place_id"); + visitsToRemove.push(id); + pagesToInspect.add(place_id); + + if (onResult) { + onResultData.push({ + date: new Date(row.getResultByName("date")), + transition: row.getResultByName("visit_type"), + }); + } + } + ); + + if (!visitsToRemove.length) { + // Nothing to do + return false; + } + + let pages = []; + await db.executeTransaction(async function () { + // 2. Remove all offending visits. + for (let chunk of lazy.PlacesUtils.chunkArray( + visitsToRemove, + db.variableLimit + )) { + await db.execute( + `DELETE FROM moz_historyvisits + WHERE id IN (${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk + ); + } + + // 3. Find out which pages have been orphaned + for (let chunk of lazy.PlacesUtils.chunkArray( + [...pagesToInspect], + db.variableLimit + )) { + await db.execute( + `SELECT id, url, url_hash, guid, + (foreign_count != 0) AS has_foreign, + (last_visit_date NOTNULL) as has_visits + FROM moz_places + WHERE id IN (${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk, + row => { + let page = { + id: row.getResultByName("id"), + guid: row.getResultByName("guid"), + hasForeign: row.getResultByName("has_foreign"), + hasVisits: row.getResultByName("has_visits"), + url: new URL(row.getResultByName("url")), + hash: row.getResultByName("url_hash"), + }; + pages.push(page); + } + ); + } + + // 4. Clean up and notify + await cleanupPages(db, pages); + }); + + notifyCleanup(db, pages, transition); + notifyOnResult(onResultData, onResult); // don't wait + + return !!visitsToRemove.length; +}; + +// Inner implementation of History.removeByFilter +var removeByFilter = async function (db, filter, onResult = null) { + // 1. Create fragment for date filtration + let dateFilterSQLFragment = ""; + let conditions = []; + let params = {}; + if ("beginDate" in filter) { + conditions.push("v.visit_date >= :begin"); + params.begin = lazy.PlacesUtils.toPRTime(filter.beginDate); + } + if ("endDate" in filter) { + conditions.push("v.visit_date <= :end"); + params.end = lazy.PlacesUtils.toPRTime(filter.endDate); + } + + if (conditions.length !== 0) { + dateFilterSQLFragment = `EXISTS + (SELECT id FROM moz_historyvisits v WHERE v.place_id = h.id AND + ${conditions.join(" AND ")} + LIMIT 1)`; + } + + // 2. Create fragment for host and subhost filtering + let hostFilterSQLFragment = ""; + if (filter.host) { + // There are four cases that we need to consider: + // mozilla.org, .mozilla.org, localhost, and local files + let revHost = filter.host.split("").reverse().join(""); + if (filter.host == ".") { + // Local files. + hostFilterSQLFragment = `h.rev_host = :revHost`; + } else if (filter.host.startsWith(".")) { + // Remove the subhost wildcard. + revHost = revHost.slice(0, -1); + hostFilterSQLFragment = `h.rev_host between :revHost || "." and :revHost || "/"`; + } else { + // This covers non-wildcarded hosts (e.g.: mozilla.org, localhost) + hostFilterSQLFragment = `h.rev_host = :revHost || "."`; + } + params.revHost = revHost; + } + + // 3. Find out what needs to be removed + let fragmentArray = [hostFilterSQLFragment, dateFilterSQLFragment]; + let query = `SELECT h.id, url, url_hash, rev_host, guid, title, frecency, foreign_count + FROM moz_places h WHERE + (${fragmentArray.filter(f => f !== "").join(") AND (")})`; + let onResultData = onResult ? [] : null; + let pages = []; + let hasPagesToRemove = false; + + await db.executeCached(query, params, row => { + let hasForeign = row.getResultByName("foreign_count") != 0; + if (!hasForeign) { + hasPagesToRemove = true; + } + let id = row.getResultByName("id"); + let guid = row.getResultByName("guid"); + let url = row.getResultByName("url"); + let page = { + id, + guid, + hasForeign, + hasVisits: false, + url: new URL(url), + hash: row.getResultByName("url_hash"), + }; + pages.push(page); + if (onResult) { + onResultData.push({ + guid, + title: row.getResultByName("title"), + frecency: row.getResultByName("frecency"), + url: new URL(url), + }); + } + }); + + if (pages.length === 0) { + // Nothing to do + return false; + } + + await db.executeTransaction(async function () { + // 4. Actually remove visits + let pageIds = pages.map(p => p.id); + for (let chunk of lazy.PlacesUtils.chunkArray(pageIds, db.variableLimit)) { + await db.execute( + `DELETE FROM moz_historyvisits + WHERE place_id IN(${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk + ); + } + // 5. Clean up and notify + await cleanupPages(db, pages); + }); + + notifyCleanup(db, pages); + notifyOnResult(onResultData, onResult); + + return hasPagesToRemove; +}; + +// Inner implementation of History.remove. +var remove = async function (db, { guids, urls }, onResult = null) { + // 1. Find out what needs to be removed + let onResultData = onResult ? [] : null; + let pages = []; + let hasPagesToRemove = false; + function onRow(row) { + let hasForeign = row.getResultByName("foreign_count") != 0; + if (!hasForeign) { + hasPagesToRemove = true; + } + let id = row.getResultByName("id"); + let guid = row.getResultByName("guid"); + let url = row.getResultByName("url"); + let page = { + id, + guid, + hasForeign, + hasVisits: false, + url: new URL(url), + hash: row.getResultByName("url_hash"), + }; + pages.push(page); + if (onResult) { + onResultData.push({ + guid, + title: row.getResultByName("title"), + frecency: row.getResultByName("frecency"), + url: new URL(url), + }); + } + } + for (let chunk of lazy.PlacesUtils.chunkArray(guids, db.variableLimit)) { + let query = `SELECT id, url, url_hash, guid, foreign_count, title, frecency + FROM moz_places + WHERE guid IN (${lazy.PlacesUtils.sqlBindPlaceholders(guids)}) + `; + await db.execute(query, chunk, onRow); + } + for (let chunk of lazy.PlacesUtils.chunkArray(urls, db.variableLimit)) { + // Make an array of variables like `["?1", "?2", ...]`, up to the length of + // the chunk. This lets us bind each URL once, reusing the binding for the + // `url_hash IN (...)` and `url IN (...)` clauses. We add 1 because indexed + // parameters start at 1, not 0. + let variables = Array.from( + { length: chunk.length }, + (_, i) => "?" + (i + 1) + ); + let query = `SELECT id, url, url_hash, guid, foreign_count, title, frecency + FROM moz_places + WHERE url_hash IN (${variables.map(v => `hash(${v})`).join(",")}) AND + url IN (${variables.join(",")}) + `; + await db.execute(query, chunk, onRow); + } + + if (!pages.length) { + // Nothing to do + return false; + } + + await db.executeTransaction(async function () { + // 2. Remove all visits to these pages. + let pageIds = pages.map(p => p.id); + for (let chunk of lazy.PlacesUtils.chunkArray(pageIds, db.variableLimit)) { + await db.execute( + `DELETE FROM moz_historyvisits + WHERE place_id IN (${lazy.PlacesUtils.sqlBindPlaceholders(chunk)})`, + chunk + ); + } + + // 3. Clean up and notify + await cleanupPages(db, pages); + }); + + notifyCleanup(db, pages); + notifyOnResult(onResultData, onResult); // don't wait + + return hasPagesToRemove; +}; + +/** + * Merges an updateInfo object, as returned by asyncHistory.updatePlaces + * into a PageInfo object as defined in this file. + * + * @param updateInfo: (Object) + * An object that represents a page that is generated by + * asyncHistory.updatePlaces. + * @param pageInfo: (PageInfo) + * An PageInfo object into which to merge the data from updateInfo. + * Defaults to an empty object so that this method can be used + * to simply convert an updateInfo object into a PageInfo object. + * + * @return (PageInfo) + * A PageInfo object populated with data from updateInfo. + */ +function mergeUpdateInfoIntoPageInfo(updateInfo, pageInfo = {}) { + pageInfo.guid = updateInfo.guid; + pageInfo.title = updateInfo.title; + if (!pageInfo.url) { + pageInfo.url = URL.fromURI(updateInfo.uri); + pageInfo.title = updateInfo.title; + pageInfo.visits = updateInfo.visits.map(visit => { + return { + date: lazy.PlacesUtils.toDate(visit.visitDate), + transition: visit.transitionType, + referrer: visit.referrerURI ? URL.fromURI(visit.referrerURI) : null, + }; + }); + } + return pageInfo; +} + +// Inner implementation of History.insert. +var insert = function (db, pageInfo) { + let info = convertForUpdatePlaces(pageInfo); + + return new Promise((resolve, reject) => { + lazy.asyncHistory.updatePlaces(info, { + handleError: error => { + reject(error); + }, + handleResult: result => { + pageInfo = mergeUpdateInfoIntoPageInfo(result, pageInfo); + }, + handleCompletion: () => { + resolve(pageInfo); + }, + }); + }); +}; + +// Inner implementation of History.insertMany. +var insertMany = function (db, pageInfos, onResult, onError) { + let infos = []; + let onResultData = []; + let onErrorData = []; + + for (let pageInfo of pageInfos) { + let info = convertForUpdatePlaces(pageInfo); + infos.push(info); + } + + return new Promise((resolve, reject) => { + lazy.asyncHistory.updatePlaces(infos, { + handleError: (resultCode, result) => { + let pageInfo = mergeUpdateInfoIntoPageInfo(result); + onErrorData.push(pageInfo); + }, + handleResult: result => { + let pageInfo = mergeUpdateInfoIntoPageInfo(result); + onResultData.push(pageInfo); + }, + ignoreErrors: !onError, + ignoreResults: !onResult, + handleCompletion: updatedCount => { + notifyOnResult(onResultData, onResult); + notifyOnResult(onErrorData, onError); + if (updatedCount > 0) { + resolve(); + } else { + reject({ message: "No items were added to history." }); + } + }, + }); + }); +}; + +// Inner implementation of History.update. +var update = async function (db, pageInfo) { + // Check for page existence first; we can skip most of the work if it doesn't + // exist and anyway we'll need the place id multiple times later. + // Prefer GUID over url if it's present. + let id; + if (typeof pageInfo.guid === "string") { + let rows = await db.executeCached( + "SELECT id FROM moz_places WHERE guid = :guid", + { guid: pageInfo.guid } + ); + id = rows.length ? rows[0].getResultByName("id") : null; + } else { + let rows = await db.executeCached( + "SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url", + { url: pageInfo.url.href } + ); + id = rows.length ? rows[0].getResultByName("id") : null; + } + if (!id) { + return; + } + + let updateFragments = []; + let params = {}; + if ("description" in pageInfo) { + updateFragments.push("description"); + params.description = pageInfo.description; + } + if ("siteName" in pageInfo) { + updateFragments.push("site_name"); + params.site_name = pageInfo.siteName; + } + if ("previewImageURL" in pageInfo) { + updateFragments.push("preview_image_url"); + params.preview_image_url = pageInfo.previewImageURL + ? pageInfo.previewImageURL.href + : null; + } + if (updateFragments.length) { + // Since this data may be written at every visit and is textual, avoid + // overwriting the existing record if it didn't change. + await db.execute( + ` + UPDATE moz_places + SET ${updateFragments.map(v => `${v} = :${v}`).join(", ")} + WHERE id = :id + AND (${updateFragments + .map(v => `IFNULL(${v}, '') <> IFNULL(:${v}, '')`) + .join(" OR ")}) + `, + { id, ...params } + ); + } + + if (pageInfo.annotations) { + let annosToRemove = []; + let annosToUpdate = []; + + for (let anno of pageInfo.annotations) { + anno[1] ? annosToUpdate.push(anno[0]) : annosToRemove.push(anno[0]); + } + + await db.executeTransaction(async function () { + if (annosToUpdate.length) { + await db.execute( + ` + INSERT OR IGNORE INTO moz_anno_attributes (name) + VALUES ${Array.from(annosToUpdate.keys()) + .map(k => `(:${k})`) + .join(", ")} + `, + Object.assign({}, annosToUpdate) + ); + + for (let anno of annosToUpdate) { + let content = pageInfo.annotations.get(anno); + // TODO: We only really need to save the type whilst we still support + // accessing page annotations via the annotation service. + let type = + typeof content == "string" + ? History.ANNOTATION_TYPE_STRING + : History.ANNOTATION_TYPE_INT64; + let date = lazy.PlacesUtils.toPRTime(new Date()); + + // This will replace the id every time an annotation is updated. This is + // not currently an issue as we're not joining on the id field. + await db.execute( + ` + INSERT OR REPLACE INTO moz_annos + (place_id, anno_attribute_id, content, flags, + expiration, type, dateAdded, lastModified) + VALUES (:id, + (SELECT id FROM moz_anno_attributes WHERE name = :anno_name), + :content, 0, :expiration, :type, :date_added, + :last_modified) + `, + { + id, + anno_name: anno, + content, + expiration: History.ANNOTATION_EXPIRE_NEVER, + type, + // The date fields are unused, so we just set them both to the latest. + date_added: date, + last_modified: date, + } + ); + } + } + + for (let anno of annosToRemove) { + // We don't remove anything from the moz_anno_attributes table. If we + // delete the last item of a given name, that item really should go away. + // It will be cleaned up by expiration. + await db.execute( + ` + DELETE FROM moz_annos + WHERE place_id = :id + AND anno_attribute_id = + (SELECT id FROM moz_anno_attributes WHERE name = :anno_name) + `, + { id, anno_name: anno } + ); + } + }); + } +}; diff --git a/toolkit/components/places/INativePlacesEventCallback.h b/toolkit/components/places/INativePlacesEventCallback.h new file mode 100644 index 0000000000..46e52146e2 --- /dev/null +++ b/toolkit/components/places/INativePlacesEventCallback.h @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_image_INativePlacesEventCallback_h +#define mozilla_image_INativePlacesEventCallback_h + +#include "mozilla/dom/PlacesObserversBinding.h" +#include "mozilla/WeakPtr.h" +#include "nsISupports.h" +#include "nsTArray.h" + +namespace mozilla { +namespace places { + +class INativePlacesEventCallback : public SupportsWeakPtr { + public: + typedef dom::Sequence> PlacesEventSequence; + + NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING + + virtual void HandlePlacesEvent(const PlacesEventSequence& aEvents) = 0; + + protected: + virtual ~INativePlacesEventCallback() = default; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_image_INativePlacesEventCallback_h diff --git a/toolkit/components/places/NotifyRankingChanged.h b/toolkit/components/places/NotifyRankingChanged.h new file mode 100644 index 0000000000..e902fe8016 --- /dev/null +++ b/toolkit/components/places/NotifyRankingChanged.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_NotifyRankingChanged_h_ +#define mozilla_places_NotifyRankingChanged_h_ + +#include "mozilla/dom/PlacesObservers.h" +#include "mozilla/dom/PlacesRanking.h" + +using namespace mozilla::dom; + +namespace mozilla { +namespace places { + +class NotifyRankingChanged final : public Runnable { + public: + NotifyRankingChanged() : Runnable("places::NotifyRankingChanged") {} + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is marked + // MOZ_CAN_RUN_SCRIPT. See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY + NS_IMETHOD Run() override { + MOZ_ASSERT(NS_IsMainThread(), "This should be called on the main thread"); + + RefPtr event = new PlacesRanking(); + Sequence> events; + bool success = !!events.AppendElement(event.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + PlacesObservers::NotifyListeners(events); + + return NS_OK; + } +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_NotifyRankingChanged_h_ diff --git a/toolkit/components/places/PageIconProtocolHandler.cpp b/toolkit/components/places/PageIconProtocolHandler.cpp new file mode 100644 index 0000000000..ae85532718 --- /dev/null +++ b/toolkit/components/places/PageIconProtocolHandler.cpp @@ -0,0 +1,396 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PageIconProtocolHandler.h" + +#include "mozilla/NullPrincipal.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Components.h" +#include "nsFaviconService.h" +#include "nsStringStream.h" +#include "nsStreamUtils.h" +#include "nsIChannel.h" +#include "nsIFaviconService.h" +#include "nsIIOService.h" +#include "nsILoadInfo.h" +#include "nsIOutputStream.h" +#include "nsIPipe.h" +#include "nsIRequestObserver.h" +#include "nsIURIMutator.h" +#include "nsNetUtil.h" +#include "SimpleChannel.h" + +#define PAGE_ICON_SCHEME "page-icon" + +using mozilla::net::IsNeckoChild; +using mozilla::net::NeckoChild; +using mozilla::net::RemoteStreamGetter; +using mozilla::net::RemoteStreamInfo; + +namespace mozilla::places { + +struct FaviconMetadata { + nsCOMPtr mStream; + nsCString mContentType; + int64_t mContentLength = 0; +}; + +StaticRefPtr PageIconProtocolHandler::sSingleton; + +namespace { + +class DefaultFaviconObserver final : public nsIRequestObserver { + public: + explicit DefaultFaviconObserver(nsIOutputStream* aOutputStream) + : mOutputStream(aOutputStream) { + MOZ_ASSERT(aOutputStream); + } + + NS_DECL_ISUPPORTS + NS_DECL_NSIREQUESTOBSERVER + private: + ~DefaultFaviconObserver() = default; + + nsCOMPtr mOutputStream; +}; + +NS_IMPL_ISUPPORTS(DefaultFaviconObserver, nsIRequestObserver); + +NS_IMETHODIMP DefaultFaviconObserver::OnStartRequest(nsIRequest*) { + return NS_OK; +} + +NS_IMETHODIMP DefaultFaviconObserver::OnStopRequest(nsIRequest*, nsresult) { + // We must close the outputStream regardless. + mOutputStream->Close(); + return NS_OK; +} + +} // namespace + +static nsresult MakeDefaultFaviconChannel(nsIURI* aURI, nsILoadInfo* aLoadInfo, + nsIChannel** aOutChannel) { + nsCOMPtr ios = mozilla::components::IO::Service(); + nsCOMPtr chan; + nsCOMPtr defaultFaviconURI; + + auto* faviconService = nsFaviconService::GetFaviconService(); + if (MOZ_UNLIKELY(!faviconService)) { + return NS_ERROR_UNEXPECTED; + } + + nsresult rv = + faviconService->GetDefaultFavicon(getter_AddRefs(defaultFaviconURI)); + NS_ENSURE_SUCCESS(rv, rv); + rv = ios->NewChannelFromURIWithLoadInfo(defaultFaviconURI, aLoadInfo, + getter_AddRefs(chan)); + NS_ENSURE_SUCCESS(rv, rv); + chan->SetOriginalURI(aURI); + chan->SetContentType(nsLiteralCString(FAVICON_DEFAULT_MIMETYPE)); + chan.forget(aOutChannel); + return NS_OK; +} + +static nsresult StreamDefaultFavicon(nsIURI* aURI, nsILoadInfo* aLoadInfo, + nsIOutputStream* aOutputStream) { + auto closeStreamOnError = + mozilla::MakeScopeExit([&] { aOutputStream->Close(); }); + + auto observer = MakeRefPtr(aOutputStream); + + nsCOMPtr listener; + nsresult rv = NS_NewSimpleStreamListener(getter_AddRefs(listener), + aOutputStream, observer); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr defaultIconChannel; + rv = MakeDefaultFaviconChannel(aURI, aLoadInfo, + getter_AddRefs(defaultIconChannel)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = defaultIconChannel->AsyncOpen(listener); + NS_ENSURE_SUCCESS(rv, rv); + + closeStreamOnError.release(); + return NS_OK; +} + +namespace { + +class FaviconDataCallback final : public nsIFaviconDataCallback { + public: + FaviconDataCallback(nsIURI* aURI, nsILoadInfo* aLoadInfo) + : mURI(aURI), mLoadInfo(aLoadInfo) { + MOZ_ASSERT(aURI); + MOZ_ASSERT(aLoadInfo); + } + + NS_DECL_ISUPPORTS + NS_DECL_NSIFAVICONDATACALLBACK + + RefPtr Promise() { + return mPromiseHolder.Ensure(__func__); + } + + private: + ~FaviconDataCallback(); + nsCOMPtr mURI; + MozPromiseHolder mPromiseHolder; + nsCOMPtr mLoadInfo; +}; + +NS_IMPL_ISUPPORTS(FaviconDataCallback, nsIFaviconDataCallback); + +FaviconDataCallback::~FaviconDataCallback() { + mPromiseHolder.RejectIfExists(NS_ERROR_FAILURE, __func__); +} + +NS_IMETHODIMP FaviconDataCallback::OnComplete(nsIURI* aURI, uint32_t aDataLen, + const uint8_t* aData, + const nsACString& aMimeType, + uint16_t aWidth) { + if (!aDataLen) { + mPromiseHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__); + return NS_OK; + } + + nsCOMPtr inputStream; + nsresult rv = + NS_NewByteInputStream(getter_AddRefs(inputStream), + AsChars(Span{aData, aDataLen}), NS_ASSIGNMENT_COPY); + if (NS_FAILED(rv)) { + mPromiseHolder.Reject(rv, __func__); + return rv; + } + + FaviconMetadata metadata; + metadata.mStream = inputStream; + metadata.mContentType = aMimeType; + metadata.mContentLength = aDataLen; + mPromiseHolder.Resolve(std::move(metadata), __func__); + + return NS_OK; +} + +} // namespace + +NS_IMPL_ISUPPORTS(PageIconProtocolHandler, nsIProtocolHandler, + nsISupportsWeakReference); + +NS_IMETHODIMP PageIconProtocolHandler::GetScheme(nsACString& aScheme) { + aScheme.AssignLiteral(PAGE_ICON_SCHEME); + return NS_OK; +} + +NS_IMETHODIMP PageIconProtocolHandler::AllowPort(int32_t, const char*, + bool* aAllow) { + *aAllow = false; + return NS_OK; +} + +NS_IMETHODIMP PageIconProtocolHandler::NewChannel(nsIURI* aURI, + nsILoadInfo* aLoadInfo, + nsIChannel** aOutChannel) { + // Load the URI remotely if accessed from a child. + if (IsNeckoChild()) { + MOZ_TRY(SubstituteRemoteChannel(aURI, aLoadInfo, aOutChannel)); + return NS_OK; + } + + nsresult rv = NewChannelInternal(aURI, aLoadInfo, aOutChannel); + if (NS_SUCCEEDED(rv)) { + return rv; + } + return MakeDefaultFaviconChannel(aURI, aLoadInfo, aOutChannel); +} + +Result PageIconProtocolHandler::SubstituteRemoteChannel( + nsIURI* aURI, nsILoadInfo* aLoadInfo, nsIChannel** aRetVal) { + MOZ_ASSERT(IsNeckoChild()); + MOZ_TRY(aURI ? NS_OK : NS_ERROR_INVALID_ARG); + MOZ_TRY(aLoadInfo ? NS_OK : NS_ERROR_INVALID_ARG); + + RefPtr streamGetter = + new RemoteStreamGetter(aURI, aLoadInfo); + + NewSimpleChannel(aURI, aLoadInfo, streamGetter, aRetVal); + return Ok(); +} + +nsresult PageIconProtocolHandler::NewChannelInternal(nsIURI* aURI, + nsILoadInfo* aLoadInfo, + nsIChannel** aOutChannel) { + // Create a pipe that will give us an output stream that we can use once + // we got all the favicon data. + nsCOMPtr pipeIn; + nsCOMPtr pipeOut; + GetStreams(getter_AddRefs(pipeIn), getter_AddRefs(pipeOut)); + + // Create our channel. + nsCOMPtr channel; + { + // We override the channel's loadinfo below anyway, so using a null + // principal here is alright. + nsCOMPtr loadingPrincipal = + NullPrincipal::CreateWithoutOriginAttributes(); + nsresult rv = NS_NewInputStreamChannel( + getter_AddRefs(channel), aURI, pipeIn.forget(), loadingPrincipal, + nsILoadInfo::SEC_REQUIRE_SAME_ORIGIN_DATA_IS_BLOCKED, + nsIContentPolicy::TYPE_INTERNAL_IMAGE); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsresult rv = channel->SetLoadInfo(aLoadInfo); + NS_ENSURE_SUCCESS(rv, rv); + + GetFaviconData(aURI, aLoadInfo) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [pipeOut, channel](const FaviconMetadata& aMetadata) { + channel->SetContentType(aMetadata.mContentType); + channel->SetContentLength(aMetadata.mContentLength); + + nsresult rv; + const nsCOMPtr target = + do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID, &rv); + + if (NS_WARN_IF(NS_FAILED(rv))) { + channel->CancelWithReason(NS_BINDING_ABORTED, + "GetFaviconData failed"_ns); + return; + } + + NS_AsyncCopy(aMetadata.mStream, pipeOut, target); + }, + [uri = nsCOMPtr{aURI}, loadInfo = nsCOMPtr{aLoadInfo}, pipeOut, + channel](nsresult aRv) { + // There are a few reasons why this might fail. For example, one + // reason is that the URI might not actually be properly parsable. + // In that case, we'll try one last time to stream the default + // favicon before giving up. + channel->SetContentType(nsLiteralCString(FAVICON_DEFAULT_MIMETYPE)); + channel->SetContentLength(-1); + Unused << StreamDefaultFavicon(uri, loadInfo, pipeOut); + }); + channel.forget(aOutChannel); + return NS_OK; +} + +RefPtr PageIconProtocolHandler::GetFaviconData( + nsIURI* aPageIconURI, nsILoadInfo* aLoadInfo) { + auto* faviconService = nsFaviconService::GetFaviconService(); + if (MOZ_UNLIKELY(!faviconService)) { + return FaviconMetadataPromise::CreateAndReject(NS_ERROR_UNEXPECTED, + __func__); + } + + uint16_t preferredSize = 0; + faviconService->PreferredSizeFromURI(aPageIconURI, &preferredSize); + + nsCOMPtr pageURI; + nsresult rv; + { + // NOTE: We don't need to strip #size= fragments because + // GetFaviconDataForPage strips them when doing the database lookup. + nsAutoCString pageQuery; + aPageIconURI->GetPathQueryRef(pageQuery); + rv = NS_NewURI(getter_AddRefs(pageURI), pageQuery); + if (NS_FAILED(rv)) { + return FaviconMetadataPromise::CreateAndReject(rv, __func__); + } + } + + auto faviconCallback = + MakeRefPtr(aPageIconURI, aLoadInfo); + rv = faviconService->GetFaviconDataForPage(pageURI, faviconCallback, + preferredSize); + if (NS_FAILED(rv)) { + return FaviconMetadataPromise::CreateAndReject(rv, __func__); + } + + return faviconCallback->Promise(); +} + +RefPtr PageIconProtocolHandler::NewStream( + nsIURI* aChildURI, nsILoadInfo* aLoadInfo, bool* aTerminateSender) { + MOZ_ASSERT(!IsNeckoChild()); + MOZ_ASSERT(NS_IsMainThread()); + + if (!aChildURI || !aLoadInfo || !aTerminateSender) { + return RemoteStreamPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__); + } + + *aTerminateSender = true; + + // We should never receive a URI that isn't for a page-icon because + // these requests ordinarily come from the child's PageIconProtocolHandler. + // Ensure this request is for a page-icon URI. A compromised child process + // could send us any URI. + bool isPageIconScheme = false; + if (NS_FAILED(aChildURI->SchemeIs(PAGE_ICON_SCHEME, &isPageIconScheme)) || + !isPageIconScheme) { + return RemoteStreamPromise::CreateAndReject(NS_ERROR_UNKNOWN_PROTOCOL, + __func__); + } + + // For errors after this point, we want to propagate the error to + // the child, but we don't force the child process to be terminated. + *aTerminateSender = false; + + RefPtr outerPromise = + new RemoteStreamPromise::Private(__func__); + nsCOMPtr uri(aChildURI); + nsCOMPtr loadInfo(aLoadInfo); + RefPtr self = this; + + GetFaviconData(uri, loadInfo) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [outerPromise](const FaviconMetadata& aMetadata) { + RemoteStreamInfo info(aMetadata.mStream, aMetadata.mContentType, + aMetadata.mContentLength); + outerPromise->Resolve(std::move(info), __func__); + }, + [self, uri, loadInfo, outerPromise](nsresult aRv) { + nsCOMPtr pipeIn; + nsCOMPtr pipeOut; + self->GetStreams(getter_AddRefs(pipeIn), getter_AddRefs(pipeOut)); + + RemoteStreamInfo info( + pipeIn, nsLiteralCString(FAVICON_DEFAULT_MIMETYPE), -1); + Unused << StreamDefaultFavicon(uri, loadInfo, pipeOut); + outerPromise->Resolve(std::move(info), __func__); + }); + return outerPromise; +} + +void PageIconProtocolHandler::GetStreams(nsIAsyncInputStream** inStream, + nsIAsyncOutputStream** outStream) { + static constexpr size_t kSegmentSize = 4096; + nsCOMPtr pipeIn; + nsCOMPtr pipeOut; + NS_NewPipe2(getter_AddRefs(pipeIn), getter_AddRefs(pipeOut), true, true, + kSegmentSize, + nsIFaviconService::MAX_FAVICON_BUFFER_SIZE / kSegmentSize); + + pipeIn.forget(inStream); + pipeOut.forget(outStream); +} + +// static +void PageIconProtocolHandler::NewSimpleChannel( + nsIURI* aURI, nsILoadInfo* aLoadInfo, RemoteStreamGetter* aStreamGetter, + nsIChannel** aRetVal) { + nsCOMPtr channel = NS_NewSimpleChannel( + aURI, aLoadInfo, aStreamGetter, + [](nsIStreamListener* listener, nsIChannel* simpleChannel, + RemoteStreamGetter* getter) -> RequestOrReason { + return getter->GetAsync(listener, simpleChannel, + &NeckoChild::SendGetPageIconStream); + }); + + channel.swap(*aRetVal); +} + +} // namespace mozilla::places diff --git a/toolkit/components/places/PageIconProtocolHandler.h b/toolkit/components/places/PageIconProtocolHandler.h new file mode 100644 index 0000000000..7cfeeae99b --- /dev/null +++ b/toolkit/components/places/PageIconProtocolHandler.h @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_PageIconProtocolHandler_h +#define mozilla_places_PageIconProtocolHandler_h + +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/MozPromise.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/net/RemoteStreamGetter.h" +#include "nsIProtocolHandler.h" +#include "nsThreadUtils.h" +#include "nsWeakReference.h" + +namespace mozilla::places { + +struct FaviconMetadata; +using FaviconMetadataPromise = + mozilla::MozPromise; + +using net::RemoteStreamPromise; + +class PageIconProtocolHandler final : public nsIProtocolHandler, + public nsSupportsWeakReference { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIPROTOCOLHANDLER + + static already_AddRefed GetSingleton() { + MOZ_ASSERT(NS_IsMainThread()); + if (MOZ_UNLIKELY(!sSingleton)) { + sSingleton = new PageIconProtocolHandler(); + ClearOnShutdown(&sSingleton); + } + return do_AddRef(sSingleton); + } + + /** + * To be called in the parent process to obtain an input stream for the + * given icon. + * + * @param aChildURI a page-icon URI sent from the child. + * @param aLoadInfo the nsILoadInfo for the load attempt from the child. + * @param aTerminateSender out param set to true when the params are invalid + * and indicate the child should be terminated. If |aChildURI| is + * not a page-icon URI, the child is in an invalid state and + * should be terminated. This outparam will be set synchronously. + * + * @return RemoteStreamPromise + * The RemoteStreamPromise will resolve with an RemoteStreamInfo on + * success, and reject with an nsresult on failure. + */ + RefPtr NewStream(nsIURI* aChildURI, + nsILoadInfo* aLoadInfo, + bool* aTerminateSender); + + private: + ~PageIconProtocolHandler() = default; + + /** + * This replaces the provided channel with a channel that will proxy the load + * to the parent process. + * + * @param aURI the page-icon: URI. + * @param aLoadInfo the loadinfo for the request. + * @param aRetVal in/out channel param referring to the channel that + * might need to be substituted with a remote channel. + * @return NS_OK if the replacement channel was created successfully. + * Otherwise, returns an error. + */ + Result SubstituteRemoteChannel(nsIURI* aURI, + nsILoadInfo* aLoadInfo, + nsIChannel** aRetVal); + + RefPtr GetFaviconData(nsIURI* aPageIconURI, + nsILoadInfo* aLoadInfo); + + nsresult NewChannelInternal(nsIURI*, nsILoadInfo*, nsIChannel**); + + void GetStreams(nsIAsyncInputStream** inStream, + nsIAsyncOutputStream** outStream); + + // Gets a SimpleChannel that wraps the provided channel. + static void NewSimpleChannel(nsIURI* aURI, nsILoadInfo* aLoadinfo, + mozilla::net::RemoteStreamGetter* aStreamGetter, + nsIChannel** aRetVal); + static StaticRefPtr sSingleton; +}; + +} // namespace mozilla::places + +#endif diff --git a/toolkit/components/places/PlaceInfo.cpp b/toolkit/components/places/PlaceInfo.cpp new file mode 100644 index 0000000000..fc489ddb78 --- /dev/null +++ b/toolkit/components/places/PlaceInfo.cpp @@ -0,0 +1,120 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PlaceInfo.h" +#include "VisitInfo.h" +#include "nsIURI.h" +#include "nsServiceManagerUtils.h" +#include "nsIXPConnect.h" +#include "jsapi.h" +#include "js/Array.h" // JS::NewArrayObject +#include "js/PropertyAndElement.h" // JS_DefineElement + +namespace mozilla { +namespace places { + +//////////////////////////////////////////////////////////////////////////////// +//// PlaceInfo + +PlaceInfo::PlaceInfo(int64_t aId, const nsCString& aGUID, + already_AddRefed aURI, const nsString& aTitle, + int64_t aFrecency) + : mId(aId), + mGUID(aGUID), + mURI(aURI), + mTitle(aTitle), + mFrecency(aFrecency), + mVisitsAvailable(false) { + MOZ_ASSERT(mURI, "Must provide a non-null uri!"); +} + +PlaceInfo::PlaceInfo(int64_t aId, const nsCString& aGUID, + already_AddRefed aURI, const nsString& aTitle, + int64_t aFrecency, const VisitsArray& aVisits) + : mId(aId), + mGUID(aGUID), + mURI(aURI), + mTitle(aTitle), + mFrecency(aFrecency), + mVisits(aVisits.Clone()), + mVisitsAvailable(true) { + MOZ_ASSERT(mURI, "Must provide a non-null uri!"); +} + +//////////////////////////////////////////////////////////////////////////////// +//// mozIPlaceInfo + +NS_IMETHODIMP +PlaceInfo::GetPlaceId(int64_t* _placeId) { + *_placeId = mId; + return NS_OK; +} + +NS_IMETHODIMP +PlaceInfo::GetGuid(nsACString& _guid) { + _guid = mGUID; + return NS_OK; +} + +NS_IMETHODIMP +PlaceInfo::GetUri(nsIURI** _uri) { + NS_ADDREF(*_uri = mURI); + return NS_OK; +} + +NS_IMETHODIMP +PlaceInfo::GetTitle(nsAString& _title) { + _title = mTitle; + return NS_OK; +} + +NS_IMETHODIMP +PlaceInfo::GetFrecency(int64_t* _frecency) { + *_frecency = mFrecency; + return NS_OK; +} + +NS_IMETHODIMP +PlaceInfo::GetVisits(JSContext* aContext, + JS::MutableHandle _visits) { + // If the visits data was not provided, return null rather + // than an empty array to distinguish this case from the case + // of a place without any visit. + if (!mVisitsAvailable) { + _visits.setNull(); + return NS_OK; + } + + // TODO bug 625913 when we use this in situations that have more than one + // visit here, we will likely want to make this cache the value. + JS::Rooted visits(aContext, JS::NewArrayObject(aContext, 0)); + NS_ENSURE_TRUE(visits, NS_ERROR_OUT_OF_MEMORY); + + JS::Rooted global(aContext, JS::CurrentGlobalOrNull(aContext)); + NS_ENSURE_TRUE(global, NS_ERROR_UNEXPECTED); + + nsCOMPtr xpc = nsIXPConnect::XPConnect(); + + for (VisitsArray::size_type idx = 0; idx < mVisits.Length(); idx++) { + JS::Rooted jsobj(aContext); + nsresult rv = xpc->WrapNative(aContext, global, mVisits[idx], + NS_GET_IID(mozIVisitInfo), jsobj.address()); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_STATE(jsobj); + + bool rc = JS_DefineElement(aContext, visits, idx, jsobj, JSPROP_ENUMERATE); + NS_ENSURE_TRUE(rc, NS_ERROR_UNEXPECTED); + } + + _visits.setObject(*visits); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsISupports + +NS_IMPL_ISUPPORTS(PlaceInfo, mozIPlaceInfo) + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/PlaceInfo.h b/toolkit/components/places/PlaceInfo.h new file mode 100644 index 0000000000..6363ae1386 --- /dev/null +++ b/toolkit/components/places/PlaceInfo.h @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_PlaceInfo_h__ +#define mozilla_places_PlaceInfo_h__ + +#include "mozIAsyncHistory.h" +#include "nsCOMPtr.h" +#include "nsIURI.h" +#include "nsString.h" +#include "nsTArray.h" +#include "mozilla/Attributes.h" + +class mozIVisitInfo; + +namespace mozilla { +namespace places { + +class PlaceInfo final : public mozIPlaceInfo { + public: + NS_DECL_ISUPPORTS + NS_DECL_MOZIPLACEINFO + + typedef nsTArray > VisitsArray; + + PlaceInfo(int64_t aId, const nsCString& aGUID, already_AddRefed aURI, + const nsString& aTitle, int64_t aFrecency); + PlaceInfo(int64_t aId, const nsCString& aGUID, already_AddRefed aURI, + const nsString& aTitle, int64_t aFrecency, + const VisitsArray& aVisits); + + private: + ~PlaceInfo() = default; + + const int64_t mId; + const nsCString mGUID; + nsCOMPtr mURI; + const nsString mTitle; + const int64_t mFrecency; + const VisitsArray mVisits; + bool mVisitsAvailable; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_PlaceInfo_h__ diff --git a/toolkit/components/places/PlacesBackups.sys.mjs b/toolkit/components/places/PlacesBackups.sys.mjs new file mode 100644 index 0000000000..833e7b0a22 --- /dev/null +++ b/toolkit/components/places/PlacesBackups.sys.mjs @@ -0,0 +1,519 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=2 ts=2 sts=2 expandtab filetype=javascript + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + BookmarkJSONUtils: "resource://gre/modules/BookmarkJSONUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter( + lazy, + "filenamesRegex", + () => + /^bookmarks-([0-9-]+)(?:_([0-9]+)){0,1}(?:_([a-z0-9=+-]{24})){0,1}\.(json(lz4)?)$/i +); + +async function limitBackups(aMaxBackups, backupFiles) { + if ( + typeof aMaxBackups == "number" && + aMaxBackups > -1 && + backupFiles.length >= aMaxBackups + ) { + let numberOfBackupsToDelete = backupFiles.length - aMaxBackups; + while (numberOfBackupsToDelete--) { + let oldestBackup = backupFiles.pop(); + await IOUtils.remove(oldestBackup); + } + } +} + +/** + * Appends meta-data information to a given filename. + */ +function appendMetaDataToFilename(aFilename, aMetaData) { + let matches = aFilename.match(lazy.filenamesRegex); + return ( + "bookmarks-" + + matches[1] + + "_" + + aMetaData.count + + "_" + + aMetaData.hash + + "." + + matches[4] + ); +} + +/** + * Gets the hash from a backup filename. + * + * @return the extracted hash or null. + */ +function getHashFromFilename(aFilename) { + let matches = aFilename.match(lazy.filenamesRegex); + if (matches && matches[3]) { + return matches[3]; + } + return null; +} + +/** + * Given two filenames, checks if they contain the same date. + */ +function isFilenameWithSameDate(aSourceName, aTargetName) { + let sourceMatches = aSourceName.match(lazy.filenamesRegex); + let targetMatches = aTargetName.match(lazy.filenamesRegex); + + return sourceMatches && targetMatches && sourceMatches[1] == targetMatches[1]; +} + +/** + * Given a filename, searches for another backup with the same date. + * + * @return path string or null. + */ +function getBackupFileForSameDate(aFilename) { + return (async function () { + let backupFiles = await PlacesBackups.getBackupFiles(); + for (let backupFile of backupFiles) { + if (isFilenameWithSameDate(PathUtils.filename(backupFile), aFilename)) { + return backupFile; + } + } + return null; + })(); +} + +export var PlacesBackups = { + /** + * Matches the backup filename: + * 0: file name + * 1: date in form Y-m-d + * 2: bookmarks count + * 3: contents hash + * 4: file extension + */ + get filenamesRegex() { + return lazy.filenamesRegex; + }, + + /** + * Gets backup folder asynchronously. + * @return {Promise} + * @resolve the folder (the folder string path). + */ + getBackupFolder: function PB_getBackupFolder() { + return (async () => { + if (this._backupFolder) { + return this._backupFolder; + } + let backupsDirPath = PathUtils.join( + PathUtils.profileDir, + this.profileRelativeFolderPath + ); + await IOUtils.makeDirectory(backupsDirPath); + return (this._backupFolder = backupsDirPath); + })(); + }, + + get profileRelativeFolderPath() { + return "bookmarkbackups"; + }, + + /** + * Cache current backups in a sorted (by date DESC) array. + * @return {Promise} + * @resolve a sorted array of string paths. + */ + getBackupFiles: function PB_getBackupFiles() { + return (async () => { + if (this._backupFiles) { + return this._backupFiles; + } + + this._backupFiles = []; + + let backupFolderPath = await this.getBackupFolder(); + let children = await IOUtils.getChildren(backupFolderPath); + let list = []; + for (const entry of children) { + // Since IOUtils I/O is serialized, we can safely remove .tmp files + // without risking to remove ongoing backups. + let filename = PathUtils.filename(entry); + if (filename.endsWith(".tmp")) { + list.push(IOUtils.remove(entry)); + continue; + } + + if (lazy.filenamesRegex.test(filename)) { + // Remove bogus backups in future dates. + if (this.getDateForFile(entry) > new Date()) { + list.push(IOUtils.remove(entry)); + continue; + } + this._backupFiles.push(entry); + } + } + await Promise.all(list); + + this._backupFiles.sort((a, b) => { + let aDate = this.getDateForFile(a); + let bDate = this.getDateForFile(b); + return bDate - aDate; + }); + + return this._backupFiles; + })(); + }, + + /** + * Invalidates the internal cache for testing purposes. + */ + invalidateCache() { + this._backupFiles = null; + }, + + /** + * Generates a ISO date string (YYYY-MM-DD) from a Date object. + * + * @param dateObj + * The date object to parse. + * @return an ISO date string. + */ + toISODateString: function toISODateString(dateObj) { + if (!dateObj || dateObj.constructor.name != "Date" || !dateObj.getTime()) { + throw new Error("invalid date object"); + } + let padDate = val => ("0" + val).substr(-2, 2); + return [ + dateObj.getFullYear(), + padDate(dateObj.getMonth() + 1), + padDate(dateObj.getDate()), + ].join("-"); + }, + + /** + * Creates a filename for bookmarks backup files. + * + * @param [optional] aDateObj + * Date object used to build the filename. + * Will use current date if empty. + * @param [optional] bool - aCompress + * Determines if file extension is json or jsonlz4 + Default is json + * @return A bookmarks backup filename. + */ + getFilenameForDate: function PB_getFilenameForDate(aDateObj, aCompress) { + let dateObj = aDateObj || new Date(); + // Use YYYY-MM-DD (ISO 8601) as it doesn't contain illegal characters + // and makes the alphabetical order of multiple backup files more useful. + return ( + "bookmarks-" + + PlacesBackups.toISODateString(dateObj) + + ".json" + + (aCompress ? "lz4" : "") + ); + }, + + /** + * Creates a Date object from a backup file. The date is the backup + * creation date. + * + * @param {Sring} aBackupFile The path of the backup. + * @return {Date} A Date object for the backup's creation time. + */ + getDateForFile: function PB_getDateForFile(aBackupFile) { + let filename = PathUtils.filename(aBackupFile); + let matches = filename.match(lazy.filenamesRegex); + if (!matches) { + throw new Error(`Invalid backup file name: ${filename}`); + } + return new Date(matches[1].replace(/-/g, "/")); + }, + + /** + * Get the most recent backup file. + * + * @return {Promise} + * @result the path to the file. + */ + getMostRecentBackup: function PB_getMostRecentBackup() { + return (async () => { + let entries = await this.getBackupFiles(); + for (let entry of entries) { + let rx = /\.json(lz4)?$/; + if (PathUtils.filename(entry).match(rx)) { + return entry; + } + } + return null; + })(); + }, + + /** + * Returns whether a recent enough backup exists, using these heuristic: if + * a backup exists, it should be newer than the last browser session date, + * otherwise it should not be older than maxDays. + * If the backup is older than the last session, the calculated time is + * reported to telemetry. + * + * @param [maxDays] The maximum number of days a backup can be old. + */ + async hasRecentBackup({ maxDays = 3 } = {}) { + let lastBackupFile = await PlacesBackups.getMostRecentBackup(); + if (!lastBackupFile) { + return false; + } + let lastBackupTime = PlacesBackups.getDateForFile(lastBackupFile); + let profileLastUse = Services.appinfo.replacedLockTime || Date.now(); + if (lastBackupTime > profileLastUse) { + return true; + } + let backupAge = Math.round((profileLastUse - lastBackupTime) / 86400000); + // Telemetry the age of the last available backup. + try { + Services.telemetry + .getHistogramById("PLACES_BACKUPS_DAYSFROMLAST") + .add(backupAge); + } catch (ex) { + console.error(new Error("Unable to report telemetry.")); + } + return backupAge <= maxDays; + }, + + /** + * Serializes bookmarks using JSON, and writes to the supplied file. + * + * @param aFilePath + * path for the "bookmarks.json" file to be created. + * @return {Promise} + * @resolves the number of serialized uri nodes. + */ + async saveBookmarksToJSONFile(aFilePath) { + let { count: nodeCount, hash: hash } = + await lazy.BookmarkJSONUtils.exportToFile(aFilePath); + + let backupFolderPath = await this.getBackupFolder(); + if (PathUtils.profileDir == backupFolderPath) { + // We are creating a backup in the default backups folder, + // so just update the internal cache. + if (!this._backupFiles) { + await this.getBackupFiles(); + } + this._backupFiles.unshift(aFilePath); + } else { + let aMaxBackup = Services.prefs.getIntPref( + "browser.bookmarks.max_backups" + ); + if (aMaxBackup === 0) { + if (!this._backupFiles) { + await this.getBackupFiles(); + } + limitBackups(aMaxBackup, this._backupFiles); + return nodeCount; + } + // If we are saving to a folder different than our backups folder, then + // we also want to create a new compressed version in it. + // This way we ensure the latest valid backup is the same saved by the + // user. See bug 424389. + let mostRecentBackupFile = await this.getMostRecentBackup(); + if ( + !mostRecentBackupFile || + hash != getHashFromFilename(PathUtils.filename(mostRecentBackupFile)) + ) { + let name = this.getFilenameForDate(undefined, true); + let newFilename = appendMetaDataToFilename(name, { + count: nodeCount, + hash, + }); + let newFilePath = PathUtils.join(backupFolderPath, newFilename); + let backupFile = await getBackupFileForSameDate(name); + if (backupFile) { + // There is already a backup for today, replace it. + await IOUtils.remove(backupFile); + if (!this._backupFiles) { + await this.getBackupFiles(); + } else { + this._backupFiles.shift(); + } + this._backupFiles.unshift(newFilePath); + } else { + // There is no backup for today, add the new one. + if (!this._backupFiles) { + await this.getBackupFiles(); + } + this._backupFiles.unshift(newFilePath); + } + let jsonString = await IOUtils.read(aFilePath); + await IOUtils.write(newFilePath, jsonString, { + compress: true, + }); + await limitBackups(aMaxBackup, this._backupFiles); + } + } + return nodeCount; + }, + + /** + * Creates a dated backup in /bookmarkbackups. + * Stores the bookmarks using a lz4 compressed JSON file. + * + * @param [optional] int aMaxBackups + * The maximum number of backups to keep. If set to 0 + * all existing backups are removed and aForceBackup is + * ignored, so a new one won't be created. + * @param [optional] bool aForceBackup + * Forces creating a backup even if one was already + * created that day (overwrites). + * @return {Promise} + */ + create: function PB_create(aMaxBackups, aForceBackup) { + return (async () => { + if (aMaxBackups === 0) { + // Backups are disabled, delete any existing one and bail out. + if (!this._backupFiles) { + await this.getBackupFiles(); + } + await limitBackups(0, this._backupFiles); + return; + } + + // Ensure to initialize _backupFiles + if (!this._backupFiles) { + await this.getBackupFiles(); + } + let newBackupFilename = this.getFilenameForDate(undefined, true); + // If we already have a backup for today we should do nothing, unless we + // were required to enforce a new backup. + let backupFile = await getBackupFileForSameDate(newBackupFilename); + if (backupFile && !aForceBackup) { + return; + } + + if (backupFile) { + // In case there is a backup for today we should recreate it. + this._backupFiles.shift(); + await IOUtils.remove(backupFile); + } + + // Now check the hash of the most recent backup, and try to create a new + // backup, if that fails due to hash conflict, just rename the old backup. + let mostRecentBackupFile = await this.getMostRecentBackup(); + let mostRecentHash = + mostRecentBackupFile && + getHashFromFilename(PathUtils.filename(mostRecentBackupFile)); + + // Save bookmarks to a backup file. + let backupFolder = await this.getBackupFolder(); + let newBackupFile = PathUtils.join(backupFolder, newBackupFilename); + let newFilenameWithMetaData; + try { + let { count: nodeCount, hash: hash } = + await lazy.BookmarkJSONUtils.exportToFile(newBackupFile, { + compress: true, + failIfHashIs: mostRecentHash, + }); + newFilenameWithMetaData = appendMetaDataToFilename(newBackupFilename, { + count: nodeCount, + hash, + }); + } catch (ex) { + if (!ex.becauseSameHash) { + throw ex; + } + // The last backup already contained up-to-date information, just + // rename it as if it was today's backup. + this._backupFiles.shift(); + newBackupFile = mostRecentBackupFile; + // Ensure we retain the proper extension when renaming + // the most recent backup file. + if (/\.json$/.test(PathUtils.filename(mostRecentBackupFile))) { + newBackupFilename = this.getFilenameForDate(); + } + newFilenameWithMetaData = appendMetaDataToFilename(newBackupFilename, { + count: this.getBookmarkCountForFile(mostRecentBackupFile), + hash: mostRecentHash, + }); + } + + // Append metadata to the backup filename. + let newBackupFileWithMetadata = PathUtils.join( + backupFolder, + newFilenameWithMetaData + ); + await IOUtils.move(newBackupFile, newBackupFileWithMetadata); + this._backupFiles.unshift(newBackupFileWithMetadata); + + // Limit the number of backups. + await limitBackups(aMaxBackups, this._backupFiles); + })(); + }, + + /** + * Gets the bookmark count for backup file. + * + * @param aFilePath + * File path The backup file. + * + * @return the bookmark count or null. + */ + getBookmarkCountForFile: function PB_getBookmarkCountForFile(aFilePath) { + let count = null; + let filename = PathUtils.filename(aFilePath); + let matches = filename.match(lazy.filenamesRegex); + if (matches && matches[2]) { + count = matches[2]; + } + return count; + }, + + /** + * Gets a bookmarks tree representation usable to create backups in different + * file formats. The root or the tree is PlacesUtils.bookmarks.rootGuid. + * + * @return an object representing a tree with the places root as its root. + * Each bookmark is represented by an object having these properties: + * * id: the item id (make this not enumerable after bug 824502) + * * title: the title + * * guid: unique id + * * parent: item id of the parent folder, not enumerable + * * index: the position in the parent + * * dateAdded: microseconds from the epoch + * * lastModified: microseconds from the epoch + * * type: type of the originating node as defined in PlacesUtils + * The following properties exist only for a subset of bookmarks: + * * annos: array of annotations + * * uri: url + * * iconUri: favicon's url + * * keyword: associated keyword + * * charset: last known charset + * * tags: csv string of tags + * * root: string describing whether this represents a root + * * children: array of child items in a folder + */ + async getBookmarksTree() { + let startTime = Date.now(); + let root = await lazy.PlacesUtils.promiseBookmarksTree( + lazy.PlacesUtils.bookmarks.rootGuid, + { + includeItemIds: true, + } + ); + + try { + Services.telemetry + .getHistogramById("PLACES_BACKUPS_BOOKMARKSTREE_MS") + .add(Date.now() - startTime); + } catch (ex) { + console.error("Unable to report telemetry."); + } + return [root, root.itemsCount]; + }, +}; diff --git a/toolkit/components/places/PlacesDBUtils.sys.mjs b/toolkit/components/places/PlacesDBUtils.sys.mjs new file mode 100644 index 0000000000..05a771efad --- /dev/null +++ b/toolkit/components/places/PlacesDBUtils.sys.mjs @@ -0,0 +1,1443 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=2 ts=2 sts=2 expandtab filetype=javascript + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const BYTES_PER_MEBIBYTE = 1048576; +const MS_PER_DAY = 86400000; +// Threshold value for removeOldCorruptDBs. +// Corrupt DBs older than this value are removed. +const CORRUPT_DB_RETAIN_DAYS = 14; + +// Seconds between maintenance runs. +const MAINTENANCE_INTERVAL_SECONDS = 7 * 86400; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesPreviews: "resource://gre/modules/PlacesPreviews.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Sqlite: "resource://gre/modules/Sqlite.sys.mjs", +}); + +export var PlacesDBUtils = { + _isShuttingDown: false, + + _clearTaskQueue: false, + clearPendingTasks() { + PlacesDBUtils._clearTaskQueue = true; + }, + + /** + * Executes integrity check and common maintenance tasks. + * + * @return a Map[taskName(String) -> Object]. The Object has the following properties: + * - succeeded: boolean + * - logs: an array of strings containing the messages logged by the task. + */ + async maintenanceOnIdle() { + let tasks = [ + this.checkIntegrity, + this.invalidateCaches, + this.checkCoherence, + this._refreshUI, + this.originFrecencyStats, + this.incrementalVacuum, + this.removeOldCorruptDBs, + this.deleteOrphanPreviews, + ]; + let telemetryStartTime = Date.now(); + let taskStatusMap = await PlacesDBUtils.runTasks(tasks); + + Services.prefs.setIntPref( + "places.database.lastMaintenance", + parseInt(Date.now() / 1000) + ); + Services.telemetry + .getHistogramById("PLACES_IDLE_MAINTENANCE_TIME_MS") + .add(Date.now() - telemetryStartTime); + return taskStatusMap; + }, + + /** + * Executes integrity check, common and advanced maintenance tasks (like + * expiration and vacuum). Will also collect statistics on the database. + * + * Note: although this function isn't actually async, we keep it async to + * allow us to maintain a simple, consistent API for the tasks within this object. + * + * @return {Promise} + * A promise that resolves with a Map[taskName(String) -> Object]. + * The Object has the following properties: + * - succeeded: boolean + * - logs: an array of strings containing the messages logged by the task. + */ + async checkAndFixDatabase() { + let tasks = [ + this.checkIntegrity, + this.invalidateCaches, + this.checkCoherence, + this.expire, + this.originFrecencyStats, + this.vacuum, + this.stats, + this._refreshUI, + ]; + return PlacesDBUtils.runTasks(tasks); + }, + + /** + * Forces a full refresh of Places views. + * + * Note: although this function isn't actually async, we keep it async to + * allow us to maintain a simple, consistent API for the tasks within this object. + * + * @returns {Array} An empty array. + */ + async _refreshUI() { + PlacesObservers.notifyListeners([new PlacesPurgeCaches()]); + return []; + }, + + /** + * Checks integrity and tries to fix the database through a reindex. + * + * @return {Promise} resolves if database is sane or is made sane. + * @resolves to an array of logs for this task. + * @rejects if we're unable to fix corruption or unable to check status. + */ + async checkIntegrity() { + let logs = []; + + async function check(dbName) { + try { + await integrity(dbName); + logs.push(`The ${dbName} database is sane`); + } catch (ex) { + PlacesDBUtils.clearPendingTasks(); + if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) { + logs.push(`The ${dbName} database is corrupt`); + Services.prefs.setCharPref( + "places.database.replaceDatabaseOnStartup", + dbName + ); + throw new Error( + `Unable to fix corruption, ${dbName} will be replaced on next startup` + ); + } + throw new Error(`Unable to check ${dbName} integrity: ${ex}`); + } + } + + await check("places.sqlite"); + await check("favicons.sqlite"); + + return logs; + }, + + invalidateCaches() { + let logs = []; + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesDBUtils: invalidate caches", + async db => { + let idsWithStaleGuidsRows = await db.execute( + `SELECT id FROM moz_bookmarks + WHERE guid IS NULL OR + NOT IS_VALID_GUID(guid) OR + (type = :bookmark_type AND fk IS NULL) OR + (type <> :bookmark_type AND fk NOT NULL) OR + type IS NULL`, + { bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK } + ); + for (let row of idsWithStaleGuidsRows) { + let id = row.getResultByName("id"); + lazy.PlacesUtils.invalidateCachedGuidFor(id); + } + logs.push("The caches have been invalidated"); + return logs; + } + ).catch(ex => { + PlacesDBUtils.clearPendingTasks(); + throw new Error("Unable to invalidate caches"); + }); + }, + + /** + * Checks data coherence and tries to fix most common errors. + * + * @return {Promise} resolves when coherence is checked. + * @resolves to an array of logs for this task. + * @rejects if database is not coherent. + */ + async checkCoherence() { + let logs = []; + let stmts = await PlacesDBUtils._getCoherenceStatements(); + let coherenceCheck = true; + await lazy.PlacesUtils.withConnectionWrapper( + "PlacesDBUtils: coherence check:", + db => + db.executeTransaction(async () => { + for (let { query, params } of stmts) { + try { + await db.execute(query, params || null); + } catch (ex) { + console.error(ex); + coherenceCheck = false; + } + } + }) + ); + + if (coherenceCheck) { + logs.push("The database is coherent"); + } else { + PlacesDBUtils.clearPendingTasks(); + throw new Error("Unable to complete the coherence check"); + } + return logs; + }, + + /** + * Runs incremental vacuum on databases supporting it. + * + * @return {Promise} resolves when done. + * @resolves to an array of logs for this task. + * @rejects if we were unable to vacuum. + */ + async incrementalVacuum() { + let logs = []; + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesDBUtils: incrementalVacuum", + async db => { + let count = ( + await db.execute("PRAGMA favicons.freelist_count") + )[0].getResultByIndex(0); + if (count < 10) { + logs.push( + `The favicons database has only ${count} free pages, not vacuuming.` + ); + } else { + logs.push( + `The favicons database has ${count} free pages, vacuuming.` + ); + await db.execute("PRAGMA favicons.incremental_vacuum"); + count = ( + await db.execute("PRAGMA favicons.freelist_count") + )[0].getResultByIndex(0); + logs.push( + `The database has been vacuumed and has now ${count} free pages.` + ); + } + return logs; + } + ).catch(ex => { + PlacesDBUtils.clearPendingTasks(); + throw new Error( + "Unable to incrementally vacuum the favicons database " + ex + ); + }); + }, + + /** + * Expire orphan previews that don't have a Places entry anymore. + * + * @return {Promise} resolves when done. + * @resolves to an array of logs for this task. + */ + async deleteOrphanPreviews() { + let logs = []; + try { + let deleted = await lazy.PlacesPreviews.deleteOrphans(); + if (deleted) { + logs.push(`Orphan previews deleted.`); + } + } catch (ex) { + throw new Error("Unable to delete orphan previews " + ex); + } + return logs; + }, + + async _getCoherenceStatements() { + let cleanupStatements = [ + // MOZ_PLACES + // L.1 remove duplicate URLs. + // This task uses a temp table of potential dupes, and a trigger to remove + // them. It runs first because it relies on subsequent tasks to clean up + // orphaned foreign key references. The task works like this: first, we + // insert all rows with the same hash into the temp table. This lets + // SQLite use the `url_hash` index for scanning `moz_places`. Hashes + // aren't unique, so two different URLs might have the same hash. To find + // the actual dupes, we use a unique constraint on the URL in the temp + // table. If that fails, we bump the dupe count. Then, we delete all dupes + // from the table. This fires the cleanup trigger, which updates all + // foreign key references to point to one of the duplicate Places, then + // deletes the others. + { + query: `CREATE TEMP TABLE IF NOT EXISTS moz_places_dupes_temp( + id INTEGER PRIMARY KEY + , hash INTEGER NOT NULL + , url TEXT UNIQUE NOT NULL + , count INTEGER NOT NULL DEFAULT 0 + )`, + }, + { + query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_places_remove_dupes_temp_trigger + AFTER DELETE ON moz_places_dupes_temp + FOR EACH ROW + BEGIN + /* Reassign history visits. */ + UPDATE moz_historyvisits SET + place_id = OLD.id + WHERE place_id IN (SELECT id FROM moz_places + WHERE id <> OLD.id AND + url_hash = OLD.hash AND + url = OLD.url); + + /* Merge autocomplete history entries. */ + INSERT INTO moz_inputhistory(place_id, input, use_count) + SELECT OLD.id, a.input, a.use_count + FROM moz_inputhistory a + JOIN moz_places h ON h.id = a.place_id + WHERE h.id <> OLD.id AND + h.url_hash = OLD.hash AND + h.url = OLD.url + ON CONFLICT(place_id, input) DO UPDATE SET + place_id = excluded.place_id, + use_count = use_count + excluded.use_count; + + /* Merge page annos, ignoring annos with the same name that are + already set on the destination. */ + INSERT OR IGNORE INTO moz_annos(id, place_id, anno_attribute_id, + content, flags, expiration, type, + dateAdded, lastModified) + SELECT (SELECT k.id FROM moz_annos k + WHERE k.place_id = OLD.id AND + k.anno_attribute_id = a.anno_attribute_id), OLD.id, + a.anno_attribute_id, a.content, a.flags, a.expiration, a.type, + a.dateAdded, a.lastModified + FROM moz_annos a + JOIN moz_places h ON h.id = a.place_id + WHERE h.id <> OLD.id AND + url_hash = OLD.hash AND + url = OLD.url; + + /* Reassign bookmarks, and bump the Sync change counter just in case + we have new keywords. */ + UPDATE moz_bookmarks SET + fk = OLD.id, + syncChangeCounter = syncChangeCounter + 1 + WHERE fk IN (SELECT id FROM moz_places + WHERE url_hash = OLD.hash AND + url = OLD.url); + + /* Reassign keywords. */ + UPDATE moz_keywords SET + place_id = OLD.id + WHERE place_id IN (SELECT id FROM moz_places + WHERE id <> OLD.id AND + url_hash = OLD.hash AND + url = OLD.url); + + /* Now that we've updated foreign key references, drop the + conflicting source. */ + DELETE FROM moz_places + WHERE id <> OLD.id AND + url_hash = OLD.hash AND + url = OLD.url; + + /* Recalculate frecency for the destination. */ + UPDATE moz_places SET recalc_frecency = 1, recalc_alt_frecency = 1 + WHERE id = OLD.id; + END`, + }, + { + query: `INSERT INTO moz_places_dupes_temp(id, hash, url, count) + SELECT h.id, h.url_hash, h.url, 1 + FROM moz_places h + JOIN (SELECT url_hash FROM moz_places + GROUP BY url_hash + HAVING count(*) > 1) d ON d.url_hash = h.url_hash + ON CONFLICT(url) DO UPDATE SET + count = count + 1`, + }, + { query: `DELETE FROM moz_places_dupes_temp WHERE count > 1` }, + { query: `DROP TABLE moz_places_dupes_temp` }, + + // MOZ_ANNO_ATTRIBUTES + // A.1 remove obsolete annotations from moz_annos. + // The 'weave0' idiom exploits character ordering (0 follows /) to + // efficiently select all annos with a 'weave/' prefix. + { + query: `DELETE FROM moz_annos + WHERE type = 4 OR anno_attribute_id IN ( + SELECT id FROM moz_anno_attributes + WHERE name = 'downloads/destinationFileName' OR + name BETWEEN 'weave/' AND 'weave0' + )`, + }, + + // A.3 remove unused attributes. + { + query: `DELETE FROM moz_anno_attributes WHERE id IN ( + SELECT id FROM moz_anno_attributes n + WHERE NOT EXISTS + (SELECT id FROM moz_annos WHERE anno_attribute_id = n.id LIMIT 1) + )`, + }, + + // MOZ_ANNOS + // B.1 remove annos with an invalid attribute + { + query: `DELETE FROM moz_annos WHERE id IN ( + SELECT id FROM moz_annos a + WHERE NOT EXISTS + (SELECT id FROM moz_anno_attributes + WHERE id = a.anno_attribute_id LIMIT 1) + )`, + }, + + // B.2 remove orphan annos + { + query: `DELETE FROM moz_annos WHERE id IN ( + SELECT id FROM moz_annos a + WHERE NOT EXISTS + (SELECT id FROM moz_places WHERE id = a.place_id LIMIT 1) + )`, + }, + + // D.1 remove items that are not uri bookmarks from tag containers + { + query: `DELETE FROM moz_bookmarks WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT b.id FROM moz_bookmarks b + WHERE b.parent IN + (SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) + AND b.type <> :bookmark_type + )`, + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.2 remove empty tags + { + query: `DELETE FROM moz_bookmarks WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT b.id FROM moz_bookmarks b + WHERE b.id IN + (SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) + AND NOT EXISTS + (SELECT id from moz_bookmarks WHERE parent = b.id LIMIT 1) + )`, + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.3 move orphan items to unsorted folder + { + query: `UPDATE moz_bookmarks SET + parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid) + WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT b.id FROM moz_bookmarks b + WHERE NOT EXISTS + (SELECT id FROM moz_bookmarks WHERE id = b.parent LIMIT 1) + )`, + params: { + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.4 Insert tombstones for any synced items with the wrong type. + // Sync doesn't support changing the type of an existing item while + // keeping its GUID. To avoid confusing other clients, we insert + // tombstones for all synced items with the wrong type, so that we + // can reupload them with the correct type and a new GUID. + { + query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved) + SELECT guid, :dateRemoved + FROM moz_bookmarks + WHERE syncStatus <> :syncStatus AND + ((type IN (:folder_type, :separator_type) AND + fk NOTNULL) OR + (type = :bookmark_type AND + fk IS NULL) OR + type IS NULL)`, + params: { + dateRemoved: lazy.PlacesUtils.toPRTime(new Date()), + syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW, + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + }, + + // D.5 fix wrong item types + // Folders and separators should not have an fk. + // If they have a valid fk, convert them to bookmarks, and give them new + // GUIDs. If the item has children, we'll move them to the unfiled root + // in D.8. If the `fk` doesn't exist in `moz_places`, we'll remove the + // item in D.9. + { + query: `UPDATE moz_bookmarks + SET guid = GENERATE_GUID(), + type = :bookmark_type + WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT id FROM moz_bookmarks b + WHERE type IN (:folder_type, :separator_type) + AND fk NOTNULL + )`, + params: { + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.6 fix wrong item types + // Bookmarks should have an fk, if they don't have any, convert them to + // folders. + { + query: `UPDATE moz_bookmarks + SET guid = GENERATE_GUID(), + type = :folder_type + WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT id FROM moz_bookmarks b + WHERE type = :bookmark_type + AND fk IS NULL + )`, + params: { + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.7 fix wrong item types + // `moz_bookmarks.type` doesn't have a NOT NULL constraint, so it's + // possible for an item to not have a type (bug 1586427). + { + query: `UPDATE moz_bookmarks + SET guid = GENERATE_GUID(), + type = CASE WHEN fk NOT NULL THEN :bookmark_type ELSE :folder_type END + WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND type IS NULL`, + params: { + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.8 fix wrong parents + // Items cannot have separators or other bookmarks + // as parent, if they have bad parent move them to unsorted bookmarks. + { + query: `UPDATE moz_bookmarks SET + parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid) + WHERE guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND id IN ( + SELECT id FROM moz_bookmarks b + WHERE EXISTS + (SELECT id FROM moz_bookmarks WHERE id = b.parent + AND type IN (:bookmark_type, :separator_type) + LIMIT 1) + )`, + params: { + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.9 remove items without a valid place + // We've already converted folders with an `fk` to bookmarks in D.5, + // and bookmarks without an `fk` to folders in D.6. However, the `fk` + // might not reference an existing `moz_places.id`, even if it's + // NOT NULL. This statement takes care of those. + { + query: `DELETE FROM moz_bookmarks AS b + WHERE b.guid NOT IN ( + :rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */ + ) AND b.fk NOT NULL + AND b.type = :bookmark_type + AND NOT EXISTS (SELECT 1 FROM moz_places h WHERE h.id = b.fk)`, + params: { + bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + rootGuid: lazy.PlacesUtils.bookmarks.rootGuid, + menuGuid: lazy.PlacesUtils.bookmarks.menuGuid, + toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid, + }, + }, + + // D.10 fix non-consecutive positions. + { + query: ` + WITH positions(item_id, pos, seq) AS ( + SELECT id, position AS pos, + (row_number() OVER (PARTITION BY parent ORDER BY position)) - 1 AS seq + FROM moz_bookmarks + ) + UPDATE moz_bookmarks + SET position = seq + FROM positions + WHERE item_id = moz_bookmarks.id AND seq <> pos`, + }, + + // D.12 Fix empty-named tags. + // Tags were allowed to have empty names due to a UI bug. Fix them by + // replacing their title with "(notitle)", and bumping the change counter + // for all bookmarks with the fixed tags. + { + query: `UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1 + WHERE fk IN (SELECT b.fk FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE length(p.title) = 0 AND p.type = :folder_type AND + p.parent = :tags_folder)`, + params: { + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + tags_folder: lazy.PlacesUtils.tagsFolderId, + }, + }, + { + query: `UPDATE moz_bookmarks SET title = :empty_title + WHERE length(title) = 0 AND type = :folder_type + AND parent = :tags_folder`, + params: { + empty_title: "(notitle)", + folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER, + tags_folder: lazy.PlacesUtils.tagsFolderId, + }, + }, + + // MOZ_ICONS + // E.1 remove orphan icon entries. + { + query: `DELETE FROM moz_pages_w_icons WHERE page_url_hash NOT IN ( + SELECT url_hash FROM moz_places + )`, + }, + + // Remove icons whose origin is not in moz_origins, unless referenced. + { + query: `DELETE FROM moz_icons WHERE id IN ( + SELECT id FROM moz_icons WHERE root = 0 + UNION ALL + SELECT id FROM moz_icons + WHERE root = 1 + AND get_host_and_port(icon_url) NOT IN (SELECT host FROM moz_origins) + AND fixup_url(get_host_and_port(icon_url)) NOT IN (SELECT host FROM moz_origins) + EXCEPT + SELECT icon_id FROM moz_icons_to_pages + )`, + }, + + // MOZ_HISTORYVISITS + // F.1 remove orphan visits + { + query: `DELETE FROM moz_historyvisits WHERE id IN ( + SELECT id FROM moz_historyvisits v + WHERE NOT EXISTS + (SELECT id FROM moz_places WHERE id = v.place_id LIMIT 1) + )`, + }, + + // MOZ_INPUTHISTORY + // G.1 remove orphan input history + { + query: `DELETE FROM moz_inputhistory WHERE place_id IN ( + SELECT place_id FROM moz_inputhistory i + WHERE NOT EXISTS + (SELECT id FROM moz_places WHERE id = i.place_id LIMIT 1) + )`, + }, + + // MOZ_KEYWORDS + // I.1 remove unused keywords + { + query: `DELETE FROM moz_keywords WHERE id IN ( + SELECT id FROM moz_keywords k + WHERE NOT EXISTS + (SELECT 1 FROM moz_places h WHERE k.place_id = h.id) + )`, + }, + + // MOZ_PLACES + // L.2 recalculate visit_count and last_visit_date + { + query: `UPDATE moz_places + SET visit_count = (SELECT count(*) FROM moz_historyvisits + WHERE place_id = moz_places.id AND visit_type NOT IN (0,4,7,8,9)), + last_visit_date = (SELECT MAX(visit_date) FROM moz_historyvisits + WHERE place_id = moz_places.id) + WHERE id IN ( + SELECT h.id FROM moz_places h + WHERE visit_count <> (SELECT count(*) FROM moz_historyvisits v + WHERE v.place_id = h.id AND visit_type NOT IN (0,4,7,8,9)) + OR last_visit_date IS NOT + (SELECT MAX(visit_date) FROM moz_historyvisits v WHERE v.place_id = h.id) + )`, + }, + + // L.3 recalculate hidden for redirects. + { + query: `UPDATE moz_places + SET hidden = 1 + WHERE id IN ( + SELECT h.id FROM moz_places h + JOIN moz_historyvisits src ON src.place_id = h.id + JOIN moz_historyvisits dst ON dst.from_visit = src.id AND dst.visit_type IN (5,6) + LEFT JOIN moz_bookmarks on fk = h.id AND fk ISNULL + GROUP BY src.place_id HAVING count(*) = visit_count + )`, + }, + + // L.4 recalculate foreign_count. + { + query: `UPDATE moz_places SET foreign_count = + (SELECT count(*) FROM moz_bookmarks WHERE fk = moz_places.id ) + + (SELECT count(*) FROM moz_keywords WHERE place_id = moz_places.id )`, + }, + + // L.5 recalculate missing hashes. + { + query: `UPDATE moz_places SET url_hash = hash(url) WHERE url_hash = 0`, + }, + + // L.6 fix invalid Place GUIDs. + { + query: `UPDATE moz_places + SET guid = GENERATE_GUID() + WHERE guid IS NULL OR + NOT IS_VALID_GUID(guid)`, + }, + + // MOZ_BOOKMARKS + // S.1 fix invalid GUIDs for synced bookmarks. + // This requires multiple related statements. + // First, we insert tombstones for all synced bookmarks with invalid + // GUIDs, so that we can delete them on the server. Second, we add a + // temporary trigger to bump the change counter for the parents of any + // items we update, since Sync stores the list of child GUIDs on the + // parent. Finally, we assign new GUIDs for all items with missing and + // invalid GUIDs, bump their change counters, and reset their sync + // statuses to NEW so that they're considered for deduping. + { + query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved) + SELECT guid, :dateRemoved + FROM moz_bookmarks + WHERE syncStatus <> :syncStatus AND + guid NOT NULL AND + NOT IS_VALID_GUID(guid)`, + params: { + dateRemoved: lazy.PlacesUtils.toPRTime(new Date()), + syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW, + }, + }, + { + query: `UPDATE moz_bookmarks + SET guid = GENERATE_GUID(), + syncStatus = :syncStatus + WHERE guid IS NULL OR + NOT IS_VALID_GUID(guid)`, + params: { + syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW, + }, + }, + + // S.2 drop tombstones for bookmarks that aren't deleted. + { + query: `DELETE FROM moz_bookmarks_deleted + WHERE guid IN (SELECT guid FROM moz_bookmarks)`, + }, + + // S.3 set missing added and last modified dates. + { + query: `UPDATE moz_bookmarks + SET dateAdded = COALESCE(NULLIF(dateAdded, 0), NULLIF(lastModified, 0), NULLIF(( + SELECT MIN(visit_date) FROM moz_historyvisits + WHERE place_id = fk + ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000), + lastModified = COALESCE(NULLIF(lastModified, 0), NULLIF(dateAdded, 0), NULLIF(( + SELECT MAX(visit_date) FROM moz_historyvisits + WHERE place_id = fk + ), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000) + WHERE NULLIF(dateAdded, 0) IS NULL OR + NULLIF(lastModified, 0) IS NULL`, + }, + + // S.4 reset added dates that are ahead of last modified dates. + { + query: `UPDATE moz_bookmarks + SET dateAdded = lastModified + WHERE dateAdded > lastModified`, + }, + ]; + + // Create triggers for updating Sync metadata. The "sync change" trigger + // bumps the parent's change counter when we update a GUID or move an item + // to a different folder, since Sync stores the list of child GUIDs on the + // parent. The "sync tombstone" trigger inserts tombstones for deleted + // synced bookmarks. + cleanupStatements.unshift({ + query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_change_temp_trigger + AFTER UPDATE OF guid, parent, position ON moz_bookmarks + FOR EACH ROW + BEGIN + UPDATE moz_bookmarks + SET syncChangeCounter = syncChangeCounter + 1 + WHERE id IN (OLD.parent, NEW.parent, NEW.id); + END`, + }); + cleanupStatements.unshift({ + query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_tombstone_temp_trigger + AFTER DELETE ON moz_bookmarks + FOR EACH ROW WHEN OLD.guid NOT NULL AND + OLD.syncStatus <> 1 + BEGIN + UPDATE moz_bookmarks + SET syncChangeCounter = syncChangeCounter + 1 + WHERE id = OLD.parent; + + INSERT INTO moz_bookmarks_deleted(guid, dateRemoved) + VALUES(OLD.guid, STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000); + END`, + }); + cleanupStatements.push({ + query: `DROP TRIGGER moz_bm_sync_change_temp_trigger`, + }); + cleanupStatements.push({ + query: `DROP TRIGGER moz_bm_sync_tombstone_temp_trigger`, + }); + + return cleanupStatements; + }, + + /** + * Tries to vacuum the database. + * + * Note: although this function isn't actually async, we keep it async to + * allow us to maintain a simple, consistent API for the tasks within this object. + * + * @return {Promise} resolves when database is vacuumed. + * @resolves to an array of logs for this task. + * @rejects if we are unable to vacuum database. + */ + async vacuum() { + let logs = []; + let placesDbPath = PathUtils.join(PathUtils.profileDir, "places.sqlite"); + let info = await IOUtils.stat(placesDbPath); + logs.push(`Initial database size is ${parseInt(info.size / 1024)}KiB`); + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesDBUtils: vacuum", + async db => { + await db.execute("VACUUM"); + logs.push("The database has been vacuumed"); + info = await IOUtils.stat(placesDbPath); + logs.push(`Final database size is ${parseInt(info.size / 1024)}KiB`); + return logs; + } + ).catch(() => { + PlacesDBUtils.clearPendingTasks(); + throw new Error("Unable to vacuum database"); + }); + }, + + /** + * Forces a full expiration on the database. + * + * Note: although this function isn't actually async, we keep it async to + * allow us to maintain a simple, consistent API for the tasks within this object. + * + * @return {Promise} resolves when the database in cleaned up. + * @resolves to an array of logs for this task. + */ + async expire() { + let logs = []; + + let expiration = Cc["@mozilla.org/places/expiration;1"].getService( + Ci.nsIObserver + ); + + let returnPromise = new Promise(res => { + let observer = (subject, topic, data) => { + Services.obs.removeObserver(observer, topic); + logs.push("Database cleaned up"); + res(logs); + }; + Services.obs.addObserver( + observer, + lazy.PlacesUtils.TOPIC_EXPIRATION_FINISHED + ); + }); + + // Force an orphans expiration step. + expiration.observe(null, "places-debug-start-expiration", 0); + return returnPromise; + }, + + /** + * Collects statistical data on the database. + * + * @return {Promise} resolves when statistics are collected. + * @resolves to an array of logs for this task. + * @rejects if we are unable to collect stats for some reason. + */ + async stats() { + let logs = []; + let placesDbPath = PathUtils.join(PathUtils.profileDir, "places.sqlite"); + let info = await IOUtils.stat(placesDbPath); + logs.push(`Places.sqlite size is ${parseInt(info.size / 1024)}KiB`); + let faviconsDbPath = PathUtils.join( + PathUtils.profileDir, + "favicons.sqlite" + ); + info = await IOUtils.stat(faviconsDbPath); + logs.push(`Favicons.sqlite size is ${parseInt(info.size / 1024)}KiB`); + + // Execute each step async. + let pragmas = [ + "user_version", + "page_size", + "cache_size", + "journal_mode", + "synchronous", + ].map(p => `pragma_${p}`); + let pragmaQuery = `SELECT * FROM ${pragmas.join(", ")}`; + await lazy.PlacesUtils.withConnectionWrapper( + "PlacesDBUtils: pragma for stats", + async db => { + let row = (await db.execute(pragmaQuery))[0]; + for (let i = 0; i != pragmas.length; i++) { + logs.push(`${pragmas[i]} is ${row.getResultByIndex(i)}`); + } + } + ).catch(() => { + logs.push("Could not set pragma for stat collection"); + }); + + // Get maximum number of unique URIs. + try { + let limitURIs = await Cc["@mozilla.org/places/expiration;1"] + .getService(Ci.nsISupports) + .wrappedJSObject.getPagesLimit(); + logs.push( + "History can store a maximum of " + limitURIs + " unique pages" + ); + } catch (ex) {} + + let query = "SELECT name FROM sqlite_master WHERE type = :type"; + let params = {}; + let _getTableCount = async tableName => { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.execute(`SELECT count(*) FROM ${tableName}`); + logs.push( + `Table ${tableName} has ${rows[0].getResultByIndex(0)} records` + ); + }; + + try { + params.type = "table"; + let db = await lazy.PlacesUtils.promiseDBConnection(); + await db.execute(query, params, r => + _getTableCount(r.getResultByIndex(0)) + ); + } catch (ex) { + throw new Error("Unable to collect stats."); + } + + let details = await PlacesDBUtils.getEntitiesStats(); + logs.push( + `Pages sequentiality: ${details.get("moz_places").sequentialityPerc}` + ); + let entities = Array.from(details.keys()).sort((a, b) => { + return details.get(a).sizePerc - details.get(b).sizePerc; + }); + for (let key of entities) { + let info = details.get(key); + logs.push( + `${key}: ${info.sizeBytes / 1024}KiB (${info.sizePerc}%), ${ + info.efficiencyPerc + }% eff.` + ); + } + + return logs; + }, + + /** + * Recalculates statistical data on the origin frecencies in the database. + * + * @return {Promise} resolves when statistics are collected. + */ + originFrecencyStats() { + return new Promise(resolve => { + lazy.PlacesUtils.history.recalculateOriginFrecencyStats(() => + resolve(["Recalculated origin frecency stats"]) + ); + }); + }, + + /** + * Collects telemetry data and reports it to Telemetry. + * + * Note: although this function isn't actually async, we keep it async to + * allow us to maintain a simple, consistent API for the tasks within this object. + * + */ + async telemetry() { + // This will be populated with one integer property for each probe result, + // using the histogram name as key. + let probeValues = {}; + + // The following array contains an ordered list of entries that are + // processed to collect telemetry data. Each entry has these properties: + // + // histogram: Name of the telemetry histogram to update. + // query: This is optional. If present, contains a database command + // that will be executed asynchronously, and whose result will + // be added to the telemetry histogram. + // callback: This is optional. If present, contains a function that must + // return the value that will be added to the telemetry + // histogram. If a query is also present, its result is passed + // as the first argument of the function. If the function + // raises an exception, no data is added to the histogram. + // + // Since all queries are executed in order by the database backend, the + // callbacks can also use the result of previous queries stored in the + // probeValues object. + let probes = [ + { + histogram: "PLACES_PAGES_COUNT", + query: "SELECT count(*) FROM moz_places", + }, + + { + histogram: "PLACES_BOOKMARKS_COUNT", + query: `SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks t ON t.id = b.parent + AND t.parent <> :tags_folder + WHERE b.type = :type_bookmark`, + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + }, + + { + histogram: "PLACES_TAGS_COUNT", + query: `SELECT count(*) FROM moz_bookmarks + WHERE parent = :tags_folder`, + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + }, + }, + + { + histogram: "PLACES_KEYWORDS_COUNT", + query: "SELECT count(*) FROM moz_keywords", + }, + + { + histogram: "PLACES_SORTED_BOOKMARKS_PERC", + query: `SELECT IFNULL(ROUND(( + SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + JOIN moz_bookmarks g ON g.id = p.parent + WHERE g.guid <> :root_guid + AND g.guid <> :tags_guid + AND b.type = :type_bookmark + ) * 100 / ( + SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + JOIN moz_bookmarks g ON g.id = p.parent + AND g.guid <> :tags_guid + WHERE b.type = :type_bookmark + )), 0)`, + params: { + root_guid: lazy.PlacesUtils.bookmarks.rootGuid, + tags_guid: lazy.PlacesUtils.bookmarks.tagsGuid, + type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + }, + + { + histogram: "PLACES_TAGGED_BOOKMARKS_PERC", + query: `SELECT IFNULL(ROUND(( + SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks t ON t.id = b.parent + AND t.parent = :tags_folder + ) * 100 / ( + SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks t ON t.id = b.parent + AND t.parent <> :tags_folder + WHERE b.type = :type_bookmark + )), 0)`, + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + }, + + { + histogram: "PLACES_DATABASE_FILESIZE_MB", + async callback() { + let placesDbPath = PathUtils.join( + PathUtils.profileDir, + "places.sqlite" + ); + let info = await IOUtils.stat(placesDbPath); + return parseInt(info.size / BYTES_PER_MEBIBYTE); + }, + }, + + { + histogram: "PLACES_DATABASE_FAVICONS_FILESIZE_MB", + async callback() { + let faviconsDbPath = PathUtils.join( + PathUtils.profileDir, + "favicons.sqlite" + ); + let info = await IOUtils.stat(faviconsDbPath); + return parseInt(info.size / BYTES_PER_MEBIBYTE); + }, + }, + + { + histogram: "PLACES_ANNOS_PAGES_COUNT", + query: "SELECT count(*) FROM moz_annos", + }, + + { + histogram: "PLACES_MAINTENANCE_DAYSFROMLAST", + callback() { + try { + let lastMaintenance = Services.prefs.getIntPref( + "places.database.lastMaintenance" + ); + let nowSeconds = parseInt(Date.now() / 1000); + return parseInt((nowSeconds - lastMaintenance) / 86400); + } catch (ex) { + return 60; + } + }, + }, + { + scalar: "places.pages_need_frecency_recalculation", + query: "SELECT count(*) FROM moz_places WHERE recalc_frecency = 1", + }, + { + scalar: "places.previousday_visits", + query: `SELECT COUNT(*) from moz_places + WHERE hidden=0 AND last_visit_date < (strftime('%s', 'now', 'start of day') * 1000000) + AND last_visit_date > (strftime('%s', 'now', 'start of day', '-1 day') * 1000000) + AND last_visit_date IS NOT NULL;`, + }, + ]; + + for (let probe of probes) { + let val; + if ("query" in probe) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + val = ( + await db.execute(probe.query, probe.params || {}) + )[0].getResultByIndex(0); + } + // Report the result of the probe through Telemetry. + // The resulting promise cannot reject. + if ("callback" in probe) { + val = await probe.callback(val); + } + probeValues[probe.histogram || probe.scalar] = val; + if (probe.histogram) { + Services.telemetry.getHistogramById(probe.histogram).add(val); + } else if (probe.scalar) { + Services.telemetry.scalarSet(probe.scalar, val); + } else { + throw new Error("Unknwon telemetry probe type"); + } + } + }, + + /** + * Remove old and useless places.sqlite.corrupt files. + * + * @resolves to an array of logs for this task. + * + */ + async removeOldCorruptDBs() { + let logs = []; + logs.push( + "> Cleanup profile from places.sqlite.corrupt files older than " + + CORRUPT_DB_RETAIN_DAYS + + " days." + ); + let re = /places\.sqlite(-\d)?\.corrupt$/; + let currentTime = Date.now(); + let children = await IOUtils.getChildren(PathUtils.profileDir); + try { + for (let entry of children) { + let fileInfo = await IOUtils.stat(entry); + let lastModificationDate; + if (fileInfo.type == "regular" && re.test(entry)) { + lastModificationDate = fileInfo.lastModified; + try { + // Convert milliseconds to days. + let days = Math.ceil( + (currentTime - lastModificationDate) / MS_PER_DAY + ); + if (days >= CORRUPT_DB_RETAIN_DAYS || days < 0) { + await IOUtils.remove(entry); + } + } catch (error) { + logs.push("Could not remove file: " + entry, error); + } + } + } + } catch (error) { + logs.push("removeOldCorruptDBs failed", error); + } + return logs; + }, + + /** + * Gets detailed statistics about database entities like tables and indices. + * @returns {Map} a Map by table name, containing an object with the following + * properties: + * - efficiencyPerc: percentage filling of pages, an high + * efficiency means most pages are filled up almost completely. + * This value is not particularly useful with a low number of + * pages. + * - sizeBytes: size of the entity in bytes + * - pages: number of pages of the entity + * - sizePerc: percentage of the total database size + * - sequentialityPerc: percentage of sequential pages, this is + * a global value of the database, thus it's the same for every + * entity, and it can be used to evaluate fragmentation and the + * need for vacuum. + */ + async getEntitiesStats() { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.execute(` + /* do not warn (bug no): no need for index */ + SELECT name, + round((pgsize - unused) * 100.0 / pgsize, 1) as efficiency_perc, + pgsize as size_bytes, pageno as pages, + round(pgsize * 100.0 / (SELECT sum(pgsize) FROM dbstat WHERE aggregate = TRUE), 1) as size_perc, + round(( + WITH s(row, pageno) AS ( + SELECT row_number() OVER (ORDER BY path), pageno FROM dbstat ORDER BY path + ) + SELECT sum(s1.pageno+1==s2.pageno)*100.0/count(*) + FROM s AS s1, s AS s2 + WHERE s1.row+1=s2.row + ), 1) AS sequentiality_perc + FROM dbstat + WHERE aggregate = TRUE + `); + let entitiesByName = new Map(); + for (let row of rows) { + let details = { + efficiencyPerc: row.getResultByName("efficiency_perc"), + pages: row.getResultByName("pages"), + sizeBytes: row.getResultByName("size_bytes"), + sizePerc: row.getResultByName("size_perc"), + sequentialityPerc: row.getResultByName("sequentiality_perc"), + }; + entitiesByName.set(row.getResultByName("name"), details); + } + return entitiesByName; + }, + + /** + * Gets detailed statistics about database entities and their respective row + * counts. + * @returns {Array} An array that augments each object returned by + * {@link getEntitiesStats} with the following extra properties: + * - entity: name of the entity + * - count: row count of the entity + */ + async getEntitiesStatsAndCounts() { + let stats = await PlacesDBUtils.getEntitiesStats(); + let data = []; + let db = await lazy.PlacesUtils.promiseDBConnection(); + for (let [entity, value] of stats) { + let count = "-"; + try { + if ( + entity.startsWith("moz_") && + !entity.endsWith("index") && + entity != "moz_places_visitcount" /* bug in index name */ + ) { + count = ( + await db.execute(`SELECT count(*) FROM ${entity}`) + )[0].getResultByIndex(0); + } + } catch (ex) { + console.error(ex); + } + data.push(Object.assign(value, { entity, count })); + } + return data; + }, + + /** + * Runs a list of tasks, returning a Map when done. + * + * @param tasks + * Array of tasks to be executed, in form of pointers to methods in + * this module. + * @return {Promise} + * A promise that resolves with a Map[taskName(String) -> Object]. + * The Object has the following properties: + * - succeeded: boolean + * - logs: an array of strings containing the messages logged by the task + */ + async runTasks(tasks) { + if (!this._registeredShutdownObserver) { + this._registeredShutdownObserver = true; + lazy.PlacesUtils.registerShutdownFunction(() => { + this._isShuttingDown = true; + }); + } + PlacesDBUtils._clearTaskQueue = false; + let tasksMap = new Map(); + for (let task of tasks) { + if (PlacesDBUtils._isShuttingDown) { + tasksMap.set(task.name, { + succeeded: false, + logs: ["Shutting down, will not schedule the task."], + }); + continue; + } + + if (PlacesDBUtils._clearTaskQueue) { + tasksMap.set(task.name, { + succeeded: false, + logs: ["The task queue was cleared by an error in another task."], + }); + continue; + } + + let result = await task() + .then((logs = [`${task.name} complete`]) => ({ succeeded: true, logs })) + .catch(err => ({ succeeded: false, logs: [err.message] })); + tasksMap.set(task.name, result); + } + return tasksMap; + }, +}; + +async function integrity(dbName) { + async function check(db) { + let row; + await db.execute("PRAGMA integrity_check", null, (r, cancel) => { + row = r; + cancel(); + }); + return row.getResultByIndex(0) === "ok"; + } + + // Create a new connection for this check, so we can operate independently + // from a broken Places service. + // openConnection returns an exception with .result == Cr.NS_ERROR_FILE_CORRUPTED, + // we should do the same everywhere we want maintenance to try replacing the + // database on next startup. + let path = PathUtils.join(PathUtils.profileDir, dbName); + let db = await lazy.Sqlite.openConnection({ path }); + try { + if (await check(db)) { + return; + } + + // We stopped due to an integrity corruption, try to fix it if possible. + // First, try to reindex, this often fixes simple indices problems. + try { + await db.execute("REINDEX"); + } catch (ex) { + throw new Components.Exception( + "Impossible to reindex database", + Cr.NS_ERROR_FILE_CORRUPTED + ); + } + + // Check again. + if (!(await check(db))) { + throw new Components.Exception( + "The database is still corrupt", + Cr.NS_ERROR_FILE_CORRUPTED + ); + } + } finally { + await db.close(); + } +} + +export function PlacesDBUtilsIdleMaintenance() {} + +PlacesDBUtilsIdleMaintenance.prototype = { + observe(subject, topic, data) { + switch (topic) { + case "idle-daily": + // Once a week run places.sqlite maintenance tasks. + let lastMaintenance = Services.prefs.getIntPref( + "places.database.lastMaintenance", + 0 + ); + let nowSeconds = parseInt(Date.now() / 1000); + if (lastMaintenance < nowSeconds - MAINTENANCE_INTERVAL_SECONDS) { + PlacesDBUtils.maintenanceOnIdle(); + } + break; + default: + throw new Error("Trying to handle an unknown category."); + } + }, + classID: Components.ID("d38926e0-29c1-11eb-8588-0800200c9a66"), + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), +}; diff --git a/toolkit/components/places/PlacesExpiration.sys.mjs b/toolkit/components/places/PlacesExpiration.sys.mjs new file mode 100644 index 0000000000..4db494d63e --- /dev/null +++ b/toolkit/components/places/PlacesExpiration.sys.mjs @@ -0,0 +1,957 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=2 ts=2 sts=2 expandtab + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This component handles history and orphans expiration. + * Expiration runs: + * - At idle, but just once, we stop any other kind of expiration during idle + * to preserve batteries in portable devices. + * - At shutdown, only if the database is dirty, we should still avoid to + * expire too heavily on shutdown. + * - On a repeating timer we expire in small chunks. + * + * Expiration algorithm will adapt itself based on: + * - Memory size of the device. + * - Status of the database (clean or dirty). + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +// Last expiration step should run before the final sync. +const TOPIC_DEBUG_START_EXPIRATION = "places-debug-start-expiration"; +const TOPIC_IDLE_BEGIN = "idle"; +const TOPIC_IDLE_END = "active"; +const TOPIC_IDLE_DAILY = "idle-daily"; +const TOPIC_TESTING_MODE = "testing-mode"; +const TOPIC_TEST_INTERVAL_CHANGED = "test-interval-changed"; + +// This value determines which systems we consider to have limited memory. +// This is used to protect against large database sizes on those systems. +const DATABASE_MEMORY_CONSTRAINED_THRESHOLD = 2147483648; // 2 GiB + +// This value determines which systems we consider to have limited disk space. +// This is used to protect against large database sizes on those systems. +const DATABASE_DISK_CONSTRAINED_THRESHOLD = 5368709120; // 5 GiB + +// Maximum size of the optimal database. High-end hardware has plenty of +// memory and disk space, but performances don't grow linearly. +const DATABASE_MAX_SIZE = 78643200; // 75 MiB +// If the physical memory size is bogus, fallback to this. +const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 MiB +// If the disk available space is bogus, fallback to this. +const DISKSIZE_FALLBACK_BYTES = 268435456; // 256 MiB + +// Max number of entries to expire at each expiration step. +// This value is globally used for different kind of data we expire, can be +// tweaked based on data type. See below in getQuery. +const EXPIRE_LIMIT_PER_STEP = 6; +// When we run a large expiration step, the above limit is multiplied by this. +const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER = 10; + +// When history is clean or dirty enough we will adapt the expiration algorithm +// to be more lazy or more aggressive. +// This is done acting on the interval between expiration steps and the number +// of expirable items. +// 1. Clean history: +// We expire at (default interval * EXPIRE_AGGRESSIVITY_MULTIPLIER) the +// default number of entries. +// 2. Dirty history: +// We expire at the default interval, but a greater number of entries +// (default number of entries * EXPIRE_AGGRESSIVITY_MULTIPLIER). +const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3; + +// This is the average size in bytes of an URI entry in the database. +// Magic numbers are determined through analysis of the distribution of a ratio +// between number of unique URIs and database size among our users. +// Used as a fall back value when it's not possible to calculate the real value. +const URIENTRY_AVG_SIZE = 700; + +// Seconds of idle time before starting a larger expiration step. +// Notice during idle we stop the expiration timer since we don't want to hurt +// stand-by or mobile devices batteries. +const IDLE_TIMEOUT_SECONDS = 5 * 60; + +// If the number of pages over history limit is greater than this threshold, +// expiration will be more aggressive, to bring back history to a saner size. +const OVERLIMIT_PAGES_THRESHOLD = 1000; + +// Milliseconds in a day. +const MSECS_PER_DAY = 86400000; + +// When we expire we can use these limits: +// - SMALL for usual partial expirations, will expire a small chunk. +// - LARGE for idle or shutdown expirations, will expire a large chunk. +// - UNLIMITED will expire all the orphans. +// - DEBUG will use a known limit, passed along with the debug notification. +const LIMIT = { + SMALL: 0, + LARGE: 1, + UNLIMITED: 2, + DEBUG: 3, +}; + +// Represents the status of history database. +const STATUS = { + CLEAN: 0, + DIRTY: 1, + UNKNOWN: 2, +}; + +// Represents actions on which a query will run. +const ACTION = { + TIMED: 1 << 0, // happens every this.intervalSeconds + TIMED_OVERLIMIT: 1 << 1, // like TIMED but only when history is over limits + SHUTDOWN_DIRTY: 1 << 2, // happens at shutdown for DIRTY state + IDLE_DIRTY: 1 << 3, // happens on idle for DIRTY state + IDLE_DAILY: 1 << 4, // happens once a day on idle + DEBUG: 1 << 5, // happens on TOPIC_DEBUG_START_EXPIRATION +}; + +// The queries we use to expire. +const EXPIRATION_QUERIES = { + // Some visits can be expired more often than others, cause they are less + // useful to the user and can pollute awesomebar results: + // 1. urls over 255 chars having only one visit + // 2. downloads + // 3. non-typed hidden single-visit urls + // We never expire redirect targets, because they are currently necessary to + // recognize redirect sources (see Bug 468710 for better options). + QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE: { + sql: `INSERT INTO expiration_notify (v_id, url, guid, visit_date, reason) + WITH visits AS ( + SELECT v.id, url, guid, visit_type, visit_date, visit_count, hidden, typed + FROM moz_historyvisits v + JOIN moz_places h ON h.id = v.place_id + WHERE visit_date < strftime('%s','now','localtime','start of day','-90 days','utc') * 1000000 + AND visit_type NOT IN (5,6) + ) + SELECT id, url, guid, visit_date, "exotic" + FROM visits + WHERE (hidden = 1 AND typed = 0 AND visit_count <= 1) OR visit_type = 7 + UNION ALL + SELECT id, url, guid, visit_date, "exotic" + FROM visits + WHERE visit_count = 1 AND LENGTH(url) > 255 + ORDER BY visit_date ASC + LIMIT :limit_visits`, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Finds visits to be expired when history is over the unique pages limit, + // otherwise will return nothing. + // This explicitly excludes any visits added in the last 7 days, to protect + // users with thousands of bookmarks from constantly losing history. + QUERY_FIND_VISITS_TO_EXPIRE: { + sql: `INSERT INTO expiration_notify + (v_id, url, guid, visit_date, expected_results) + SELECT v.id, h.url, h.guid, v.visit_date, :limit_visits + FROM moz_historyvisits v + JOIN moz_places h ON h.id = v.place_id + WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris + AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 + ORDER BY v.visit_date ASC + LIMIT :limit_visits`, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Removes the previously found visits. + QUERY_EXPIRE_VISITS: { + sql: `DELETE FROM moz_historyvisits WHERE id IN ( + SELECT v_id FROM expiration_notify WHERE v_id NOTNULL + )`, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Finds orphan URIs in the database. + // Notice we won't notify single removed URIs on History.clear(), so we don't + // run this query in such a case, but just delete URIs. + // This could run in the middle of adding a visit or bookmark to a new page. + // In such a case since it is async, could end up expiring the orphan page + // before it actually gets the new visit or bookmark. + // Thus, since new pages get frecency -1, we filter on that. + QUERY_FIND_URIS_TO_EXPIRE: { + sql: `INSERT INTO expiration_notify (p_id, url, guid, visit_date) + SELECT h.id, h.url, h.guid, h.last_visit_date + FROM moz_places h + LEFT JOIN moz_historyvisits v ON h.id = v.place_id + WHERE h.last_visit_date IS NULL + AND h.foreign_count = 0 + AND v.id IS NULL + AND frecency <> -1 + LIMIT :limit_uris`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire found URIs from the database. + QUERY_EXPIRE_URIS: { + sql: `DELETE FROM moz_places WHERE id IN ( + SELECT p_id FROM expiration_notify WHERE p_id NOTNULL + ) AND foreign_count = 0 AND last_visit_date ISNULL`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Hosts accumulated during the places delete are updated through a trigger + // (see nsPlacesTriggers.h). + QUERY_UPDATE_HOSTS: { + sql: `DELETE FROM moz_updateoriginsdelete_temp`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire from favicons any page that has only relations older than 180 days, + // if the page is not bookmarked, and we have a root icon that can be used + // as a placeholder until the page is visited again. + // The moz_pages_to_icons entries are removed by the table's FOREIGN KEY, + // while orphan icons are removed by the following queries. + QUERY_EXPIRE_OLD_FAVICONS: { + sql: ` + DELETE FROM moz_pages_w_icons WHERE id IN ( + WITH pages_with_old_relations (page_id, page_url_hash) AS ( + SELECT page_id, page_url_hash + FROM moz_icons_to_pages ip + JOIN moz_pages_w_icons p ON p.id = page_id + GROUP BY page_id + HAVING max(expire_ms) < strftime('%s','now','localtime','start of day','-180 days','utc') * 1000 + ) + SELECT page_id + FROM pages_with_old_relations + JOIN moz_places h ON h.url_hash = page_url_hash + JOIN moz_origins o ON h.origin_id = o.id + WHERE foreign_count = 0 + AND EXISTS ( + SELECT 1 FROM moz_icons + WHERE root = 1 + AND fixed_icon_url_hash = hash(fixup_url(o.host) || '/favicon.ico') + ) + LIMIT 100 + )`, + actions: ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG, + }, + + // Expire orphan pages from the icons database. + QUERY_EXPIRE_FAVICONS_PAGES: { + sql: `DELETE FROM moz_pages_w_icons + WHERE page_url_hash NOT IN ( + SELECT url_hash FROM moz_places + ) OR NOT EXISTS ( + SELECT 1 FROM moz_icons_to_pages WHERE page_id = moz_pages_w_icons.id + )`, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire orphan icons from the database. + QUERY_EXPIRE_FAVICONS: { + sql: `DELETE FROM moz_icons WHERE id IN ( + SELECT id FROM moz_icons WHERE root = 0 + EXCEPT + SELECT icon_id FROM moz_icons_to_pages + )`, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire orphan page annotations from the database. + QUERY_EXPIRE_ANNOS: { + sql: `DELETE FROM moz_annos WHERE id in ( + SELECT a.id FROM moz_annos a + LEFT JOIN moz_places h ON a.place_id = h.id + WHERE h.id IS NULL + LIMIT :limit_annos + )`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire orphan inputhistory. + QUERY_EXPIRE_INPUTHISTORY: { + sql: `DELETE FROM moz_inputhistory + WHERE place_id IN (SELECT p_id FROM expiration_notify) + AND place_id IN ( + SELECT i.place_id FROM moz_inputhistory i + LEFT JOIN moz_places h ON h.id = i.place_id + WHERE h.id IS NULL + LIMIT :limit_inputhistory + )`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Select entries for notifications. + // If p_id is set whole_entry = 1, then we have expired the full page. + // Either p_id or v_id are always set. + QUERY_SELECT_NOTIFICATIONS: { + sql: `/* do not warn (bug no): temp table has no index */ + SELECT url, guid, MAX(visit_date) AS visit_date, + MAX(IFNULL(MIN(p_id, 1), MIN(v_id, 0))) AS whole_entry, + MAX(expected_results) AS expected_results, + (SELECT MAX(visit_date) FROM expiration_notify + WHERE reason = "expired" AND url = n.url AND p_id ISNULL + ) AS most_recent_expired_visit + FROM expiration_notify n + GROUP BY url`, + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Empty the notifications table. + QUERY_DELETE_NOTIFICATIONS: { + sql: "DELETE FROM expiration_notify", + actions: + ACTION.TIMED | + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, + + // Expire interactions older than N days. + QUERY_EXPIRE_INTERACTIONS: { + sql: `DELETE FROM moz_places_metadata + WHERE id IN ( + SELECT id FROM moz_places_metadata + WHERE updated_at < strftime('%s','now','localtime','-' || :days_interactions || ' day','start of day','utc') * 1000 + ORDER BY updated_at ASC + LIMIT :limit_interactions + )`, + get disabled() { + return !Services.prefs.getBoolPref( + "browser.places.interactions.enabled", + false + ); + }, + actions: + ACTION.TIMED_OVERLIMIT | + ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | + ACTION.DEBUG, + }, +}; + +export function nsPlacesExpiration() { + // Allows other components to easily access getPagesLimit. + this.wrappedJSObject = this; + + XPCOMUtils.defineLazyServiceGetter( + this, + "_idle", + "@mozilla.org/widget/useridleservice;1", + "nsIUserIdleService" + ); + + // Max number of unique URIs to retain in history. + // Notice this is a lazy limit. This means we will start to expire if we will + // go over it, but we won't ensure that we will stop exactly when we reach it, + // instead we will stop after the next expiration step that will bring us + // below it. + // If this preference does not exist or has a negative value, we will + // calculate a limit based on current hardware. + XPCOMUtils.defineLazyPreferenceGetter( + this, + "maxPages", + "places.history.expiration.max_pages", + -1, + () => { + // Clear the cache. + dump("max_pages changing\n"); + this._pagesLimit = null; + } + ); + + // Seconds between each expiration step. + XPCOMUtils.defineLazyPreferenceGetter( + this, + "intervalSeconds", + "places.history.expiration.interval_seconds", + 3 * 60, // 3 minutes + () => { + // Renew the timer with the new interval value. + this._newTimer(); + }, + v => (v > 0 ? v : 3 * 60) // Accept only positive values. + ); + + this._dbInitializedPromise = lazy.PlacesUtils.withConnectionWrapper( + "PlacesExpiration.jsm: setup", + async db => { + await db.execute( + `CREATE TEMP TABLE expiration_notify ( + id INTEGER PRIMARY KEY, + v_id INTEGER, + p_id INTEGER, + url TEXT NOT NULL, + guid TEXT NOT NULL, + visit_date INTEGER, + expected_results INTEGER NOT NULL DEFAULT 0, + reason TEXT NOT NULL DEFAULT "expired" + )` + ); + } + ) + .then(() => { + // Start the expiration timer. + this._newTimer(); + // Expire daily on idle. + Services.obs.addObserver(this, TOPIC_IDLE_DAILY, true); + }) + .catch(console.error); + + // Block shutdown. + let shutdownClient = + lazy.PlacesUtils.history.connectionShutdownClient.jsclient; + shutdownClient.addBlocker("Places Expiration: shutdown", () => { + if (this._shuttingDown) { + return; + } + this._shuttingDown = true; + this.expireOnIdle = false; + if (this._timer) { + this._timer.cancel(); + this._timer = null; + } + // If the database is dirty, we want to expire some entries, to speed up + // the expiration process. + if (this.status == STATUS.DIRTY) { + this._expire(ACTION.SHUTDOWN_DIRTY, LIMIT.LARGE).catch(console.error); + } + }); +} + +nsPlacesExpiration.prototype = { + observe(aSubject, aTopic, aData) { + if (this._shuttingDown) { + return; + } + + if (aTopic == TOPIC_DEBUG_START_EXPIRATION) { + // The passed-in limit is the maximum number of visits to expire when + // history is over capacity. Mind to correctly handle the NaN value. + let limit = parseInt(aData); + if (limit == -1) { + // Everything should be expired without any limit. If history is over + // capacity then all existing visits will be expired. + // Should only be used in tests, since may cause dataloss. + this._expire(ACTION.DEBUG, LIMIT.UNLIMITED).catch(console.error); + } else if (limit > 0) { + // The number of expired visits is limited by this amount. It may be + // used for testing purposes, like checking that limited queries work. + this._debugLimit = limit; + this._expire(ACTION.DEBUG, LIMIT.DEBUG).catch(console.error); + } else { + // Any other value is intended as a 0 limit, that means no visits + // will be expired. Even if this doesn't touch visits, it will remove + // any orphan pages, icons, annotations and similar from the database, + // so it may be used for cleanup purposes. + this._debugLimit = -1; + this._expire(ACTION.DEBUG, LIMIT.DEBUG).catch(console.error); + } + } else if (aTopic == TOPIC_IDLE_BEGIN) { + // Stop the expiration timer. We don't want to keep up expiring on idle + // to preserve batteries on mobile devices and avoid killing stand-by. + if (this._timer) { + this._timer.cancel(); + this._timer = null; + } + if (this.expireOnIdle) { + this._expire(ACTION.IDLE_DIRTY, LIMIT.LARGE).catch(console.error); + } + } else if (aTopic == TOPIC_IDLE_END) { + // Restart the expiration timer. + if (!this._timer) { + this._newTimer(); + } + } else if (aTopic == TOPIC_IDLE_DAILY) { + this._expire(ACTION.IDLE_DAILY, LIMIT.LARGE).catch(console.error); + } else if (aTopic == TOPIC_TESTING_MODE) { + this._testingMode = true; + } else if (aTopic == lazy.PlacesUtils.TOPIC_INIT_COMPLETE) { + const placesObserver = new PlacesWeakCallbackWrapper( + // History status is clean after a clear history. + () => { + this.status = STATUS.CLEAN; + } + ); + PlacesObservers.addListener(["history-cleared"], placesObserver); + } + }, + + // nsINamed + + name: "nsPlacesExpiration", + + // nsITimerCallback + + notify() { + // Run at the first idle, or after 5 minutes, whatever comes first. + Services.tm.idleDispatchToMainThread(async () => { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let pagesCount = ( + await db.executeCached("SELECT count(*) AS count FROM moz_places") + )[0].getResultByName("count"); + let pagesLimit = await this.getPagesLimit(); + // Check if we are over history capacity, if so visits must be expired. + let overLimitPages = pagesCount - pagesLimit; + let action = overLimitPages > 0 ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED; + // Adapt expiration aggressivity to the number of pages over the limit. + let limit = + overLimitPages > OVERLIMIT_PAGES_THRESHOLD ? LIMIT.LARGE : LIMIT.SMALL; + this._expire(action, limit).catch(console.error); + }, 300000); + }, + + _handleQueryResultAndAddNotification(row, notifications) { + // We don't want to notify after shutdown. + if (this._shuttingDown) { + return; + } + + // expected_results is set to the number of expected visits by + // QUERY_FIND_VISITS_TO_EXPIRE. We decrease that counter for each found + // visit and if it reaches zero we mark the database as dirty, since all + // the expected visits were expired, so it's likely the next run will + // find more. + let expectedResults = row.getResultByName("expected_results"); + if (expectedResults > 0) { + if (!("_expectedResultsCount" in this)) { + this._expectedResultsCount = expectedResults; + } + if (this._expectedResultsCount > 0) { + this._expectedResultsCount--; + } + } + + let uri = Services.io.newURI(row.getResultByName("url")); + let guid = row.getResultByName("guid"); + let visitDate = row.getResultByName("visit_date"); + let wholeEntry = row.getResultByName("whole_entry"); + let mostRecentExpiredVisit = row.getResultByName( + "most_recent_expired_visit" + ); + + if (mostRecentExpiredVisit) { + let days = parseInt( + (Date.now() - mostRecentExpiredVisit / 1000) / MSECS_PER_DAY + ); + if (!this._mostRecentExpiredVisitDays) { + this._mostRecentExpiredVisitDays = days; + } else if (days < this._mostRecentExpiredVisitDays) { + this._mostRecentExpiredVisitDays = days; + } + } + + // Dispatch expiration notifications to history. + const isRemovedFromStore = !!wholeEntry; + notifications.push( + new PlacesVisitRemoved({ + url: uri.spec, + pageGuid: guid, + reason: PlacesVisitRemoved.REASON_EXPIRED, + isRemovedFromStore, + isPartialVisistsRemoval: !isRemovedFromStore && visitDate > 0, + }) + ); + }, + + _shuttingDown: false, + + _status: STATUS.UNKNOWN, + set status(aNewStatus) { + if (aNewStatus != this._status) { + // If status changes we should restart the timer. + this._status = aNewStatus; + this._newTimer(); + // If needed add/remove the cleanup step on idle. We want to expire on + // idle only if history is dirty, to preserve mobile devices batteries. + this.expireOnIdle = aNewStatus == STATUS.DIRTY; + } + }, + get status() { + return this._status; + }, + + async getPagesLimit() { + if (this._pagesLimit != null) { + return this._pagesLimit; + } + if (this.maxPages >= 0) { + return (this._pagesLimit = this.maxPages); + } + + // The user didn't specify a custom limit, so we calculate the number of + // unique places that may fit an optimal database size on this hardware. + // Oldest pages over this threshold will be expired. + let memSizeBytes = MEMSIZE_FALLBACK_BYTES; + try { + // Limit the size on systems with small memory. + memSizeBytes = Services.sysinfo.getProperty("memsize"); + } catch (ex) {} + if (memSizeBytes <= 0) { + memSizeBytes = MEMSIZE_FALLBACK_BYTES; + } + + let diskAvailableBytes = DISKSIZE_FALLBACK_BYTES; + try { + // Protect against a full disk or tiny quota. + diskAvailableBytes = + lazy.PlacesUtils.history.DBConnection.databaseFile.QueryInterface( + Ci.nsIFile + ).diskSpaceAvailable; + } catch (ex) {} + if (diskAvailableBytes <= 0) { + diskAvailableBytes = DISKSIZE_FALLBACK_BYTES; + } + + const isMemoryConstrained = + memSizeBytes < DATABASE_MEMORY_CONSTRAINED_THRESHOLD; + const isDiskConstrained = + diskAvailableBytes < DATABASE_DISK_CONSTRAINED_THRESHOLD; + + let optimalDatabaseSize = DATABASE_MAX_SIZE; + if (isMemoryConstrained || isDiskConstrained) { + // This size is used to protect against a large database size + // on disks with limited space or on systems with small memory + optimalDatabaseSize /= 2; + } + + // Calculate avg size of a URI in the database. + let db; + try { + db = await lazy.PlacesUtils.promiseDBConnection(); + if (db) { + let row = ( + await db.execute(`SELECT * FROM pragma_page_size(), + pragma_page_count(), + pragma_freelist_count(), + (SELECT count(*) FROM moz_places)`) + )[0]; + let pageSize = row.getResultByIndex(0); + let pageCount = row.getResultByIndex(1); + let freelistCount = row.getResultByIndex(2); + let uriCount = row.getResultByIndex(3); + let dbSize = (pageCount - freelistCount) * pageSize; + let avgURISize = Math.ceil(dbSize / uriCount); + // For new profiles this value may be too large, due to the Sqlite header, + // or Infinity when there are no pages. Thus we must limit it. + if (avgURISize > URIENTRY_AVG_SIZE * 3) { + avgURISize = URIENTRY_AVG_SIZE; + } + return (this._pagesLimit = Math.ceil(optimalDatabaseSize / avgURISize)); + } + } catch (ex) { + // We may have been initialized late in the shutdown process, maybe + // by a call to clear history on shutdown. + // If we're unable to get a connection clone, we'll just proceed with a + // large default value, it should not be critical at this point in the + // application life-cycle. + } + return (this._pagesLimit = 100000); + }, + + _isIdleObserver: false, + _expireOnIdle: false, + set expireOnIdle(aExpireOnIdle) { + // Observe idle regardless aExpireOnIdle, since we always want to stop + // timed expiration on idle, to preserve mobile battery life. + if (!this._isIdleObserver && !this._shuttingDown) { + this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS); + this._isIdleObserver = true; + } else if (this._isIdleObserver && this._shuttingDown) { + this._idle.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS); + this._isIdleObserver = false; + } + + // If running a debug expiration we need full control of what happens + // but idle cleanup could activate in the middle, since tinderboxes are + // permanently idle. That would cause unexpected oranges, so disable it. + if (this._debugLimit !== undefined) { + this._expireOnIdle = false; + } else { + this._expireOnIdle = aExpireOnIdle; + } + }, + get expireOnIdle() { + return this._expireOnIdle; + }, + + // Number of expiration steps needed to reach a CLEAN status. + _telemetrySteps: 1, + + /** + * Expires visits and orphans. + * + * @param aAction + * The ACTION we are expiring for. See the ACTION const for values. + * @param aLimit + * Whether to use small, large or no limits when expiring. See the + * LIMIT const for values. + */ + async _expire(aAction, aLimit) { + // Don't try to further expire after shutdown. + if (this._shuttingDown && aAction != ACTION.SHUTDOWN_DIRTY) { + return; + } + await this._dbInitializedPromise; + + try { + let notifications = []; + await lazy.PlacesUtils.withConnectionWrapper( + "PlacesExpiration.jsm: expire", + async db => { + await db.executeTransaction(async () => { + for (let queryType in EXPIRATION_QUERIES) { + let query = EXPIRATION_QUERIES[queryType]; + if (query.actions & aAction && !query.disabled) { + let params = await this._getQueryParams( + queryType, + aLimit, + aAction + ); + await db.executeCached(query.sql, params, row => { + this._handleQueryResultAndAddNotification(row, notifications); + }); + } + } + }); + } + ); + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } + } catch (ex) { + console.error(ex); + return; + } + + if (this._mostRecentExpiredVisitDays) { + try { + Services.telemetry + .getHistogramById("PLACES_MOST_RECENT_EXPIRED_VISIT_DAYS") + .add(this._mostRecentExpiredVisitDays); + } catch (ex) { + console.error("Unable to report telemetry."); + } finally { + delete this._mostRecentExpiredVisitDays; + } + } + + if ("_expectedResultsCount" in this) { + // Adapt the aggressivity of steps based on the status of history. + // A dirty history will return all the entries we are expecting bringing + // our countdown to zero, while a clean one will not. + let oldStatus = this.status; + this.status = + this._expectedResultsCount == 0 ? STATUS.DIRTY : STATUS.CLEAN; + + // Collect or send telemetry data. + if (this.status == STATUS.DIRTY) { + this._telemetrySteps++; + } else { + // Avoid reporting the common cases where the database is clean, or + // a single step is needed. + if (oldStatus == STATUS.DIRTY) { + try { + Services.telemetry + .getHistogramById("PLACES_EXPIRATION_STEPS_TO_CLEAN2") + .add(this._telemetrySteps); + } catch (ex) { + console.error("Unable to report telemetry."); + } + } + this._telemetrySteps = 1; + } + + delete this._expectedResultsCount; + } + + // Dispatch a notification that expiration has finished. + Services.obs.notifyObservers( + null, + lazy.PlacesUtils.TOPIC_EXPIRATION_FINISHED + ); + }, + + /** + * Generate a query used for expiration. + * + * @param aQueryType + * Type of the query. + * @param aLimit + * Whether to use small, large or no limits when expiring. See the + * LIMIT const for values. + * @param aAction + * Current action causing the expiration. See the ACTION const. + */ + async _getQueryParams(aQueryType, aLimit, aAction) { + let baseLimit; + switch (aLimit) { + case LIMIT.UNLIMITED: + baseLimit = -1; + break; + case LIMIT.SMALL: + baseLimit = EXPIRE_LIMIT_PER_STEP; + break; + case LIMIT.LARGE: + baseLimit = + EXPIRE_LIMIT_PER_STEP * EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER; + break; + case LIMIT.DEBUG: + baseLimit = this._debugLimit; + break; + } + if ( + this.status == STATUS.DIRTY && + aAction != ACTION.DEBUG && + baseLimit > 0 + ) { + baseLimit *= EXPIRE_AGGRESSIVITY_MULTIPLIER; + } + + switch (aQueryType) { + case "QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE": + return { + // Avoid expiring all visits in case of an unlimited debug expiration, + // just remove orphans instead. + limit_visits: + aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit, + }; + case "QUERY_FIND_VISITS_TO_EXPIRE": + return { + max_uris: await this.getPagesLimit(), + // Avoid expiring all visits in case of an unlimited debug expiration, + // just remove orphans instead. + limit_visits: + aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit, + }; + case "QUERY_FIND_URIS_TO_EXPIRE": + return { + limit_uris: baseLimit, + }; + case "QUERY_EXPIRE_ANNOS": + return { + // Each page may have multiple annos. + limit_annos: baseLimit * EXPIRE_AGGRESSIVITY_MULTIPLIER, + }; + case "QUERY_EXPIRE_INPUTHISTORY": + return { + limit_inputhistory: baseLimit, + }; + case "QUERY_EXPIRE_INTERACTIONS": + return { + days_interactions: Services.prefs.getIntPref( + "browser.places.interactions.expireDays", + 60 + ), + limit_interactions: + aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit, + }; + } + return undefined; + }, + + /** + * Creates a new timer based on this.intervalSeconds. + * + * @return a REPEATING_SLACK nsITimer that runs every this.intervalSeconds. + */ + _newTimer() { + if (this._timer) { + this._timer.cancel(); + } + if (this._shuttingDown) { + return undefined; + } + + if (!this._isIdleObserver) { + this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS); + this._isIdleObserver = true; + } + + let interval = + this.status != STATUS.DIRTY + ? this.intervalSeconds * EXPIRE_AGGRESSIVITY_MULTIPLIER + : this.intervalSeconds; + + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback( + this, + interval * 1000, + Ci.nsITimer.TYPE_REPEATING_SLACK_LOW_PRIORITY + ); + if (this._testingMode) { + Services.obs.notifyObservers(null, TOPIC_TEST_INTERVAL_CHANGED, interval); + } + return (this._timer = timer); + }, + + classID: Components.ID("705a423f-2f69-42f3-b9fe-1517e0dee56f"), + + QueryInterface: ChromeUtils.generateQI([ + "nsINamed", + "nsIObserver", + "nsISupportsWeakReference", + "nsITimerCallback", + ]), +}; diff --git a/toolkit/components/places/PlacesFrecencyRecalculator.sys.mjs b/toolkit/components/places/PlacesFrecencyRecalculator.sys.mjs new file mode 100644 index 0000000000..192ef49bf5 --- /dev/null +++ b/toolkit/components/places/PlacesFrecencyRecalculator.sys.mjs @@ -0,0 +1,541 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=2 ts=2 sts=2 expandtab + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This component handles frecency recalculations and decay on idle. + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs", + DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + PromiseUtils: "resource://gre/modules/PromiseUtils.sys.mjs", +}); + +ChromeUtils.defineModuleGetter( + lazy, + "ObjectUtils", + "resource://gre/modules/ObjectUtils.jsm" +); + +XPCOMUtils.defineLazyGetter(lazy, "logger", function () { + return lazy.PlacesUtils.getLogger({ prefix: "FrecencyRecalculator" }); +}); + +// Decay rate applied daily to frecency scores. +// A scaling factor of .975 results in an half-life of 28 days. +const FRECENCY_DECAYRATE = "0.975"; +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "frecencyDecayRate", + "places.frecency.decayRate", + FRECENCY_DECAYRATE, + null, + val => { + if (typeof val == "string") { + val = parseFloat(val); + } + if (val > 1.0) { + lazy.logger.error("Invalid frecency decay rate value: " + val); + val = parseFloat(FRECENCY_DECAYRATE); + } + return val; + } +); + +// An adaptive history entry is removed if unused for these many days. +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "adaptiveHistoryExpireDays", + "places.adaptiveHistory.expireDays", + 90 +); + +// Time between deferred task executions. +const DEFERRED_TASK_INTERVAL_MS = 2 * 60000; +// Maximum time to wait for an idle before the task is executed anyway. +const DEFERRED_TASK_MAX_IDLE_WAIT_MS = 5 * 60000; +// Number of entries to update at once. +const DEFAULT_CHUNK_SIZE = 50; + +export class PlacesFrecencyRecalculator { + classID = Components.ID("1141fd31-4c1a-48eb-8f1a-2f05fad94085"); + + /** + * A DeferredTask that runs our tasks. + */ + #task = null; + + /** + * Handler for alternative frecency. + * This allows to manager alternative ranking algorithms to experiment with. + */ + #alternativeFrecencyHelper = null; + + /** + * This is useful for testing. + */ + get alternativeFrecencyInfo() { + return this.#alternativeFrecencyHelper?.sets; + } + + constructor() { + lazy.logger.trace("Initializing Frecency Recalculator"); + + this.QueryInterface = ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]); + + // Do not initialize during shutdown. + if ( + Services.startup.isInOrBeyondShutdownPhase( + Ci.nsIAppStartup.SHUTDOWN_PHASE_APPSHUTDOWNTEARDOWN + ) + ) { + return; + } + + this.#task = new lazy.DeferredTask( + this.#taskFn.bind(this), + DEFERRED_TASK_INTERVAL_MS, + DEFERRED_TASK_MAX_IDLE_WAIT_MS + ); + lazy.AsyncShutdown.profileChangeTeardown.addBlocker( + "PlacesFrecencyRecalculator: shutdown", + () => this.#finalize() + ); + + // The public methods and properties are intended to be used by tests, and + // are exposed through the raw js object. Since this is expected to work + // based on signals or notification, it should not be necessary to expose + // any API for the product, though if that would become necessary in the + // future, we could add an interface for the service. + this.wrappedJSObject = this; + // This can be used by tests to await for the decay process. + this.pendingFrecencyDecayPromise = Promise.resolve(); + + Services.obs.addObserver(this, "idle-daily", true); + Services.obs.addObserver(this, "frecency-recalculation-needed", true); + + this.#alternativeFrecencyHelper = new AlternativeFrecencyHelper(this); + + // Run once on startup, so we pick up any leftover work. + lazy.PlacesUtils.history.shouldStartFrecencyRecalculation = true; + this.maybeStartFrecencyRecalculation(); + } + + async #taskFn() { + if (this.#task.isFinalized) { + return; + } + const refObj = {}; + const histogram = "PLACES_FRECENCY_RECALC_CHUNK_TIME_MS"; + TelemetryStopwatch.start(histogram, refObj); + try { + if (await this.recalculateSomeFrecencies()) { + TelemetryStopwatch.finish(histogram, refObj); + } else { + TelemetryStopwatch.cancel(histogram, refObj); + } + } catch (ex) { + TelemetryStopwatch.cancel(histogram, refObj); + console.error(ex); + lazy.logger.error(ex); + } + } + + #finalize() { + lazy.logger.trace("Finalizing frecency recalculator"); + // We don't mind about tasks completiion, since we can execute them in the + // next session. + this.#task.disarm(); + this.#task.finalize().catch(console.error); + } + + /** + * Updates a chunk of outdated frecency values. If there's more frecency + * values to update at the end of the process, it may rearm the task. + * @param {Number} chunkSize maximum number of entries to update at a time, + * set to -1 to update any entry. + * @resolves {Number} Number of affected pages. + */ + async recalculateSomeFrecencies({ chunkSize = DEFAULT_CHUNK_SIZE } = {}) { + lazy.logger.trace( + `Recalculate ${chunkSize >= 0 ? chunkSize : "infinite"} frecency values` + ); + let affectedCount = 0; + let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection(); + await db.executeTransaction(async function () { + let affected = await db.executeCached( + `UPDATE moz_places + SET frecency = CALCULATE_FRECENCY(id) + WHERE id IN ( + SELECT id FROM moz_places + WHERE recalc_frecency = 1 + ORDER BY frecency DESC + LIMIT ${chunkSize} + ) + RETURNING id` + ); + affectedCount += affected.length; + // We've finished recalculating frecencies. Trigger frecency updates for + // any affected origin. + await db.executeCached(`DELETE FROM moz_updateoriginsupdate_temp`); + }); + + if (affectedCount) { + PlacesObservers.notifyListeners([new PlacesRanking()]); + } + + // If alternative frecency is enabled, also recalculate a chunk of it. + affectedCount += + await this.#alternativeFrecencyHelper.recalculateSomeAlternativeFrecencies( + { chunkSize } + ); + + if (chunkSize > 0 && affectedCount >= chunkSize) { + // There's more entries to recalculate, rearm the task. + this.#task.arm(); + } else { + // There's nothing left to recalculate, wait for the next change. + lazy.PlacesUtils.history.shouldStartFrecencyRecalculation = false; + this.#task.disarm(); + } + return affectedCount; + } + + /** + * Forces full recalculation of any outdated frecency values. + * This exists for testing purposes; in tests we don't want to wait for + * the deferred task to run, this can enforce a recalculation. + */ + async recalculateAnyOutdatedFrecencies() { + this.#task.disarm(); + return this.recalculateSomeFrecencies({ chunkSize: -1 }); + } + + /** + * Whether a recalculation task is pending. + */ + get isRecalculationPending() { + return this.#task.isArmed; + } + + /** + * Invoked periodically to eventually start a recalculation task. + */ + maybeStartFrecencyRecalculation() { + if ( + lazy.PlacesUtils.history.shouldStartFrecencyRecalculation && + !this.#task.isFinalized + ) { + lazy.logger.trace("Arm frecency recalculation"); + this.#task.arm(); + } + } + + /** + * Decays frecency and adaptive history. + * @resolves once the process is complete. Never rejects. + */ + async decay() { + lazy.logger.trace("Decay frecency"); + let refObj = {}; + TelemetryStopwatch.start("PLACES_IDLE_FRECENCY_DECAY_TIME_MS", refObj); + // Ensure moz_places_afterupdate_frecency_trigger ignores decaying + // frecency changes. + lazy.PlacesUtils.history.isFrecencyDecaying = true; + try { + let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection(); + await db.executeTransaction(async function () { + // Decay all frecency rankings to reduce value of pages that haven't + // been visited in a while. + await db.executeCached( + `UPDATE moz_places SET frecency = ROUND(frecency * :decay_rate) + WHERE frecency > 0 AND recalc_frecency = 0`, + { decay_rate: lazy.frecencyDecayRate } + ); + // Decay potentially unused adaptive entries (e.g. those that are at 1) + // to allow better chances for new entries that will start at 1. + await db.executeCached( + `UPDATE moz_inputhistory SET use_count = use_count * :decay_rate`, + { decay_rate: lazy.frecencyDecayRate } + ); + // Delete any adaptive entries that won't help in ordering anymore. + await db.executeCached( + `DELETE FROM moz_inputhistory WHERE use_count < :use_count`, + { + use_count: Math.pow( + lazy.frecencyDecayRate, + lazy.adaptiveHistoryExpireDays + ), + } + ); + // We've finished decaying frecencies. Trigger frecency updates for + // any affected origin. + await db.executeCached(`DELETE FROM moz_updateoriginsupdate_temp`); + + TelemetryStopwatch.finish("PLACES_IDLE_FRECENCY_DECAY_TIME_MS", refObj); + PlacesObservers.notifyListeners([new PlacesRanking()]); + }); + } catch (ex) { + TelemetryStopwatch.cancel("PLACES_IDLE_FRECENCY_DECAY_TIME_MS", refObj); + console.error(ex); + lazy.logger.error(ex); + } finally { + lazy.PlacesUtils.history.isFrecencyDecaying = false; + } + } + + observe(subject, topic, data) { + lazy.logger.trace(`Got ${topic} topic`); + switch (topic) { + case "idle-daily": + this.pendingFrecencyDecayPromise = this.decay(); + // Also recalculate frecencies. + lazy.logger.trace("Frecency recalculation on idle"); + lazy.PlacesUtils.history.shouldStartFrecencyRecalculation = true; + this.maybeStartFrecencyRecalculation(); + return; + case "frecency-recalculation-needed": + lazy.logger.trace("Frecency recalculation requested"); + this.maybeStartFrecencyRecalculation(); + return; + case "test-execute-taskFn": + subject.promise = this.#taskFn(); + return; + case "test-alternative-frecency-init": + this.#alternativeFrecencyHelper = new AlternativeFrecencyHelper(this); + subject.promise = + this.#alternativeFrecencyHelper.initializedDeferred.promise; + } + } +} + +class AlternativeFrecencyHelper { + initializedDeferred = lazy.PromiseUtils.defer(); + #recalculator = null; + + sets = { + pages: { + // This pref is only read once and used to kick-off recalculations. + enabled: Services.prefs.getBoolPref( + "places.frecency.pages.alternative.featureGate", + false + ), + // Key used to store variables in the moz_meta table. + metadataKey: "page_alternative_frecency", + // The table containing frecency. + table: "moz_places", + // Object containing variables influencing the calculation. + // Any change to this object will cause a full recalculation on restart. + variables: { + // Current version of origins alternative frecency. + // ! IMPORTANT: Always bump up when making changes to the algorithm. + version: 1, + highWeight: Services.prefs.getIntPref( + "places.frecency.pages.alternative.highWeight", + 100 + ), + mediumWeight: Services.prefs.getIntPref( + "places.frecency.pages.alternative.mediumWeight", + 50 + ), + lowWeight: Services.prefs.getIntPref( + "places.frecency.pages.alternative.lowWeight", + 20 + ), + halfLifeDays: Services.prefs.getIntPref( + "places.frecency.pages.alternative.halfLifeDays", + 30 + ), + numSampledVisits: Services.prefs.getIntPref( + "places.frecency.pages.alternative.numSampledVisits", + 10 + ), + }, + method: this.#recalculateSomePagesAlternativeFrecencies, + }, + + origins: { + // This pref is only read once and used to kick-off recalculations. + enabled: Services.prefs.getBoolPref( + "places.frecency.origins.alternative.featureGate", + false + ), + // Key used to store variables in the moz_meta table. + metadataKey: "origin_alternative_frecency", + // The table containing frecency. + table: "moz_origins", + // Object containing variables influencing the calculation. + // Any change to this object will cause a full recalculation on restart. + variables: { + // Current version of origins alternative frecency. + // ! IMPORTANT: Always bump up when making changes to the algorithm. + version: 2, + // Frecencies of pages are ignored after these many days. + daysCutOff: Services.prefs.getIntPref( + "places.frecency.origins.alternative.daysCutOff", + 90 + ), + }, + method: this.#recalculateSomeOriginsAlternativeFrecencies, + }, + }; + + constructor(recalculator) { + this.#recalculator = recalculator; + this.#kickOffAlternativeFrecencies() + .catch(console.error) + .finally(() => this.initializedDeferred.resolve()); + } + + async #kickOffAlternativeFrecencies() { + let recalculateFirstChunk = false; + for (let [type, set] of Object.entries(this.sets)) { + // Now check the variables cached in the moz_meta table. If not found we + // assume alternative frecency was disabled in the previous session. + let storedVariables = await lazy.PlacesUtils.metadata.get( + set.metadataKey, + null + ); + + // Check whether this is the first-run, that happens when the alternative + // ranking is enabled and it was not at the previous session, or variables + // were changed. We should recalculate all the alternative frecency values. + if ( + set.enabled && + !lazy.ObjectUtils.deepEqual(set.variables, storedVariables) + ) { + lazy.logger.trace( + `Alternative frecency of ${type} must be recalculated` + ); + await lazy.PlacesUtils.withConnectionWrapper( + `PlacesFrecencyRecalculator :: ${type} alternative frecency set recalc`, + async db => { + await db.execute(`UPDATE ${set.table} SET recalc_alt_frecency = 1`); + } + ); + await lazy.PlacesUtils.metadata.set(set.metadataKey, set.variables); + recalculateFirstChunk = true; + continue; + } + + if (!set.enabled && storedVariables) { + lazy.logger.trace(`Clean up alternative frecency of ${type}`); + // Clear alternative frecency to save on space. + await lazy.PlacesUtils.withConnectionWrapper( + `PlacesFrecencyRecalculator :: ${type} alternative frecency set NULL`, + async db => { + await db.execute(`UPDATE ${set.table} SET alt_frecency = NULL`); + } + ); + await lazy.PlacesUtils.metadata.delete(set.metadataKey); + } + } + + if (recalculateFirstChunk) { + // Do a first recalculation immediately, so we don't leave the user + // with unranked entries for too long. + await this.recalculateSomeAlternativeFrecencies(); + + // Ensure the recalculation task is armed for a second run. + lazy.PlacesUtils.history.shouldStartFrecencyRecalculation = true; + this.#recalculator.maybeStartFrecencyRecalculation(); + } + } + + /** + * Updates a chunk of outdated frecency values. + * @param {Number} chunkSize maximum number of entries to update at a time, + * set to -1 to update any entry. + * @resolves {Number} Number of affected pages. + */ + async recalculateSomeAlternativeFrecencies({ + chunkSize = DEFAULT_CHUNK_SIZE, + } = {}) { + let affected = 0; + for (let set of Object.values(this.sets)) { + if (!set.enabled) { + continue; + } + try { + affected += await set.method({ chunkSize, variables: set.variables }); + } catch (ex) { + console.error(ex); + } + } + return affected; + } + + async #recalculateSomePagesAlternativeFrecencies({ chunkSize, variables }) { + lazy.logger.trace( + `Recalculate ${chunkSize} alternative pages frecency values` + ); + let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection(); + let affected = await db.executeCached( + `UPDATE moz_places + SET alt_frecency = CALCULATE_ALT_FRECENCY(moz_places.id), + recalc_alt_frecency = 0 + WHERE id IN ( + SELECT id FROM moz_places + WHERE recalc_alt_frecency = 1 + ORDER BY frecency DESC + LIMIT ${chunkSize} + ) + RETURNING id` + ); + return affected; + } + + async #recalculateSomeOriginsAlternativeFrecencies({ chunkSize, variables }) { + lazy.logger.trace( + `Recalculate ${chunkSize} alternative origins frecency values` + ); + let affectedCount = 0; + let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection(); + await db.executeTransaction(async () => { + let affected = await db.executeCached( + ` + UPDATE moz_origins + SET alt_frecency = ( + SELECT sum(frecency) + FROM moz_places h + WHERE origin_id = moz_origins.id + AND last_visit_date > + strftime('%s','now','localtime','start of day', + '-${variables.daysCutOff} day','utc') * 1000000 + ), recalc_alt_frecency = 0 + WHERE id IN ( + SELECT id FROM moz_origins + WHERE recalc_alt_frecency = 1 + ORDER BY frecency DESC + LIMIT ${chunkSize} + ) + RETURNING id` + ); + affectedCount += affected.length; + + // Calculate and store the alternative frecency threshold. Origins above + // this threshold will be considered meaningful and autofilled. + if (affected.length) { + let threshold = ( + await db.executeCached(`SELECT avg(alt_frecency) FROM moz_origins`) + )[0].getResultByIndex(0); + await lazy.PlacesUtils.metadata.set( + "origin_alt_frecency_threshold", + threshold + ); + } + }); + + return affectedCount; + } +} diff --git a/toolkit/components/places/PlacesPreviews.sys.mjs b/toolkit/components/places/PlacesPreviews.sys.mjs new file mode 100644 index 0000000000..e77e94ce13 --- /dev/null +++ b/toolkit/components/places/PlacesPreviews.sys.mjs @@ -0,0 +1,451 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { EventEmitter } from "resource://gre/modules/EventEmitter.sys.mjs"; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BackgroundPageThumbs: "resource://gre/modules/BackgroundPageThumbs.sys.mjs", + PageThumbsStorage: "resource://gre/modules/PageThumbs.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "logConsole", function () { + return console.createInstance({ + prefix: "PlacesPreviews", + maxLogLevel: Services.prefs.getBoolPref("places.previews.log", false) + ? "Debug" + : "Warn", + }); +}); + +// Toggling Places previews requires a restart, because a database trigger +// filling up tombstones is enabled on the database only when the pref is set +// on startup. +XPCOMUtils.defineLazyGetter(lazy, "previewsEnabled", function () { + return Services.prefs.getBoolPref("places.previews.enabled", false); +}); + +// Preview deletions are done in chunks of this size. +const DELETE_CHUNK_SIZE = 50; +// This is the time between deletion chunks. +const DELETE_TIMEOUT_MS = 60000; + +// The folder inside the profile folder where to store previews. +const PREVIEWS_DIRECTORY = "places-previews"; + +// How old a preview file should be before we replace it. +const DAYS_BEFORE_REPLACEMENT = 30; + +/** + * This extends Set to only keep the latest 100 entries. + */ +class LimitedSet extends Set { + #limit = 100; + add(key) { + super.add(key); + let oversize = this.size - this.#limit; + if (oversize > 0) { + for (let entry of this) { + if (oversize-- <= 0) { + break; + } + this.delete(entry); + } + } + } +} + +/** + * This class handles previews deletion from tombstones in the database. + * Deletion happens in chunks, each chunk runs after DELETE_TIMEOUT_MS, and + * the process is interrupted once there's nothing more to delete. + * Any page removal operations on the Places database will restart the timer. + */ +class DeletionHandler { + #timeoutId = null; + #shutdownProgress = {}; + + /** + * This can be set by tests to speed up the deletion process, otherwise the + * product should just use the default value. + */ + #timeout = DELETE_TIMEOUT_MS; + get timeout() { + return this.#timeout; + } + set timeout(val) { + if (this.#timeoutId) { + lazy.clearTimeout(this.#timeoutId); + this.#timeoutId = null; + } + this.#timeout = val; + this.ensureRunning(); + } + + constructor() { + // Clear any pending timeouts on shutdown. + lazy.PlacesUtils.history.shutdownClient.jsclient.addBlocker( + "PlacesPreviews.jsm::DeletionHandler", + async () => { + this.#shutdownProgress.shuttingDown = true; + lazy.clearTimeout(this.#timeoutId); + this.#timeoutId = null; + }, + { fetchState: () => this.#shutdownProgress } + ); + } + + /** + * This should be invoked everytime we expect there are tombstones to + * handle. If deletion is already pending, this is a no-op. + */ + ensureRunning() { + if (this.#timeoutId || this.#shutdownProgress.shuttingDown) { + return; + } + this.#timeoutId = lazy.setTimeout(() => { + this.#timeoutId = null; + ChromeUtils.idleDispatch(() => { + this.#deleteChunk().catch(ex => + lazy.logConsole.error("Error during previews deletion:" + ex) + ); + }); + }, this.timeout); + } + + /** + * Deletes a chunk of previews. + */ + async #deleteChunk() { + if (this.#shutdownProgress.shuttingDown) { + return; + } + // Select tombstones, delete images, then delete tombstones. This order + // ensures that in case of problems we'll try again in the future. + let db = await lazy.PlacesUtils.promiseDBConnection(); + let count; + let hashes = ( + await db.executeCached( + `SELECT hash, (SELECT count(*) FROM moz_previews_tombstones) AS count + FROM moz_previews_tombstones LIMIT ${DELETE_CHUNK_SIZE}` + ) + ).map(r => { + if (count === undefined) { + count = r.getResultByName("count"); + } + return r.getResultByName("hash"); + }); + if (!count || this.#shutdownProgress.shuttingDown) { + // There's nothing to delete, or it's too late. + return; + } + + let deleted = []; + for (let hash of hashes) { + let filePath = PlacesPreviews.getPathForHash(hash); + try { + await IOUtils.remove(filePath); + PlacesPreviews.onDelete(filePath); + deleted.push(hash); + } catch (ex) { + if (DOMException.isInstance(ex) && ex.name == "NotFoundError") { + deleted.push(hash); + } else { + lazy.logConsole.error("Unable to delete file: " + filePath); + } + } + if (this.#shutdownProgress.shuttingDown) { + return; + } + } + // Delete hashes from tombstones. + let params = deleted.reduce((p, c, i) => { + p["hash" + i] = c; + return p; + }, {}); + await lazy.PlacesUtils.withConnectionWrapper( + "PlacesPreviews.jsm::ExpirePreviews", + async db => { + await db.execute( + `DELETE FROM moz_previews_tombstones WHERE hash in + (${Object.keys(params) + .map(p => `:${p}`) + .join(",")})`, + params + ); + } + ); + + if (count > DELETE_CHUNK_SIZE) { + this.ensureRunning(); + } + } +} + +/** + * Handles previews for Places urls. + * Previews are stored in WebP format, using MD5 hash of the page url in hex + * format. All the previews are saved into a "places-previews" folder under + * the roaming profile folder. + */ +export const PlacesPreviews = new (class extends EventEmitter { + #placesObserver = null; + #deletionHandler = null; + // This is used as a cache to avoid fetching the same preview multiple + // times in a short timeframe. + #recentlyUpdatedPreviews = new LimitedSet(); + + fileExtension = ".webp"; + fileContentType = "image/webp"; + + constructor() { + super(); + // Observe page removals and delete previews when necessary. + this.#placesObserver = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + PlacesObservers.addListener( + ["history-cleared", "page-removed"], + this.#placesObserver + ); + + // Start deletion in case it was interruped during the previous session, + // it will end once there's nothing more to delete. + this.#deletionHandler = new DeletionHandler(); + this.#deletionHandler.ensureRunning(); + } + + handlePlacesEvents(events) { + for (const event of events) { + if ( + event.type == "history-cleared" || + (event.type == "page-removed" && event.isRemovedFromStore) + ) { + this.#deletionHandler.ensureRunning(); + return; + } + } + } + + /** + * Whether the feature is enabled. Use this instead of directly checking + * the pref, since it requires a restart. + */ + get enabled() { + return lazy.previewsEnabled; + } + + /** + * Returns the path to the previews folder. + * @returns {string} The path to the previews folder. + */ + getPath() { + return PathUtils.join( + Services.dirsvc.get("ProfD", Ci.nsIFile).path, + PREVIEWS_DIRECTORY + ); + } + + /** + * Returns the file path of the preview for the given url. + * This doesn't guarantee the file exists. + * @param {string} url Address of the page. + * @returns {string} File path of the preview for the given url. + */ + getPathForUrl(url) { + return PathUtils.join( + this.getPath(), + lazy.PlacesUtils.md5(url, { format: "hex" }) + this.fileExtension + ); + } + + /** + * Returns the file path of the preview having the given hash. + * @param {string} hash md5 hash in hex format. + * @returns {string } File path of the preview having the given hash. + */ + getPathForHash(hash) { + return PathUtils.join(this.getPath(), hash + this.fileExtension); + } + + /** + * Returns the moz-page-thumb: url to show the preview for the given url. + * @param {string} url Address of the page. + * @returns {string} Preview url for the given page url. + */ + getPageThumbURL(url) { + return ( + "moz-page-thumb://" + + "places-previews" + + "/?url=" + + encodeURIComponent(url) + + "&revision=" + + lazy.PageThumbsStorage.getRevision(url) + ); + } + + /** + * Updates the preview for the given page url. The update happens in + * background, using a windowless browser with very conservative privacy + * settings. Due to this, it may not look exactly like the page that the user + * is normally facing when logged in. See BackgroundPageThumbs.jsm for + * additional details. + * Unless `forceUpdate` is set, the preview is not updated if: + * - It was already fetched recently + * - The stored preview is younger than DAYS_BEFORE_REPLACEMENT + * The previem image is encoded using WebP. + * @param {string} url The address of the page. + * @param {boolean} [forceUpdate] Whether to update the preview regardless. + * @returns {boolean} Whether a preview is available and ready. + */ + async update(url, { forceUpdate = false } = {}) { + if (!this.enabled) { + return false; + } + let filePath = this.getPathForUrl(url); + if (!forceUpdate) { + if (this.#recentlyUpdatedPreviews.has(filePath)) { + lazy.logConsole.debug("Skipping update because recently updated"); + return true; + } + try { + let fileInfo = await IOUtils.stat(filePath); + if ( + fileInfo.lastModified > + Date.now() - DAYS_BEFORE_REPLACEMENT * 86400000 + ) { + // File is recent enough. + this.#recentlyUpdatedPreviews.add(filePath); + lazy.logConsole.debug("Skipping update because file is recent"); + return true; + } + } catch (ex) { + // If the file doesn't exist, we always update it. + if (!DOMException.isInstance(ex) || ex.name != "NotFoundError") { + lazy.logConsole.error("Error while trying to stat() preview" + ex); + return false; + } + } + } + + let buffer = await new Promise(resolve => { + let observer = (subject, topic, errorUrl) => { + if (errorUrl == url) { + resolve(null); + } + }; + Services.obs.addObserver(observer, "page-thumbnail:error"); + lazy.BackgroundPageThumbs.capture(url, { + dontStore: true, + contentType: this.fileContentType, + onDone: (url, reason, handle) => { + Services.obs.removeObserver(observer, "page-thumbnail:error"); + resolve(handle?.data); + }, + }); + }); + if (!buffer) { + lazy.logConsole.error("Unable to fetch preview: " + url); + return false; + } + try { + await IOUtils.makeDirectory(this.getPath(), { ignoreExisting: true }); + await IOUtils.write(filePath, new Uint8Array(buffer), { + tmpPath: filePath + ".tmp", + }); + } catch (ex) { + lazy.logConsole.error( + lazy.logConsole.error("Unable to create preview: " + ex) + ); + return false; + } + this.#recentlyUpdatedPreviews.add(filePath); + return true; + } + + /** + * Removes orphan previews that are not tracked by Places. + * Orphaning should normally not happen, but unexpected manipulation (e.g. the + * user touching the profile folder, or third party applications) could cause + * it. + * This method is slow, because it has to go through all the Places stored + * pages, thus it's suggested to only run it as periodic maintenance. + * @returns {boolean} Whether orphans deletion ran. + */ + async deleteOrphans() { + if (!this.enabled) { + return false; + } + + // From the previews directory, get all the files whose name matches our + // format. Avoid any other filenames, also for safety reasons, since we are + // injecting them into SQL. + let files = await IOUtils.getChildren(this.getPath()); + let hashes = files + .map(f => PathUtils.filename(f)) + .filter(n => /^[a-f0-9]{32}\.webp$/) + .map(n => n.substring(0, n.lastIndexOf("."))); + + await lazy.PlacesUtils.withConnectionWrapper( + "PlacesPreviews.jsm::deleteOrphans", + async db => { + await db.execute( + ` + WITH files(hash) AS ( + VALUES ${hashes.map(h => `('${h}')`).join(", ")} + ) + INSERT OR IGNORE INTO moz_previews_tombstones + SELECT hash FROM files + EXCEPT + SELECT md5hex(url) FROM moz_places + ` + ); + } + ); + this.#deletionHandler.ensureRunning(); + return true; + } + + /** + * This is invoked by #deletionHandler every time a preview file is removed. + * @param {string} filePath The path of the deleted file. + */ + onDelete(filePath) { + this.#recentlyUpdatedPreviews.delete(filePath); + this.emit("places-preview-deleted", filePath); + } + + /** + * Used by tests to change the deletion timeout between chunks. + * @param {integer} timeout New timeout in milliseconds. + */ + testSetDeletionTimeout(timeout) { + if (timeout === null) { + this.#deletionHandler.timeout = DELETE_TIMEOUT_MS; + } else { + this.#deletionHandler.timeout = timeout; + } + } +})(); + +/** + * Used to exposes nsIPlacesPreviewsHelperService to the moz-page-thumb protocol + * cpp implementation. + */ +export function PlacesPreviewsHelperService() {} + +PlacesPreviewsHelperService.prototype = { + classID: Components.ID("{bd0a4d3b-ff26-4d4d-9a62-a513e1c1bf92}"), + QueryInterface: ChromeUtils.generateQI(["nsIPlacesPreviewsHelperService"]), + + getFilePathForURL(url) { + return PlacesPreviews.getPathForUrl(url); + }, +}; diff --git a/toolkit/components/places/PlacesQuery.sys.mjs b/toolkit/components/places/PlacesQuery.sys.mjs new file mode 100644 index 0000000000..8c3fd0372f --- /dev/null +++ b/toolkit/components/places/PlacesQuery.sys.mjs @@ -0,0 +1,210 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + requestIdleCallback: "resource://gre/modules/Timer.sys.mjs", +}); +XPCOMUtils.defineLazyModuleGetters(lazy, { + ObjectUtils: "resource://gre/modules/ObjectUtils.jsm", +}); + +function isRedirectType(visitType) { + const { TRANSITIONS } = lazy.PlacesUtils.history; + return ( + visitType === TRANSITIONS.REDIRECT_PERMANENT || + visitType === TRANSITIONS.REDIRECT_TEMPORARY + ); +} + +const BULK_PLACES_EVENTS_THRESHOLD = 50; + +/** + * An object that contains details of a page visit. + * + * @typedef {object} HistoryVisit + * + * @property {Date} date + * When this page was visited. + * @property {number} id + * Visit ID from the database. + * @property {string} title + * The page's title. + * @property {string} url + * The page's URL. + */ + +/** + * Queries the places database using an async read only connection. Maintains + * an internal cache of query results which is live-updated by adding listeners + * to `PlacesObservers`. When the results are no longer needed, call `close` to + * remove the listeners. + */ +export class PlacesQuery { + /** @type HistoryVisit[] */ + #cachedHistory = null; + /** @type object */ + #cachedHistoryOptions = null; + /** @type function(PlacesEvent[]) */ + #historyListener = null; + /** @type function(HistoryVisit[]) */ + #historyListenerCallback = null; + + /** + * Get a snapshot of history visits at this moment. + * + * @param {object} [options] + * Options to apply to the database query. + * @param {number} [options.daysOld] + * The maximum number of days to go back in history. + * @returns {HistoryVisit[]} + * History visits obtained from the database query. + */ + async getHistory({ daysOld = 60 } = {}) { + const options = { daysOld }; + const cacheInvalid = + this.#cachedHistory == null || + !lazy.ObjectUtils.deepEqual(options, this.#cachedHistoryOptions); + if (cacheInvalid) { + this.#cachedHistory = []; + this.#cachedHistoryOptions = options; + const db = await lazy.PlacesUtils.promiseDBConnection(); + const sql = `SELECT v.id, visit_date, title, url, visit_type, from_visit, hidden + FROM moz_historyvisits v + JOIN moz_places h + ON v.place_id = h.id + WHERE visit_date >= (strftime('%s','now','localtime','start of day','-${Number( + daysOld + )} days','utc') * 1000000) + ORDER BY visit_date DESC`; + const rows = await db.executeCached(sql); + let lastUrl; // Avoid listing consecutive visits to the same URL. + let lastRedirectFromVisitId; // Avoid listing redirecting visits. + for (const row of rows) { + const [id, visitDate, title, url, visitType, fromVisit, hidden] = + Array.from({ length: row.numEntries }, (_, i) => + row.getResultByIndex(i) + ); + if (isRedirectType(visitType) && fromVisit > 0) { + lastRedirectFromVisitId = fromVisit; + } + if (!hidden && url !== lastUrl && id !== lastRedirectFromVisitId) { + this.#cachedHistory.push({ + date: lazy.PlacesUtils.toDate(visitDate), + id, + title, + url, + }); + lastUrl = url; + } + } + } + if (!this.#historyListener) { + this.#initHistoryListener(); + } + return this.#cachedHistory; + } + + /** + * Observe changes to the visits table. When changes are made, the callback + * is given the new list of visits. Only one callback can be active at a time + * (per instance). If one already exists, it will be replaced. + * + * @param {function(HistoryVisit[])} callback + * The function to call when changes are made. + */ + observeHistory(callback) { + this.#historyListenerCallback = callback; + } + + /** + * Close this query. Caches are cleared and listeners are removed. + */ + close() { + this.#cachedHistory = null; + this.#cachedHistoryOptions = null; + PlacesObservers.removeListener( + ["page-removed", "page-visited", "history-cleared", "page-title-changed"], + this.#historyListener + ); + this.#historyListener = null; + this.#historyListenerCallback = null; + } + + /** + * Listen for changes to the visits table and update caches accordingly. + */ + #initHistoryListener() { + this.#historyListener = async events => { + if ( + events.length >= BULK_PLACES_EVENTS_THRESHOLD || + events.some(({ type }) => type === "page-removed") + ) { + // Accounting for cascading deletes, or handling places events in bulk, + // can be expensive. In this case, we invalidate the cache once rather + // than handling each event individually. + this.#cachedHistory = null; + } else if (this.#cachedHistory != null) { + for (const event of events) { + switch (event.type) { + case "page-visited": + await this.#handlePageVisited(event); + break; + case "history-cleared": + this.#cachedHistory = []; + break; + case "page-title-changed": + this.#cachedHistory + .filter(({ url }) => url === event.url) + .forEach(visit => (visit.title = event.title)); + break; + } + } + } + if (typeof this.#historyListenerCallback === "function") { + lazy.requestIdleCallback(async () => { + const history = await this.getHistory(this.#cachedHistoryOptions); + this.#historyListenerCallback(history); + }); + } + }; + PlacesObservers.addListener( + ["page-removed", "page-visited", "history-cleared", "page-title-changed"], + this.#historyListener + ); + } + + /** + * Handle a page visited event. + * + * @param {PlacesEvent} event + * The event. + */ + async #handlePageVisited(event) { + const lastVisit = this.#cachedHistory[0]; + if ( + lastVisit != null && + (event.url === lastVisit.url || + (isRedirectType(event.transitionType) && + event.referringVisitId === lastVisit.id)) + ) { + // Remove the last visit if it duplicates this visit's URL, or if it + // redirects to this visit. + this.#cachedHistory.shift(); + } + if (!event.hidden) { + this.#cachedHistory.unshift({ + date: new Date(event.visitTime), + id: event.visitId, + title: event.lastKnownTitle, + url: event.url, + }); + } + } +} diff --git a/toolkit/components/places/PlacesSyncUtils.sys.mjs b/toolkit/components/places/PlacesSyncUtils.sys.mjs new file mode 100644 index 0000000000..5268aaf68d --- /dev/null +++ b/toolkit/components/places/PlacesSyncUtils.sys.mjs @@ -0,0 +1,1996 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Log: "resource://gre/modules/Log.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +/** + * This module exports functions for Sync to use when applying remote + * records. The calls are similar to those in `Bookmarks.jsm` and + * `nsINavBookmarksService`, with special handling for + * tags, keywords, synced annotations, and missing parents. + */ +export var PlacesSyncUtils = {}; + +const { SOURCE_SYNC } = Ci.nsINavBookmarksService; + +const MICROSECONDS_PER_SECOND = 1000000; + +const MOBILE_BOOKMARKS_PREF = "browser.bookmarks.showMobileBookmarks"; + +// These are defined as lazy getters to defer initializing the bookmarks +// service until it's needed. +XPCOMUtils.defineLazyGetter(lazy, "ROOT_RECORD_ID_TO_GUID", () => ({ + menu: lazy.PlacesUtils.bookmarks.menuGuid, + places: lazy.PlacesUtils.bookmarks.rootGuid, + tags: lazy.PlacesUtils.bookmarks.tagsGuid, + toolbar: lazy.PlacesUtils.bookmarks.toolbarGuid, + unfiled: lazy.PlacesUtils.bookmarks.unfiledGuid, + mobile: lazy.PlacesUtils.bookmarks.mobileGuid, +})); + +XPCOMUtils.defineLazyGetter(lazy, "ROOT_GUID_TO_RECORD_ID", () => ({ + [lazy.PlacesUtils.bookmarks.menuGuid]: "menu", + [lazy.PlacesUtils.bookmarks.rootGuid]: "places", + [lazy.PlacesUtils.bookmarks.tagsGuid]: "tags", + [lazy.PlacesUtils.bookmarks.toolbarGuid]: "toolbar", + [lazy.PlacesUtils.bookmarks.unfiledGuid]: "unfiled", + [lazy.PlacesUtils.bookmarks.mobileGuid]: "mobile", +})); + +XPCOMUtils.defineLazyGetter(lazy, "ROOTS", () => + Object.keys(lazy.ROOT_RECORD_ID_TO_GUID) +); + +// Gets the history transition values we ignore and do not sync, as a +// string, which is a comma-separated set of values - ie, something which can +// be used with sqlite's IN operator. Does *not* includes the parens. +XPCOMUtils.defineLazyGetter(lazy, "IGNORED_TRANSITIONS_AS_SQL_LIST", () => + // * We don't sync `TRANSITION_FRAMED_LINK` visits - these are excluded when + // rendering the history menu, so we use the same constraints for Sync. + // * We don't sync `TRANSITION_DOWNLOAD` because it makes no sense to see + // these on other devices - the downloaded file can not exist. + // * We don't want to sync TRANSITION_EMBED visits, but these aren't + // stored in the DB, so no need to specify them. + // * 0 is invalid, and hopefully don't exist, but let's exclude it anyway. + // Array.toString() semantics are well defined and exactly what we need, so.. + [ + 0, + lazy.PlacesUtils.history.TRANSITION_FRAMED_LINK, + lazy.PlacesUtils.history.TRANSITION_DOWNLOAD, + ].toString() +); + +const HistorySyncUtils = (PlacesSyncUtils.history = Object.freeze({ + SYNC_ID_META_KEY: "sync/history/syncId", + LAST_SYNC_META_KEY: "sync/history/lastSync", + + /** + * Returns the current history sync ID, or `""` if one isn't set. + */ + getSyncId() { + return lazy.PlacesUtils.metadata.get(HistorySyncUtils.SYNC_ID_META_KEY, ""); + }, + + /** + * Assigns a new sync ID. This is called when we sync for the first time with + * a new account, and when we're the first to sync after a node reassignment. + * + * @return {Promise} resolved once the ID has been updated. + * @resolves to the new sync ID. + */ + resetSyncId() { + return lazy.PlacesUtils.withConnectionWrapper( + "HistorySyncUtils: resetSyncId", + function (db) { + let newSyncId = lazy.PlacesUtils.history.makeGuid(); + return db.executeTransaction(async function () { + await setHistorySyncId(db, newSyncId); + return newSyncId; + }); + } + ); + }, + + /** + * Ensures that the existing local sync ID, if any, is up-to-date with the + * server. This is called when we sync with an existing account. + * + * @param newSyncId + * The server's sync ID. + * @return {Promise} resolved once the ID has been updated. + */ + async ensureCurrentSyncId(newSyncId) { + if (!newSyncId || typeof newSyncId != "string") { + throw new TypeError("Invalid new history sync ID"); + } + await lazy.PlacesUtils.withConnectionWrapper( + "HistorySyncUtils: ensureCurrentSyncId", + async function (db) { + let existingSyncId = await lazy.PlacesUtils.metadata.getWithConnection( + db, + HistorySyncUtils.SYNC_ID_META_KEY, + "" + ); + + if (existingSyncId == newSyncId) { + lazy.HistorySyncLog.trace("History sync ID up-to-date", { + existingSyncId, + }); + return; + } + + lazy.HistorySyncLog.info( + "History sync ID changed; resetting metadata", + { + existingSyncId, + newSyncId, + } + ); + await db.executeTransaction(function () { + return setHistorySyncId(db, newSyncId); + }); + } + ); + }, + + /** + * Returns the last sync time, in seconds, for the history collection, or 0 + * if history has never synced before. + */ + async getLastSync() { + let lastSync = await lazy.PlacesUtils.metadata.get( + HistorySyncUtils.LAST_SYNC_META_KEY, + 0 + ); + return lastSync / 1000; + }, + + /** + * Updates the history collection last sync time. + * + * @param lastSyncSeconds + * The collection last sync time, in seconds, as a number or string. + */ + async setLastSync(lastSyncSeconds) { + let lastSync = Math.floor(lastSyncSeconds * 1000); + if (!Number.isInteger(lastSync)) { + throw new TypeError("Invalid history last sync timestamp"); + } + await lazy.PlacesUtils.metadata.set( + HistorySyncUtils.LAST_SYNC_META_KEY, + lastSync + ); + }, + + /** + * Removes all history visits and pages from the database. Sync calls this + * method when it receives a command from a remote client to wipe all stored + * data. + * + * @return {Promise} resolved once all pages and visits have been removed. + */ + async wipe() { + await lazy.PlacesUtils.history.clear(); + await HistorySyncUtils.reset(); + }, + + /** + * Removes the sync ID and last sync time for the history collection. Unlike + * `wipe`, this keeps all existing history pages and visits. + * + * @return {Promise} resolved once the metadata have been removed. + */ + reset() { + return lazy.PlacesUtils.metadata.delete( + HistorySyncUtils.SYNC_ID_META_KEY, + HistorySyncUtils.LAST_SYNC_META_KEY + ); + }, + + /** + * Clamps a history visit date between the current date and the earliest + * sensible date. + * + * @param {Date} visitDate + * The visit date. + * @return {Date} The clamped visit date. + */ + clampVisitDate(visitDate) { + let currentDate = new Date(); + if (visitDate > currentDate) { + return currentDate; + } + if (visitDate < BookmarkSyncUtils.EARLIEST_BOOKMARK_TIMESTAMP) { + return new Date(BookmarkSyncUtils.EARLIEST_BOOKMARK_TIMESTAMP); + } + return visitDate; + }, + + /** + * Fetches the frecency for the URL provided + * + * @param url + * @returns {Number} The frecency of the given url + */ + async fetchURLFrecency(url) { + let canonicalURL = lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.url(url); + + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT frecency + FROM moz_places + WHERE url_hash = hash(:url) AND url = :url + LIMIT 1`, + { url: canonicalURL.href } + ); + + return rows.length ? rows[0].getResultByName("frecency") : -1; + }, + + /** + * Filters syncable places from a collection of places guids. + * + * @param guids + * + * @returns {Array} new Array with the guids that aren't syncable + */ + async determineNonSyncableGuids(guids) { + // Filter out hidden pages and transitions that we don't sync. + let db = await lazy.PlacesUtils.promiseDBConnection(); + let nonSyncableGuids = []; + for (let chunk of lazy.PlacesUtils.chunkArray(guids, db.variableLimit)) { + let rows = await db.execute( + ` + SELECT DISTINCT p.guid FROM moz_places p + JOIN moz_historyvisits v ON p.id = v.place_id + WHERE p.guid IN (${new Array(chunk.length).fill("?").join(",")}) AND + (p.hidden = 1 OR v.visit_type IN (${ + lazy.IGNORED_TRANSITIONS_AS_SQL_LIST + })) + `, + chunk + ); + nonSyncableGuids = nonSyncableGuids.concat( + rows.map(row => row.getResultByName("guid")) + ); + } + return nonSyncableGuids; + }, + + /** + * Change the guid of the given uri + * + * @param uri + * @param guid + */ + changeGuid(uri, guid) { + let canonicalURL = lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.url(uri); + let validatedGuid = lazy.PlacesUtils.BOOKMARK_VALIDATORS.guid(guid); + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesSyncUtils.history: changeGuid", + async function (db) { + await db.executeCached( + ` + UPDATE moz_places + SET guid = :guid + WHERE url_hash = hash(:page_url) AND url = :page_url`, + { guid: validatedGuid, page_url: canonicalURL.href } + ); + } + ); + }, + + /** + * Fetch the last 20 visits (date and type of it) corresponding to a given url + * + * @param url + * @returns {Array} Each element of the Array is an object with members: date and type + */ + async fetchVisitsForURL(url) { + let canonicalURL = lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.url(url); + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT visit_type type, visit_date date + FROM moz_historyvisits + JOIN moz_places h ON h.id = place_id + WHERE url_hash = hash(:url) AND url = :url + ORDER BY date DESC LIMIT 20`, + { url: canonicalURL.href } + ); + return rows.map(row => { + let visitDate = row.getResultByName("date"); + let visitType = row.getResultByName("type"); + return { date: visitDate, type: visitType }; + }); + }, + + /** + * Fetches the guid of a uri + * + * @param uri + * @returns {String} The guid of the given uri + */ + async fetchGuidForURL(url) { + let canonicalURL = lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.url(url); + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT guid + FROM moz_places + WHERE url_hash = hash(:page_url) AND url = :page_url`, + { page_url: canonicalURL.href } + ); + if (!rows.length) { + return null; + } + return rows[0].getResultByName("guid"); + }, + + /** + * Fetch information about a guid (url, title and frecency) + * + * @param guid + * @returns {Object} Object with three members: url, title and frecency of the given guid + */ + async fetchURLInfoForGuid(guid) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT url, IFNULL(title, '') AS title, frecency + FROM moz_places + WHERE guid = :guid`, + { guid } + ); + if (rows.length === 0) { + return null; + } + return { + url: rows[0].getResultByName("url"), + title: rows[0].getResultByName("title"), + frecency: rows[0].getResultByName("frecency"), + }; + }, + + /** + * Get all URLs filtered by the limit and since members of the options object. + * + * @param options + * Options object with two members, since and limit. Both of them must be provided + * @returns {Array} - Up to limit number of URLs starting from the date provided by since + * + * Note that some visit types are explicitly excluded - downloads and framed + * links. + */ + async getAllURLs(options) { + // Check that the limit property is finite number. + if (!Number.isFinite(options.limit)) { + throw new Error("The number provided in options.limit is not finite."); + } + // Check that the since property is of type Date. + if ( + !options.since || + Object.prototype.toString.call(options.since) != "[object Date]" + ) { + throw new Error( + "The property since of the options object must be of type Date." + ); + } + let db = await lazy.PlacesUtils.promiseDBConnection(); + let sinceInMicroseconds = lazy.PlacesUtils.toPRTime(options.since); + let rows = await db.executeCached( + ` + SELECT DISTINCT p.url + FROM moz_places p + JOIN moz_historyvisits v ON p.id = v.place_id + WHERE p.last_visit_date > :cutoff_date AND + p.hidden = 0 AND + v.visit_type NOT IN (${lazy.IGNORED_TRANSITIONS_AS_SQL_LIST}) + ORDER BY frecency DESC + LIMIT :max_results`, + { cutoff_date: sinceInMicroseconds, max_results: options.limit } + ); + return rows.map(row => row.getResultByName("url")); + }, +})); + +const BookmarkSyncUtils = (PlacesSyncUtils.bookmarks = Object.freeze({ + SYNC_ID_META_KEY: "sync/bookmarks/syncId", + LAST_SYNC_META_KEY: "sync/bookmarks/lastSync", + WIPE_REMOTE_META_KEY: "sync/bookmarks/wipeRemote", + + // Jan 23, 1993 in milliseconds since 1970. Corresponds roughly to the release + // of the original NCSA Mosiac. We can safely assume that any dates before + // this time are invalid. + EARLIEST_BOOKMARK_TIMESTAMP: Date.UTC(1993, 0, 23), + + KINDS: { + BOOKMARK: "bookmark", + QUERY: "query", + FOLDER: "folder", + LIVEMARK: "livemark", + SEPARATOR: "separator", + }, + + get ROOTS() { + return lazy.ROOTS; + }, + + /** + * Returns the current bookmarks sync ID, or `""` if one isn't set. + */ + getSyncId() { + return lazy.PlacesUtils.metadata.get( + BookmarkSyncUtils.SYNC_ID_META_KEY, + "" + ); + }, + + /** + * Indicates if the bookmarks engine should erase all bookmarks on the server + * and all other clients, because the user manually restored their bookmarks + * from a backup on this client. + */ + async shouldWipeRemote() { + let shouldWipeRemote = await lazy.PlacesUtils.metadata.get( + BookmarkSyncUtils.WIPE_REMOTE_META_KEY, + false + ); + return !!shouldWipeRemote; + }, + + /** + * Assigns a new sync ID, bumps the change counter, and flags all items as + * "NEW" for upload. This is called when we sync for the first time with a + * new account, when we're the first to sync after a node reassignment, and + * on the first sync after a manual restore. + * + * @return {Promise} resolved once the ID and all items have been updated. + * @resolves to the new sync ID. + */ + resetSyncId() { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: resetSyncId", + function (db) { + let newSyncId = lazy.PlacesUtils.history.makeGuid(); + return db.executeTransaction(async function () { + await setBookmarksSyncId(db, newSyncId); + await resetAllSyncStatuses( + db, + lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW + ); + return newSyncId; + }); + } + ); + }, + + /** + * Ensures that the existing local sync ID, if any, is up-to-date with the + * server. This is called when we sync with an existing account. + * + * We always take the server's sync ID. If we don't have an existing ID, + * we're either syncing for the first time with an existing account, or Places + * has automatically restored from a backup. If the sync IDs don't match, + * we're likely syncing after a node reassignment, where another client + * uploaded their bookmarks first. + * + * @param newSyncId + * The server's sync ID. + * @return {Promise} resolved once the ID and all items have been updated. + */ + async ensureCurrentSyncId(newSyncId) { + if (!newSyncId || typeof newSyncId != "string") { + throw new TypeError("Invalid new bookmarks sync ID"); + } + await lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: ensureCurrentSyncId", + async function (db) { + let existingSyncId = await lazy.PlacesUtils.metadata.getWithConnection( + db, + BookmarkSyncUtils.SYNC_ID_META_KEY, + "" + ); + + // If we don't have a sync ID, take the server's without resetting + // sync statuses. + if (!existingSyncId) { + lazy.BookmarkSyncLog.info("Taking new bookmarks sync ID", { + newSyncId, + }); + await db.executeTransaction(() => setBookmarksSyncId(db, newSyncId)); + return; + } + + // If the existing sync ID matches the server, great! + if (existingSyncId == newSyncId) { + lazy.BookmarkSyncLog.trace("Bookmarks sync ID up-to-date", { + existingSyncId, + }); + return; + } + + // Otherwise, we have a sync ID, but it doesn't match, so we were likely + // node reassigned. Take the server's sync ID and reset all items to + // "UNKNOWN" so that we can merge. + lazy.BookmarkSyncLog.info( + "Bookmarks sync ID changed; resetting sync statuses", + { existingSyncId, newSyncId } + ); + await db.executeTransaction(async function () { + await setBookmarksSyncId(db, newSyncId); + await resetAllSyncStatuses( + db, + lazy.PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN + ); + }); + } + ); + }, + + /** + * Returns the last sync time, in seconds, for the bookmarks collection, or 0 + * if bookmarks have never synced before. + */ + async getLastSync() { + let lastSync = await lazy.PlacesUtils.metadata.get( + BookmarkSyncUtils.LAST_SYNC_META_KEY, + 0 + ); + return lastSync / 1000; + }, + + /** + * Updates the bookmarks collection last sync time. + * + * @param lastSyncSeconds + * The collection last sync time, in seconds, as a number or string. + */ + async setLastSync(lastSyncSeconds) { + let lastSync = Math.floor(lastSyncSeconds * 1000); + if (!Number.isInteger(lastSync)) { + throw new TypeError("Invalid bookmarks last sync timestamp"); + } + await lazy.PlacesUtils.metadata.set( + BookmarkSyncUtils.LAST_SYNC_META_KEY, + lastSync + ); + }, + + /** + * Resets Sync metadata for bookmarks in Places. This function behaves + * differently depending on the change source, and may be called from + * `PlacesSyncUtils.bookmarks.reset` or + * `PlacesUtils.bookmarks.eraseEverything`. + * + * - RESTORE: The user is restoring from a backup. Drop the sync ID, last + * sync time, and tombstones; reset sync statuses for remaining items to + * "NEW"; then set a flag to wipe the server and all other clients. On the + * next sync, we'll replace their bookmarks with ours. + * + * - RESTORE_ON_STARTUP: Places is automatically restoring from a backup to + * recover from a corrupt database. The sync ID, last sync time, and + * tombstones don't exist, since we don't back them up; reset sync statuses + * for the roots to "UNKNOWN"; but don't wipe the server. On the next sync, + * we'll merge the restored bookmarks with the ones on the server. + * + * - SYNC: Either another client told us to erase our bookmarks + * (`PlacesSyncUtils.bookmarks.wipe`), or the user disconnected Sync + * (`PlacesSyncUtils.bookmarks.reset`). In both cases, drop the existing + * sync ID, last sync time, and tombstones; reset sync statuses for + * remaining items to "NEW"; and don't wipe the server. + * + * @param db + * the Sqlite.sys.mjs connection handle. + * @param source + * the change source constant. + */ + async resetSyncMetadata(db, source) { + if ( + ![ + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE, + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP, + lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + ].includes(source) + ) { + return; + } + + // Remove the sync ID and last sync time in all cases. + await lazy.PlacesUtils.metadata.deleteWithConnection( + db, + BookmarkSyncUtils.SYNC_ID_META_KEY, + BookmarkSyncUtils.LAST_SYNC_META_KEY + ); + + // If we're manually restoring from a backup, wipe the server and other + // clients, so that we replace their bookmarks with the restored tree. If + // we're automatically restoring to recover from a corrupt database, don't + // wipe; we want to merge the restored tree with the one on the server. + await lazy.PlacesUtils.metadata.setWithConnection( + db, + BookmarkSyncUtils.WIPE_REMOTE_META_KEY, + source == lazy.PlacesUtils.bookmarks.SOURCES.RESTORE + ); + + // Reset change counters and sync statuses for roots and remaining + // items, and drop tombstones. + let syncStatus = + source == lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP + ? lazy.PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN + : lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW; + await resetAllSyncStatuses(db, syncStatus); + }, + + /** + * Converts a Places GUID to a Sync record ID. Record IDs are identical to + * Places GUIDs for all items except roots. + */ + guidToRecordId(guid) { + return lazy.ROOT_GUID_TO_RECORD_ID[guid] || guid; + }, + + /** + * Converts a Sync record ID to a Places GUID. + */ + recordIdToGuid(recordId) { + return lazy.ROOT_RECORD_ID_TO_GUID[recordId] || recordId; + }, + + /** + * Fetches the record IDs for a folder's children, ordered by their position + * within the folder. + * Used only be tests - but that includes tps, so it lives here. + */ + fetchChildRecordIds(parentRecordId) { + lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.recordId(parentRecordId); + let parentGuid = BookmarkSyncUtils.recordIdToGuid(parentRecordId); + + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: fetchChildRecordIds", + async function (db) { + let childGuids = await fetchChildGuids(db, parentGuid); + return childGuids.map(guid => BookmarkSyncUtils.guidToRecordId(guid)); + } + ); + }, + + /** + * Migrates an array of `{ recordId, modified }` tuples from the old JSON-based + * tracker to the new sync change counter. `modified` is when the change was + * added to the old tracker, in milliseconds. + * + * Sync calls this method before the first bookmark sync after the Places + * schema migration. + */ + migrateOldTrackerEntries(entries) { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: migrateOldTrackerEntries", + function (db) { + return db.executeTransaction(async function () { + // Mark all existing bookmarks as synced, and clear their change + // counters to avoid a full upload on the next sync. Note that + // this means we'll miss changes made between startup and the first + // post-migration sync, as well as changes made on a new release + // channel that weren't synced before the user downgraded. This is + // unfortunate, but no worse than the behavior of the old tracker. + // + // We also likely have bookmarks that don't exist on the server, + // because the old tracker missed them. We'll eventually fix the + // server once we decide on a repair strategy. + await db.executeCached( + ` + WITH RECURSIVE + syncedItems(id) AS ( + SELECT b.id FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ) + UPDATE moz_bookmarks SET + syncStatus = :syncStatus, + syncChangeCounter = 0 + WHERE id IN syncedItems`, + { syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL } + ); + + await db.executeCached(`DELETE FROM moz_bookmarks_deleted`); + + await db.executeCached(`CREATE TEMP TABLE moz_bookmarks_tracked ( + guid TEXT PRIMARY KEY, + time INTEGER + )`); + + try { + for (let { recordId, modified } of entries) { + let guid = BookmarkSyncUtils.recordIdToGuid(recordId); + if (!lazy.PlacesUtils.isValidGuid(guid)) { + lazy.BookmarkSyncLog.warn( + `migrateOldTrackerEntries: Ignoring ` + + `change for invalid item ${guid}` + ); + continue; + } + let time = lazy.PlacesUtils.toPRTime( + Number.isFinite(modified) ? modified : Date.now() + ); + await db.executeCached( + ` + INSERT OR IGNORE INTO moz_bookmarks_tracked (guid, time) + VALUES (:guid, :time)`, + { guid, time } + ); + } + + // Bump the change counter for existing tracked items. + await db.executeCached(` + INSERT OR REPLACE INTO moz_bookmarks (id, fk, type, parent, + position, title, + dateAdded, lastModified, + guid, syncChangeCounter, + syncStatus) + SELECT b.id, b.fk, b.type, b.parent, b.position, b.title, + b.dateAdded, MAX(b.lastModified, t.time), b.guid, + b.syncChangeCounter + 1, b.syncStatus + FROM moz_bookmarks b + JOIN moz_bookmarks_tracked t ON b.guid = t.guid`); + + // Insert tombstones for nonexistent tracked items, using the most + // recent deletion date for more accurate reconciliation. We assume + // the tracked item belongs to a synced root. + await db.executeCached(` + INSERT OR REPLACE INTO moz_bookmarks_deleted (guid, dateRemoved) + SELECT t.guid, MAX(IFNULL((SELECT dateRemoved FROM moz_bookmarks_deleted + WHERE guid = t.guid), 0), t.time) + FROM moz_bookmarks_tracked t + LEFT JOIN moz_bookmarks b ON t.guid = b.guid + WHERE b.guid IS NULL`); + } finally { + await db.executeCached(`DROP TABLE moz_bookmarks_tracked`); + } + }); + } + ); + }, + + /** + * Reorders a folder's children, based on their order in the array of sync + * IDs. + * + * Sync uses this method to reorder all synced children after applying all + * incoming records. + * + * @return {Promise} resolved when reordering is complete. + * @rejects if an error happens while reordering. + * @throws if the arguments are invalid. + */ + order(parentRecordId, childRecordIds) { + lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS.recordId(parentRecordId); + if (!childRecordIds.length) { + return undefined; + } + let parentGuid = BookmarkSyncUtils.recordIdToGuid(parentRecordId); + if (parentGuid == lazy.PlacesUtils.bookmarks.rootGuid) { + // Reordering roots doesn't make sense, but Sync will do this on the + // first sync. + return undefined; + } + let orderedChildrenGuids = childRecordIds.map( + BookmarkSyncUtils.recordIdToGuid + ); + return lazy.PlacesUtils.bookmarks.reorder( + parentGuid, + orderedChildrenGuids, + { + source: SOURCE_SYNC, + } + ); + }, + + /** + * Resolves to true if there are known sync changes. + */ + havePendingChanges() { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: havePendingChanges", + async function (db) { + let rows = await db.executeCached(` + WITH RECURSIVE + syncedItems(id, guid, syncChangeCounter) AS ( + SELECT b.id, b.guid, b.syncChangeCounter + FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id, b.guid, b.syncChangeCounter + FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ), + changedItems(guid) AS ( + SELECT guid FROM syncedItems + WHERE syncChangeCounter >= 1 + UNION ALL + SELECT guid FROM moz_bookmarks_deleted + ) + SELECT EXISTS(SELECT guid FROM changedItems) AS haveChanges`); + return !!rows[0].getResultByName("haveChanges"); + } + ); + }, + + /** + * Returns a changeset containing local bookmark changes since the last sync. + * + * @return {Promise} resolved once all items have been fetched. + * @resolves to an object containing records for changed bookmarks, keyed by + * the record ID. + * @see pullSyncChanges for the implementation, and markChangesAsSyncing for + * an explanation of why we update the sync status. + */ + pullChanges() { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: pullChanges", + pullSyncChanges + ); + }, + + /** + * Updates the sync status of all "NEW" bookmarks to "NORMAL", so that Sync + * can recover correctly after an interrupted sync. + * + * @param changeRecords + * A changeset containing sync change records, as returned by + * `pullChanges`. + * @return {Promise} resolved once all records have been updated. + */ + markChangesAsSyncing(changeRecords) { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: markChangesAsSyncing", + db => markChangesAsSyncing(db, changeRecords) + ); + }, + + /** + * Decrements the sync change counter, updates the sync status, and cleans up + * tombstones for successfully synced items. Sync calls this method at the + * end of each bookmark sync. + * + * @param changeRecords + * A changeset containing sync change records, as returned by + * `pullChanges`. + * @return {Promise} resolved once all records have been updated. + */ + pushChanges(changeRecords) { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: pushChanges", + async function (db) { + let skippedCount = 0; + let weakCount = 0; + let updateParams = []; + let tombstoneGuidsToRemove = []; + + for (let recordId in changeRecords) { + // Validate change records to catch coding errors. + let changeRecord = validateChangeRecord( + "BookmarkSyncUtils: pushChanges", + changeRecords[recordId], + { + tombstone: { required: true }, + counter: { required: true }, + synced: { required: true }, + } + ); + + // Skip weakly uploaded records. + if (!changeRecord.counter) { + weakCount++; + continue; + } + + // Sync sets the `synced` flag for reconciled or successfully + // uploaded items. If upload failed, ignore the change; we'll + // try again on the next sync. + if (!changeRecord.synced) { + skippedCount++; + continue; + } + + let guid = BookmarkSyncUtils.recordIdToGuid(recordId); + if (changeRecord.tombstone) { + tombstoneGuidsToRemove.push(guid); + } else { + updateParams.push({ + guid, + syncChangeDelta: changeRecord.counter, + syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + } + } + + // Reduce the change counter and update the sync status for + // reconciled and uploaded items. If the bookmark was updated + // during the sync, its change counter will still be > 0 for the + // next sync. + if (updateParams.length || tombstoneGuidsToRemove.length) { + await db.executeTransaction(async function () { + if (updateParams.length) { + await db.executeCached( + ` + UPDATE moz_bookmarks + SET syncChangeCounter = MAX(syncChangeCounter - :syncChangeDelta, 0), + syncStatus = :syncStatus + WHERE guid = :guid`, + updateParams + ); + // and if there are *both* bookmarks and tombstones for these + // items, we nuke the tombstones. + // This should be unlikely, but bad if it happens. + let dupedGuids = updateParams.map(({ guid }) => guid); + await removeUndeletedTombstones(db, dupedGuids); + } + await removeTombstones(db, tombstoneGuidsToRemove); + }); + } + + lazy.BookmarkSyncLog.debug(`pushChanges: Processed change records`, { + weak: weakCount, + skipped: skippedCount, + updated: updateParams.length, + }); + } + ); + }, + + /** + * Removes items from the database. Sync buffers incoming tombstones, and + * calls this method to apply them at the end of each sync. Deletion + * happens in three steps: + * + * 1. Remove all non-folder items. Deleting a folder on a remote client + * uploads tombstones for the folder and its children at the time of + * deletion. This preserves any new children we've added locally since + * the last sync. + * 2. Reparent remaining children to the tombstoned folder's parent. This + * bumps the change counter for the children and their new parent. + * 3. Remove the tombstoned folder. Because we don't do this in a + * transaction, the user might move new items into the folder before we + * can remove it. In that case, we keep the folder and upload the new + * subtree to the server. + * + * See the comment above `BookmarksStore::deletePending` for the details on + * why delete works the way it does. + */ + remove(recordIds) { + if (!recordIds.length) { + return null; + } + + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: remove", + async function (db) { + let folderGuids = []; + for (let recordId of recordIds) { + if (recordId in lazy.ROOT_RECORD_ID_TO_GUID) { + lazy.BookmarkSyncLog.warn( + `remove: Refusing to remove root ${recordId}` + ); + continue; + } + let guid = BookmarkSyncUtils.recordIdToGuid(recordId); + let bookmarkItem = await lazy.PlacesUtils.bookmarks.fetch(guid); + if (!bookmarkItem) { + lazy.BookmarkSyncLog.trace(`remove: Item ${guid} already removed`); + continue; + } + let kind = await getKindForItem(db, bookmarkItem); + if (kind == BookmarkSyncUtils.KINDS.FOLDER) { + folderGuids.push(bookmarkItem.guid); + continue; + } + let wasRemoved = await deleteSyncedAtom(bookmarkItem); + if (wasRemoved) { + lazy.BookmarkSyncLog.trace( + `remove: Removed item ${guid} with kind ${kind}` + ); + } + } + + for (let guid of folderGuids) { + let bookmarkItem = await lazy.PlacesUtils.bookmarks.fetch(guid); + if (!bookmarkItem) { + lazy.BookmarkSyncLog.trace( + `remove: Folder ${guid} already removed` + ); + continue; + } + let wasRemoved = await deleteSyncedFolder(db, bookmarkItem); + if (wasRemoved) { + lazy.BookmarkSyncLog.trace( + `remove: Removed folder ${bookmarkItem.guid}` + ); + } + } + + // TODO (Bug 1313890): Refactor the bookmarks engine to pull change records + // before uploading, instead of returning records to merge into the engine's + // initial changeset. + return pullSyncChanges(db); + } + ); + }, + + /** + * Removes all bookmarks and tombstones from the database. Sync calls this + * method when it receives a command from a remote client to wipe all stored + * data. + * + * @return {Promise} resolved once all items have been removed. + */ + wipe() { + return lazy.PlacesUtils.bookmarks.eraseEverything({ + source: SOURCE_SYNC, + }); + }, + + /** + * Marks all bookmarks as "NEW" and removes all tombstones. Unlike `wipe`, + * this keeps all existing bookmarks, and only clears their sync change + * tracking info. + * + * @return {Promise} resolved once all items have been updated. + */ + reset() { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: reset", + function (db) { + return db.executeTransaction(async function () { + await BookmarkSyncUtils.resetSyncMetadata(db, SOURCE_SYNC); + }); + } + ); + }, + + /** + * Fetches a Sync bookmark object for an item in the tree. + * + * Should only be used by SYNC TESTS. + * We should remove this in bug XXXXXX, updating the tests to use + * PlacesUtils.bookmarks.fetch. + * + * The object contains + * the following properties, depending on the item's kind: + * + * - kind (all): A string representing the item's kind. + * - recordId (all): The item's record ID. + * - parentRecordId (all): The record ID of the item's parent. + * - parentTitle (all): The title of the item's parent, used for de-duping. + * Omitted for the Places root and parents with empty titles. + * - dateAdded (all): Timestamp in milliseconds, when the bookmark was added + * or created on a remote device if known. + * - title ("bookmark", "folder", "query"): The item's title. + * Omitted if empty. + * - url ("bookmark", "query"): The item's URL. + * - tags ("bookmark", "query"): An array containing the item's tags. + * - keyword ("bookmark"): The bookmark's keyword, if one exists. + * - childRecordIds ("folder"): An array containing the record IDs of the item's + * children, used to determine child order. + * - folder ("query"): The tag folder name, if this is a tag query. + * - index ("separator"): The separator's position within its parent. + */ + async fetch(recordId) { + let guid = BookmarkSyncUtils.recordIdToGuid(recordId); + let bookmarkItem = await lazy.PlacesUtils.bookmarks.fetch(guid); + if (!bookmarkItem) { + return null; + } + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: fetch", + async function (db) { + // Convert the Places bookmark object to a Sync bookmark and add + // kind-specific properties. Titles are required for bookmarks, + // and folders; optional for queries, and omitted for separators. + let kind = await getKindForItem(db, bookmarkItem); + let item; + switch (kind) { + case BookmarkSyncUtils.KINDS.BOOKMARK: + item = await fetchBookmarkItem(db, bookmarkItem); + break; + + case BookmarkSyncUtils.KINDS.QUERY: + item = await fetchQueryItem(db, bookmarkItem); + break; + + case BookmarkSyncUtils.KINDS.FOLDER: + item = await fetchFolderItem(db, bookmarkItem); + break; + + case BookmarkSyncUtils.KINDS.SEPARATOR: + item = await placesBookmarkToSyncBookmark(db, bookmarkItem); + item.index = bookmarkItem.index; + break; + + default: + throw new Error(`Unknown bookmark kind: ${kind}`); + } + + // Sync uses the parent title for de-duping. All Sync bookmark objects + // except the Places root should have this property. + if (bookmarkItem.parentGuid) { + let parent = await lazy.PlacesUtils.bookmarks.fetch( + bookmarkItem.parentGuid + ); + item.parentTitle = parent.title || ""; + } + + return item; + } + ); + }, + + /** + * Returns the sync change counter increment for a change source constant. + */ + determineSyncChangeDelta(source) { + // Don't bump the change counter when applying changes made by Sync, to + // avoid sync loops. + return source == lazy.PlacesUtils.bookmarks.SOURCES.SYNC ? 0 : 1; + }, + + /** + * Returns the sync status for a new item inserted by a change source. + */ + determineInitialSyncStatus(source) { + if (source == lazy.PlacesUtils.bookmarks.SOURCES.SYNC) { + // Incoming bookmarks are "NORMAL", since they already exist on the server. + return lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL; + } + if (source == lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP) { + // If the user restores from a backup, or Places automatically recovers + // from a corrupt database, all prior sync tracking is lost. Setting the + // status to "UNKNOWN" allows Sync to reconcile restored bookmarks with + // those on the server. + return lazy.PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN; + } + // For all other sources, mark items as "NEW". We'll update their statuses + // to "NORMAL" after the first sync. + return lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW; + }, + + /** + * An internal helper that bumps the change counter for all bookmarks with + * a given URL. This is used to update bookmarks when adding or changing a + * tag or keyword entry. + * + * @param db + * the Sqlite.sys.mjs connection handle. + * @param url + * the bookmark URL object. + * @param syncChangeDelta + * the sync change counter increment. + * @return {Promise} resolved when the counters have been updated. + */ + addSyncChangesForBookmarksWithURL(db, url, syncChangeDelta) { + if (!url || !syncChangeDelta) { + return Promise.resolve(); + } + return db.executeCached( + ` + UPDATE moz_bookmarks + SET syncChangeCounter = syncChangeCounter + :syncChangeDelta + WHERE type = :type AND + fk = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND + url = :url)`, + { + syncChangeDelta, + type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: url.href, + } + ); + }, + + /** + * Returns `0` if no sensible timestamp could be found. + * Otherwise, returns the earliest sensible timestamp between `existingMillis` + * and `serverMillis`. + */ + ratchetTimestampBackwards( + existingMillis, + serverMillis, + lowerBound = BookmarkSyncUtils.EARLIEST_BOOKMARK_TIMESTAMP + ) { + const possible = [+existingMillis, +serverMillis].filter( + n => !isNaN(n) && n > lowerBound + ); + if (!possible.length) { + return 0; + } + return Math.min(...possible); + }, + + /** + * Rebuilds the left pane query for the mobile root under "All Bookmarks" if + * necessary. Sync calls this method at the end of each bookmark sync. This + * code should eventually move to `PlacesUIUtils#maybeRebuildLeftPane`; see + * bug 647605. + * + * - If there are no mobile bookmarks, the query will not be created, or + * will be removed if it already exists. + * - If there are mobile bookmarks, the query will be created if it doesn't + * exist, or will be updated with the correct title and URL otherwise. + */ + async ensureMobileQuery() { + let db = await lazy.PlacesUtils.promiseDBConnection(); + + let mobileChildGuids = await fetchChildGuids( + db, + lazy.PlacesUtils.bookmarks.mobileGuid + ); + let hasMobileBookmarks = !!mobileChildGuids.length; + + Services.prefs.setBoolPref(MOBILE_BOOKMARKS_PREF, hasMobileBookmarks); + }, +})); + +PlacesSyncUtils.test = {}; +PlacesSyncUtils.test.bookmarks = Object.freeze({ + /** + * Inserts a synced bookmark into the tree. Only SYNC TESTS should call this + * method; other callers should use `PlacesUtils.bookmarks.insert`. + * + * It is in this file rather than a test-only file because it makes use of + * other internal functions here, so moving is not trivial - see bug 1662602. + * + * The following properties are supported: + * - kind: Required. + * - guid: Required. + * - parentGuid: Required. + * - url: Required for bookmarks. + * - tags: An optional array of tag strings. + * - keyword: An optional keyword string. + * + * Sync doesn't set the index, since it appends and reorders children + * after applying all incoming items. + * + * @param info + * object representing a synced bookmark. + * + * @return {Promise} resolved when the creation is complete. + * @resolves to an object representing the created bookmark. + * @rejects if it's not possible to create the requested bookmark. + * @throws if the arguments are invalid. + */ + insert(info) { + let insertInfo = validateNewBookmark("BookmarkTestUtils: insert", info); + + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkTestUtils: insert", + async db => { + // If we're inserting a tag query, make sure the tag exists and fix the + // folder ID to refer to the local tag folder. + insertInfo = await updateTagQueryFolder(db, insertInfo); + + let bookmarkInfo = syncBookmarkToPlacesBookmark(insertInfo); + let bookmarkItem = await lazy.PlacesUtils.bookmarks.insert( + bookmarkInfo + ); + let newItem = await insertBookmarkMetadata( + db, + bookmarkItem, + insertInfo + ); + + return newItem; + } + ); + }, +}); + +XPCOMUtils.defineLazyGetter(lazy, "HistorySyncLog", () => { + return lazy.Log.repository.getLogger("Sync.Engine.History.HistorySyncUtils"); +}); + +XPCOMUtils.defineLazyGetter(lazy, "BookmarkSyncLog", () => { + // Use a sub-log of the bookmarks engine, so setting the level for that + // engine also adjust the level of this log. + return lazy.Log.repository.getLogger( + "Sync.Engine.Bookmarks.BookmarkSyncUtils" + ); +}); + +function validateSyncBookmarkObject(name, input, behavior) { + return lazy.PlacesUtils.validateItemProperties( + name, + lazy.PlacesUtils.SYNC_BOOKMARK_VALIDATORS, + input, + behavior + ); +} + +// Validates a sync change record as returned by `pullChanges` and passed to +// `pushChanges`. +function validateChangeRecord(name, changeRecord, behavior) { + return lazy.PlacesUtils.validateItemProperties( + name, + lazy.PlacesUtils.SYNC_CHANGE_RECORD_VALIDATORS, + changeRecord, + behavior + ); +} + +// Similar to the private `fetchBookmarksByParent` implementation in +// `Bookmarks.jsm`. +var fetchChildGuids = async function (db, parentGuid) { + let rows = await db.executeCached( + ` + SELECT guid + FROM moz_bookmarks + WHERE parent = ( + SELECT id FROM moz_bookmarks WHERE guid = :parentGuid + ) + ORDER BY position`, + { parentGuid } + ); + return rows.map(row => row.getResultByName("guid")); +}; + +// Legacy tag queries may use a `place:` URL that refers to the tag folder ID. +// When we apply a synced tag query from a remote client, we need to update the +// URL to point to the local tag. +function updateTagQueryFolder(db, info) { + if ( + info.kind != BookmarkSyncUtils.KINDS.QUERY || + !info.folder || + !info.url || + info.url.protocol != "place:" + ) { + return info; + } + + let params = new URLSearchParams(info.url.pathname); + let type = +params.get("type"); + if (type != Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + return info; + } + + lazy.BookmarkSyncLog.debug( + `updateTagQueryFolder: Tag query folder: ${info.folder}` + ); + + // Rewrite the query to directly reference the tag. + params.delete("queryType"); + params.delete("type"); + params.delete("folder"); + params.set("tag", info.folder); + info.url = new URL(info.url.protocol + params); + return info; +} + +// Keywords are a 1 to 1 mapping between strings and pairs of (URL, postData). +// (the postData is not synced, so we ignore it). Sync associates keywords with +// bookmarks, which is not really accurate. -- We might already have a keyword +// with that name, or we might already have another bookmark with that URL with +// a different keyword, etc. +// +// If we don't handle those cases by removing the conflicting keywords first, +// the insertion will fail, and the keywords will either be wrong, or missing. +// This function handles those cases. +function removeConflictingKeywords(bookmarkURL, newKeyword) { + return lazy.PlacesUtils.withConnectionWrapper( + "BookmarkSyncUtils: removeConflictingKeywords", + async function (db) { + let entryForURL = await lazy.PlacesUtils.keywords.fetch({ + url: bookmarkURL.href, + }); + if (entryForURL && entryForURL.keyword !== newKeyword) { + await lazy.PlacesUtils.keywords.remove({ + keyword: entryForURL.keyword, + source: SOURCE_SYNC, + }); + // This will cause us to reupload this record for this sync, but + // without it, we will risk data corruption. + await BookmarkSyncUtils.addSyncChangesForBookmarksWithURL( + db, + entryForURL.url, + 1 + ); + } + if (!newKeyword) { + return; + } + let entryForNewKeyword = await lazy.PlacesUtils.keywords.fetch({ + keyword: newKeyword, + }); + if (entryForNewKeyword) { + await lazy.PlacesUtils.keywords.remove({ + keyword: entryForNewKeyword.keyword, + source: SOURCE_SYNC, + }); + await BookmarkSyncUtils.addSyncChangesForBookmarksWithURL( + db, + entryForNewKeyword.url, + 1 + ); + } + } + ); +} + +// Sets annotations, keywords, and tags on a new bookmark. Returns a Sync +// bookmark object. +async function insertBookmarkMetadata(db, bookmarkItem, insertInfo) { + let newItem = await placesBookmarkToSyncBookmark(db, bookmarkItem); + + try { + newItem.tags = tagItem(bookmarkItem, insertInfo.tags); + } catch (ex) { + lazy.BookmarkSyncLog.warn( + `insertBookmarkMetadata: Error tagging item ${insertInfo.recordId}`, + ex + ); + } + + if (insertInfo.keyword) { + await removeConflictingKeywords(bookmarkItem.url, insertInfo.keyword); + await lazy.PlacesUtils.keywords.insert({ + keyword: insertInfo.keyword, + url: bookmarkItem.url.href, + source: SOURCE_SYNC, + }); + newItem.keyword = insertInfo.keyword; + } + + return newItem; +} + +// Determines the Sync record kind for an existing bookmark. +async function getKindForItem(db, item) { + switch (item.type) { + case lazy.PlacesUtils.bookmarks.TYPE_FOLDER: { + return BookmarkSyncUtils.KINDS.FOLDER; + } + case lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK: + return item.url.protocol == "place:" + ? BookmarkSyncUtils.KINDS.QUERY + : BookmarkSyncUtils.KINDS.BOOKMARK; + + case lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR: + return BookmarkSyncUtils.KINDS.SEPARATOR; + } + return null; +} + +// Returns the `nsINavBookmarksService` bookmark type constant for a Sync +// record kind. +function getTypeForKind(kind) { + switch (kind) { + case BookmarkSyncUtils.KINDS.BOOKMARK: + case BookmarkSyncUtils.KINDS.QUERY: + return lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK; + + case BookmarkSyncUtils.KINDS.FOLDER: + return lazy.PlacesUtils.bookmarks.TYPE_FOLDER; + + case BookmarkSyncUtils.KINDS.SEPARATOR: + return lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR; + } + throw new Error(`Unknown bookmark kind: ${kind}`); +} + +function validateNewBookmark(name, info) { + let insertInfo = validateSyncBookmarkObject(name, info, { + kind: { required: true }, + recordId: { required: true }, + url: { + requiredIf: b => + [ + BookmarkSyncUtils.KINDS.BOOKMARK, + BookmarkSyncUtils.KINDS.QUERY, + ].includes(b.kind), + validIf: b => + [ + BookmarkSyncUtils.KINDS.BOOKMARK, + BookmarkSyncUtils.KINDS.QUERY, + ].includes(b.kind), + }, + parentRecordId: { required: true }, + title: { + validIf: b => + [ + BookmarkSyncUtils.KINDS.BOOKMARK, + BookmarkSyncUtils.KINDS.QUERY, + BookmarkSyncUtils.KINDS.FOLDER, + ].includes(b.kind) || b.title === "", + }, + query: { validIf: b => b.kind == BookmarkSyncUtils.KINDS.QUERY }, + folder: { validIf: b => b.kind == BookmarkSyncUtils.KINDS.QUERY }, + tags: { + validIf: b => + [ + BookmarkSyncUtils.KINDS.BOOKMARK, + BookmarkSyncUtils.KINDS.QUERY, + ].includes(b.kind), + }, + keyword: { + validIf: b => + [ + BookmarkSyncUtils.KINDS.BOOKMARK, + BookmarkSyncUtils.KINDS.QUERY, + ].includes(b.kind), + }, + dateAdded: { required: false }, + }); + + return insertInfo; +} + +function tagItem(item, tags) { + if (!item.url) { + return []; + } + + // Remove leading and trailing whitespace, then filter out empty tags. + let newTags = tags ? tags.map(tag => tag.trim()).filter(Boolean) : []; + + // Removing the last tagged item will also remove the tag. To preserve + // tag IDs, we temporarily tag a dummy URI, ensuring the tags exist. + let dummyURI = lazy.PlacesUtils.toURI("about:weave#BStore_tagURI"); + let bookmarkURI = lazy.PlacesUtils.toURI(item.url); + if (newTags && newTags.length) { + lazy.PlacesUtils.tagging.tagURI(dummyURI, newTags, SOURCE_SYNC); + } + lazy.PlacesUtils.tagging.untagURI(bookmarkURI, null, SOURCE_SYNC); + if (newTags && newTags.length) { + lazy.PlacesUtils.tagging.tagURI(bookmarkURI, newTags, SOURCE_SYNC); + } + lazy.PlacesUtils.tagging.untagURI(dummyURI, null, SOURCE_SYNC); + + return newTags; +} + +// Converts a Places bookmark to a Sync bookmark. This function maps Places +// GUIDs to record IDs and filters out extra Places properties like date added, +// last modified, and index. +async function placesBookmarkToSyncBookmark(db, bookmarkItem) { + let item = {}; + + for (let prop in bookmarkItem) { + switch (prop) { + // Record IDs are identical to Places GUIDs for all items except roots. + case "guid": + item.recordId = BookmarkSyncUtils.guidToRecordId(bookmarkItem.guid); + break; + + case "parentGuid": + item.parentRecordId = BookmarkSyncUtils.guidToRecordId( + bookmarkItem.parentGuid + ); + break; + + // Sync uses kinds instead of types, which distinguish between folders, + // livemarks, bookmarks, and queries. + case "type": + item.kind = await getKindForItem(db, bookmarkItem); + break; + + case "title": + case "url": + item[prop] = bookmarkItem[prop]; + break; + + case "dateAdded": + item[prop] = new Date(bookmarkItem[prop]).getTime(); + break; + } + } + + return item; +} + +// Converts a Sync bookmark object to a Places bookmark or livemark object. +// This function maps record IDs to Places GUIDs, and filters out extra Sync +// properties like keywords, tags. Returns an object that can be passed to +// `PlacesUtils.bookmarks.{insert, update}`. +function syncBookmarkToPlacesBookmark(info) { + let bookmarkInfo = { + source: SOURCE_SYNC, + }; + + for (let prop in info) { + switch (prop) { + case "kind": + bookmarkInfo.type = getTypeForKind(info.kind); + break; + + // Convert record IDs to Places GUIDs for roots. + case "recordId": + bookmarkInfo.guid = BookmarkSyncUtils.recordIdToGuid(info.recordId); + break; + + case "dateAdded": + bookmarkInfo.dateAdded = new Date(info.dateAdded); + break; + + case "parentRecordId": + bookmarkInfo.parentGuid = BookmarkSyncUtils.recordIdToGuid( + info.parentRecordId + ); + // Instead of providing an index, Sync reorders children at the end of + // the sync using `BookmarkSyncUtils.order`. We explicitly specify the + // default index here to prevent `PlacesUtils.bookmarks.update` from + // throwing. + bookmarkInfo.index = lazy.PlacesUtils.bookmarks.DEFAULT_INDEX; + break; + + case "title": + case "url": + bookmarkInfo[prop] = info[prop]; + break; + } + } + + return bookmarkInfo; +} + +// Creates and returns a Sync bookmark object containing the bookmark's +// tags, keyword. +var fetchBookmarkItem = async function (db, bookmarkItem) { + let item = await placesBookmarkToSyncBookmark(db, bookmarkItem); + + if (!item.title) { + item.title = ""; + } + + item.tags = lazy.PlacesUtils.tagging.getTagsForURI( + lazy.PlacesUtils.toURI(bookmarkItem.url) + ); + + let keywordEntry = await lazy.PlacesUtils.keywords.fetch({ + url: bookmarkItem.url, + }); + if (keywordEntry) { + item.keyword = keywordEntry.keyword; + } + + return item; +}; + +// Creates and returns a Sync bookmark object containing the folder's children. +async function fetchFolderItem(db, bookmarkItem) { + let item = await placesBookmarkToSyncBookmark(db, bookmarkItem); + + if (!item.title) { + item.title = ""; + } + + let childGuids = await fetchChildGuids(db, bookmarkItem.guid); + item.childRecordIds = childGuids.map(guid => + BookmarkSyncUtils.guidToRecordId(guid) + ); + + return item; +} + +// Creates and returns a Sync bookmark object containing the query's tag +// folder name. +async function fetchQueryItem(db, bookmarkItem) { + let item = await placesBookmarkToSyncBookmark(db, bookmarkItem); + + let params = new URLSearchParams(bookmarkItem.url.pathname); + let tags = params.getAll("tag"); + if (tags.length == 1) { + item.folder = tags[0]; + } + + return item; +} + +function addRowToChangeRecords(row, changeRecords) { + let guid = row.getResultByName("guid"); + if (!guid) { + throw new Error(`Changed item missing GUID`); + } + let isTombstone = !!row.getResultByName("tombstone"); + let recordId = BookmarkSyncUtils.guidToRecordId(guid); + if (recordId in changeRecords) { + let existingRecord = changeRecords[recordId]; + if (existingRecord.tombstone == isTombstone) { + // Should never happen: `moz_bookmarks.guid` has a unique index, and + // `moz_bookmarks_deleted.guid` is the primary key. + throw new Error(`Duplicate item or tombstone ${recordId} in changeset`); + } + if (!existingRecord.tombstone && isTombstone) { + // Don't replace undeleted items with tombstones... + lazy.BookmarkSyncLog.warn( + "addRowToChangeRecords: Ignoring tombstone for undeleted item", + recordId + ); + return; + } + // ...But replace undeleted tombstones with items. + lazy.BookmarkSyncLog.warn( + "addRowToChangeRecords: Replacing tombstone for undeleted item", + recordId + ); + } + let modifiedAsPRTime = row.getResultByName("modified"); + let modified = modifiedAsPRTime / MICROSECONDS_PER_SECOND; + if (Number.isNaN(modified) || modified <= 0) { + lazy.BookmarkSyncLog.error( + "addRowToChangeRecords: Invalid modified date for " + recordId, + modifiedAsPRTime + ); + modified = 0; + } + changeRecords[recordId] = { + modified, + counter: row.getResultByName("syncChangeCounter"), + status: row.getResultByName("syncStatus"), + tombstone: isTombstone, + synced: false, + }; +} + +/** + * Queries the database for synced bookmarks and tombstones, and returns a + * changeset for the Sync bookmarks engine. + * + * @param db + * The Sqlite.sys.mjs connection handle. + * @param forGuids + * Fetch Sync tracking information for only the requested GUIDs. + * @return {Promise} resolved once all items have been fetched. + * @resolves to an object containing records for changed bookmarks, keyed by + * the record ID. + */ +var pullSyncChanges = async function (db, forGuids = []) { + let changeRecords = {}; + + let itemConditions = ["syncChangeCounter >= 1"]; + let tombstoneConditions = ["1 = 1"]; + if (forGuids.length) { + let restrictToGuids = `guid IN (${forGuids + .map(guid => JSON.stringify(guid)) + .join(",")})`; + itemConditions.push(restrictToGuids); + tombstoneConditions.push(restrictToGuids); + } + + let rows = await db.executeCached( + ` + WITH RECURSIVE + syncedItems(id, guid, modified, syncChangeCounter, syncStatus) AS ( + SELECT b.id, b.guid, b.lastModified, b.syncChangeCounter, b.syncStatus + FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id, b.guid, b.lastModified, b.syncChangeCounter, b.syncStatus + FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ) + SELECT guid, modified, syncChangeCounter, syncStatus, 0 AS tombstone + FROM syncedItems + WHERE ${itemConditions.join(" AND ")} + UNION ALL + SELECT guid, dateRemoved AS modified, 1 AS syncChangeCounter, + :deletedSyncStatus, 1 AS tombstone + FROM moz_bookmarks_deleted + WHERE ${tombstoneConditions.join(" AND ")}`, + { deletedSyncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL } + ); + for (let row of rows) { + addRowToChangeRecords(row, changeRecords); + } + + return changeRecords; +}; + +// Moves a synced folder's remaining children to its parent, and deletes the +// folder if it's empty. +async function deleteSyncedFolder(db, bookmarkItem) { + // At this point, any member in the folder that remains is either a folder + // pending deletion (which we'll get to in this function), or an item that + // should not be deleted. To avoid deleting these items, we first move them + // to the parent of the folder we're about to delete. + let childGuids = await fetchChildGuids(db, bookmarkItem.guid); + if (!childGuids.length) { + // No children -- just delete the folder. + return deleteSyncedAtom(bookmarkItem); + } + + if (lazy.BookmarkSyncLog.level <= lazy.Log.Level.Trace) { + lazy.BookmarkSyncLog.trace( + `deleteSyncedFolder: Moving ${JSON.stringify(childGuids)} children of ` + + `"${bookmarkItem.guid}" to grandparent + "${BookmarkSyncUtils.guidToRecordId(bookmarkItem.parentGuid)}" before ` + + `deletion` + ); + } + + // Move children out of the parent and into the grandparent + for (let guid of childGuids) { + await lazy.PlacesUtils.bookmarks.update({ + guid, + parentGuid: bookmarkItem.parentGuid, + index: lazy.PlacesUtils.bookmarks.DEFAULT_INDEX, + // `SYNC_REPARENT_REMOVED_FOLDER_CHILDREN` bumps the change counter for + // the child and its new parent, without incrementing the bookmark + // tracker's score. + // + // We intentionally don't check if the child is one we'll remove later, + // so it's possible we'll bump the change counter of the closest living + // ancestor when it's not needed. This avoids inconsistency if removal + // is interrupted, since we don't run this operation in a transaction. + source: + lazy.PlacesUtils.bookmarks.SOURCES + .SYNC_REPARENT_REMOVED_FOLDER_CHILDREN, + }); + } + + // Delete the (now empty) parent + try { + await lazy.PlacesUtils.bookmarks.remove(bookmarkItem.guid, { + preventRemovalOfNonEmptyFolders: true, + // We don't want to bump the change counter for this deletion, because + // a tombstone for the folder is already on the server. + source: SOURCE_SYNC, + }); + } catch (e) { + // We failed, probably because someone added something to this folder + // between when we got the children and now (or the database is corrupt, + // or something else happened...) This is unlikely, but possible. To + // avoid corruption in this case, we need to reupload the record to the + // server. + // + // (Ideally this whole operation would be done in a transaction, and this + // wouldn't be possible). + lazy.BookmarkSyncLog.trace( + `deleteSyncedFolder: Error removing parent ` + + `${bookmarkItem.guid} after reparenting children`, + e + ); + return false; + } + + return true; +} + +// Removes a synced bookmark or empty folder from the database. +var deleteSyncedAtom = async function (bookmarkItem) { + try { + await lazy.PlacesUtils.bookmarks.remove(bookmarkItem.guid, { + preventRemovalOfNonEmptyFolders: true, + source: SOURCE_SYNC, + }); + } catch (ex) { + // Likely already removed. + lazy.BookmarkSyncLog.trace( + `deleteSyncedAtom: Error removing ` + bookmarkItem.guid, + ex + ); + return false; + } + + return true; +}; + +/** + * Updates the sync status on all "NEW" and "UNKNOWN" bookmarks to "NORMAL". + * + * We do this when pulling changes instead of in `pushChanges` to make sure + * we write tombstones if a new item is deleted after an interrupted sync. (For + * example, if a "NEW" record is uploaded or reconciled, then the app is closed + * before Sync calls `pushChanges`). + */ +function markChangesAsSyncing(db, changeRecords) { + let unsyncedGuids = []; + for (let recordId in changeRecords) { + if (changeRecords[recordId].tombstone) { + continue; + } + if ( + changeRecords[recordId].status == + lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL + ) { + continue; + } + let guid = BookmarkSyncUtils.recordIdToGuid(recordId); + unsyncedGuids.push(JSON.stringify(guid)); + } + if (!unsyncedGuids.length) { + return Promise.resolve(); + } + return db.execute( + ` + UPDATE moz_bookmarks + SET syncStatus = :syncStatus + WHERE guid IN (${unsyncedGuids.join(",")})`, + { syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL } + ); +} + +/** + * Removes tombstones for successfully synced items. + * + * @return {Promise} + */ +var removeTombstones = function (db, guids) { + if (!guids.length) { + return Promise.resolve(); + } + return db.execute(` + DELETE FROM moz_bookmarks_deleted + WHERE guid IN (${guids.map(guid => JSON.stringify(guid)).join(",")})`); +}; + +/** + * Removes tombstones for successfully synced items where the specified GUID + * exists in *both* the bookmarks and tombstones tables. + * + * @return {Promise} + */ +var removeUndeletedTombstones = function (db, guids) { + if (!guids.length) { + return Promise.resolve(); + } + // sqlite can't join in a DELETE, so we use a subquery. + return db.execute(` + DELETE FROM moz_bookmarks_deleted + WHERE guid IN (${guids.map(guid => JSON.stringify(guid)).join(",")}) + AND guid IN (SELECT guid from moz_bookmarks)`); +}; + +// Sets the history sync ID and clears the last sync time. +async function setHistorySyncId(db, newSyncId) { + await lazy.PlacesUtils.metadata.setWithConnection( + db, + HistorySyncUtils.SYNC_ID_META_KEY, + newSyncId + ); + + await lazy.PlacesUtils.metadata.deleteWithConnection( + db, + HistorySyncUtils.LAST_SYNC_META_KEY + ); +} + +// Sets the bookmarks sync ID and clears the last sync time. +async function setBookmarksSyncId(db, newSyncId) { + await lazy.PlacesUtils.metadata.setWithConnection( + db, + BookmarkSyncUtils.SYNC_ID_META_KEY, + newSyncId + ); + + await lazy.PlacesUtils.metadata.deleteWithConnection( + db, + BookmarkSyncUtils.LAST_SYNC_META_KEY, + BookmarkSyncUtils.WIPE_REMOTE_META_KEY + ); +} + +// Bumps the change counter and sets the given sync status for all bookmarks, +// and drops stale tombstones. +async function resetAllSyncStatuses(db, syncStatus) { + await db.execute( + ` + UPDATE moz_bookmarks + SET syncChangeCounter = 1, + syncStatus = :syncStatus`, + { syncStatus } + ); + + // Drop stale tombstones. + await db.execute("DELETE FROM moz_bookmarks_deleted"); +} diff --git a/toolkit/components/places/PlacesTransactions.sys.mjs b/toolkit/components/places/PlacesTransactions.sys.mjs new file mode 100644 index 0000000000..3db56d2440 --- /dev/null +++ b/toolkit/components/places/PlacesTransactions.sys.mjs @@ -0,0 +1,1725 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Overview + * -------- + * This modules serves as the transactions manager for Places (hereinafter PTM). + * It implements all the elementary transactions for its UI commands: creating + * items, editing their various properties, and so forth. + * + * Note that since the effect of invoking a Places command is not limited to the + * window in which it was performed (e.g. a folder created in the Library may be + * the parent of a bookmark created in some browser window), PTM is a singleton. + * It's therefore unnecessary to initialize PTM in any way apart importing this + * module. + * + * PTM shares most of its semantics with common command pattern implementations. + * However, the asynchronous design of contemporary and future APIs, combined + * with the commitment to serialize all UI operations, does make things a little + * bit different. For example, when |undo| is called in order to undo the top + * undo entry, the caller cannot tell for sure what entry would it be, because + * the execution of some transactions is either in process, or enqueued to be. + * + * Also note that unlike the nsITransactionManager, for example, this API is by + * no means generic. That is, it cannot be used to execute anything but the + * elementary transactions implemented here (Please file a bug if you find + * anything uncovered). More-complex transactions (e.g. creating a folder and + * moving a bookmark into it) may be implemented as a batch (see below). + * + * A note about GUIDs and item-ids + * ------------------------------- + * There's an ongoing effort (see bug 1071511) to deprecate item-ids in Places + * in favor of GUIDs. Both because new APIs (e.g. Bookmark.jsm) expose them to + * the minimum necessary, and because GUIDs play much better with implementing + * |redo|, this API doesn't support item-ids at all, and only accepts bookmark + * GUIDs, both for input (e.g. for setting the parent folder for a new bookmark) + * and for output (when the GUID for such a bookmark is propagated). + * + * Should you need to convert GUIDs to item-ids, use PlacesUtils.promiseItemId. + * + * Constructing transactions + * ------------------------- + * At the bottom of this module you will find transactions for all Places UI + * commands. They are exposed as constructors set on the PlacesTransactions + * object (e.g. PlacesTransactions.NewFolder). The input for this constructors + * is taken in the form of a single argument, a plain object consisting of the + * properties for the transaction. Input properties may be either required or + * optional (for example, |keyword| is required for the EditKeyword transaction, + * but optional for the NewBookmark transaction). + * + * To make things simple, a given input property has the same basic meaning and + * valid values across all transactions which accept it in the input object. + * Here is a list of all supported input properties along with their expected + * values: + * - url: a URL object, an nsIURI object, or a href. + * - urls: an array of urls, as above. + * - tag - a string. + * - tags: an array of strings. + * - guid, parentGuid, newParentGuid: a valid Places GUID string. + * - guids: an array of valid Places GUID strings. + * - title: a string + * - index, newIndex: the position of an item in its containing folder, + * starting from 0. + * integer and PlacesUtils.bookmarks.DEFAULT_INDEX + * + * If a required property is missing in the input object (e.g. not specifying + * parentGuid for NewBookmark), or if the value for any of the input properties + * is invalid "on the surface" (e.g. a numeric value for GUID, or a string that + * isn't 12-characters long), the transaction constructor throws right way. + * More complex errors (e.g. passing a non-existent GUID for parentGuid) only + * reveal once the transaction is executed. + * + * Executing Transactions (the |transact| method of transactions) + * -------------------------------------------------------------- + * Once a transaction is created, you must call its |transact| method for it to + * be executed and take effect. |transact| is an asynchronous method that takes + * no arguments, and returns a promise that resolves once the transaction is + * executed. Executing one of the transactions for creating items (NewBookmark, + * NewFolder, NewSeparator) resolve to the new item's GUID. + * There's no resolution value for other transactions. + * If a transaction fails to execute, |transact| rejects and the transactions + * history is not affected. + * + * |transact| throws if it's called more than once (successfully or not) on the + * same transaction object. + * + * Batches + * ------- + * Sometimes it is useful to "batch" or "merge" transactions. For example, + * something like "Bookmark All Tabs" may be implemented as one NewFolder + * transaction followed by numerous NewBookmark transactions - all to be undone + * or redone in a single undo or redo command. Use |PlacesTransactions.batch| + * in such cases. It can take either an array of transactions which will be + * executed in the given order and later be treated a a single entry in the + * transactions history, or a generator function that is passed to Task.spawn, + * that is to "contain" the batch: once the generator function is called a batch + * starts, and it lasts until the asynchronous generator iteration is complete + * All transactions executed by |transact| during this time are to be treated as + * a single entry in the transactions history. + * + * In both modes, |PlacesTransactions.batch| returns a promise that is to be + * resolved when the batch ends. In the array-input mode, there's no resolution + * value. In the generator mode, the resolution value is whatever the generator + * function returned (the semantics are the same as in Task.spawn, basically). + * + * The array-input mode of |PlacesTransactions.batch| is useful for implementing + * a batch of mostly-independent transaction (for example, |paste| into a folder + * can be implemented as a batch of multiple NewBookmark transactions). + * The generator mode is useful when the resolution value of executing one + * transaction is the input of one more subsequent transaction. + * + * In the array-input mode, if any transactions fails to execute, the batch + * continues (exceptions are logged). Only transactions that were executed + * successfully are added to the transactions history. + * + * WARNING: "nested" batches are not supported, if you call batch while another + * batch is still running, the new batch is enqueued with all other PTM work + * and thus not run until the running batch ends. The same goes for undo, redo + * and clearTransactionsHistory (note batches cannot be done partially, meaning + * undo and redo calls that during a batch are just enqueued). + * + * ***************************************************************************** + * IT'S PARTICULARLY IMPORTANT NOT TO await ANY PROMISE RETURNED BY ANY OF + * THESE METHODS (undo, redo, clearTransactionsHistory) FROM A BATCH FUNCTION. + * UNTIL WE FIND A WAY TO THROW IN THAT CASE (SEE BUG 1091446) DOING SO WILL + * COMPLETELY BREAK PTM UNTIL SHUTDOWN, NOT ALLOWING THE EXECUTION OF ANY + * TRANSACTION! + * ***************************************************************************** + * + * Serialization + * ------------- + * All |PlacesTransaction| operations are serialized. That is, even though the + * implementation is asynchronous, the order in which PlacesTransactions methods + * is called does guarantee the order in which they are to be invoked. + * + * The only exception to this rule is |transact| calls done during a batch (see + * above). |transact| calls are serialized with each other (and with undo, redo + * and clearTransactionsHistory), but they are, of course, not serialized with + * batches. + * + * The transactions-history structure + * ---------------------------------- + * The transactions-history is a two-dimensional stack of transactions: the + * transactions are ordered in reverse to the order they were committed. + * It's two-dimensional because PTM allows batching transactions together for + * the purpose of undo or redo (see Batches above). + * + * The undoPosition property is set to the index of the top entry. If there is + * no entry at that index, there is nothing to undo. + * Entries prior to undoPosition, if any, are redo entries, the first one being + * the top redo entry. + * + * [ [2nd redo txn, 1st redo txn], <= 2nd redo entry + * [2nd redo txn, 1st redo txn], <= 1st redo entry + * [1st undo txn, 2nd undo txn], <= 1st undo entry + * [1st undo txn, 2nd undo txn] <= 2nd undo entry ] + * undoPostion: 2. + * + * Note that when a new entry is created, all redo entries are removed. + */ + +const TRANSACTIONS_QUEUE_TIMEOUT_MS = 240000; // 4 Mins. + +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +function setTimeout(callback, ms) { + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(callback, ms, timer.TYPE_ONE_SHOT); +} + +class TransactionsHistoryArray extends Array { + constructor() { + super(); + + // The index of the first undo entry (if any) - See the documentation + // at the top of this file. + this._undoPosition = 0; + // Outside of this module, the API of transactions is inaccessible, and so + // are any internal properties. To achieve that, transactions are proxified + // in their constructors. This maps the proxies to their respective raw + // objects. + this.proxifiedToRaw = new WeakMap(); + } + + get undoPosition() { + return this._undoPosition; + } + + // Handy shortcuts + get topUndoEntry() { + return this.undoPosition < this.length ? this[this.undoPosition] : null; + } + get topRedoEntry() { + return this.undoPosition > 0 ? this[this.undoPosition - 1] : null; + } + + /** + * Proxify a transaction object for consumers. + * @param rawTransaction + * the raw transaction object. + * @return the proxified transaction object. + * @see getRawTransaction for retrieving the raw transaction. + */ + proxifyTransaction(rawTransaction) { + let proxy = Object.freeze({ + transact() { + return TransactionsManager.transact(this); + }, + }); + this.proxifiedToRaw.set(proxy, rawTransaction); + return proxy; + } + + /** + * Check if the given object is a the proxy object for some transaction. + * @param aValue + * any JS value. + * @return true if aValue is the proxy object for some transaction, false + * otherwise. + */ + isProxifiedTransactionObject(value) { + return this.proxifiedToRaw.has(value); + } + + /** + * Get the raw transaction for the given proxy. + * @param aProxy + * the proxy object + * @return the transaction proxified by aProxy; |undefined| is returned if + * aProxy is not a proxified transaction. + */ + getRawTransaction(proxy) { + return this.proxifiedToRaw.get(proxy); + } + + /** + * Add a transaction either as a new entry, if forced or if there are no undo + * entries, or to the top undo entry. + * + * @param aProxifiedTransaction + * the proxified transaction object to be added to the transaction + * history. + * @param [optional] aForceNewEntry + * Force a new entry for the transaction. Default: false. + * If false, an entry will we created only if there's no undo entry + * to extend. + */ + add(proxifiedTransaction, forceNewEntry = false) { + if (!this.isProxifiedTransactionObject(proxifiedTransaction)) { + throw new Error("aProxifiedTransaction is not a proxified transaction"); + } + + if (!this.length || forceNewEntry) { + this.clearRedoEntries(); + this.unshift([proxifiedTransaction]); + } else { + this[this.undoPosition].unshift(proxifiedTransaction); + } + } + + /** + * Clear all undo entries. + */ + clearUndoEntries() { + if (this.undoPosition < this.length) { + this.splice(this.undoPosition); + } + } + + /** + * Clear all redo entries. + */ + clearRedoEntries() { + if (this.undoPosition > 0) { + this.splice(0, this.undoPosition); + this._undoPosition = 0; + } + } + + /** + * Clear all entries. + */ + clearAllEntries() { + if (this.length) { + this.splice(0); + this._undoPosition = 0; + } + } +} + +const lazy = {}; + +XPCOMUtils.defineLazyGetter( + lazy, + "TransactionsHistory", + () => new TransactionsHistoryArray() +); + +export var PlacesTransactions = { + /** + * @see Batches in the module documentation. + */ + batch(transactionsToBatch) { + if (Array.isArray(transactionsToBatch)) { + if (!transactionsToBatch.length) { + throw new Error("Must pass a non-empty array"); + } + + if ( + transactionsToBatch.some( + o => !lazy.TransactionsHistory.isProxifiedTransactionObject(o) + ) + ) { + throw new Error("Must pass only transaction entries"); + } + return TransactionsManager.batch(async function () { + for (let txn of transactionsToBatch) { + try { + await txn.transact(); + } catch (ex) { + console.error(ex); + } + } + }); + } + if (typeof transactionsToBatch == "function") { + return TransactionsManager.batch(transactionsToBatch); + } + + throw new Error("Must pass either a function or a transactions array"); + }, + + /** + * Asynchronously undo the transaction immediately after the current undo + * position in the transactions history in the reverse order, if any, and + * adjusts the undo position. + * + * @return {Promises). The promise always resolves. + * @note All undo manager operations are queued. This means that transactions + * history may change by the time your request is fulfilled. + */ + undo() { + return TransactionsManager.undo(); + }, + + /** + * Asynchronously redo the transaction immediately before the current undo + * position in the transactions history, if any, and adjusts the undo + * position. + * + * @return {Promises). The promise always resolves. + * @note All undo manager operations are queued. This means that transactions + * history may change by the time your request is fulfilled. + */ + redo() { + return TransactionsManager.redo(); + }, + + /** + * Asynchronously clear the undo, redo, or all entries from the transactions + * history. + * + * @param [optional] undoEntries + * Whether or not to clear undo entries. Default: true. + * @param [optional] redoEntries + * Whether or not to clear undo entries. Default: true. + * + * @return {Promises). The promise always resolves. + * @throws if both aUndoEntries and aRedoEntries are false. + * @note All undo manager operations are queued. This means that transactions + * history may change by the time your request is fulfilled. + */ + clearTransactionsHistory(undoEntries = true, redoEntries = true) { + return TransactionsManager.clearTransactionsHistory( + undoEntries, + redoEntries + ); + }, + + /** + * The numbers of entries in the transactions history. + */ + get length() { + return lazy.TransactionsHistory.length; + }, + + /** + * Get the transaction history entry at a given index. Each entry consists + * of one or more transaction objects. + * + * @param index + * the index of the entry to retrieve. + * @return an array of transaction objects in their undo order (that is, + * reversely to the order they were executed). + * @throw if aIndex is invalid (< 0 or >= length). + * @note the returned array is a clone of the history entry and is not + * kept in sync with the original entry if it changes. + */ + entry(index) { + if (!Number.isInteger(index) || index < 0 || index >= this.length) { + throw new Error("Invalid index"); + } + + return lazy.TransactionsHistory[index]; + }, + + /** + * The index of the top undo entry in the transactions history. + * If there are no undo entries, it equals to |length|. + * Entries past this point + * Entries at and past this point are redo entries. + */ + get undoPosition() { + return lazy.TransactionsHistory.undoPosition; + }, + + /** + * Shortcut for accessing the top undo entry in the transaction history. + */ + get topUndoEntry() { + return lazy.TransactionsHistory.topUndoEntry; + }, + + /** + * Shortcut for accessing the top redo entry in the transaction history. + */ + get topRedoEntry() { + return lazy.TransactionsHistory.topRedoEntry; + }, +}; + +/** + * Helper for serializing the calls to TransactionsManager methods. It allows + * us to guarantee that the order in which TransactionsManager asynchronous + * methods are called also enforces the order in which they're executed, and + * that they are never executed in parallel. + * + * In other words: Enqueuer.enqueue(aFunc1); Enqueuer.enqueue(aFunc2) is roughly + * the same as Task.spawn(aFunc1).then(Task.spawn(aFunc2)). + */ +function Enqueuer() { + this._promise = Promise.resolve(); +} +Enqueuer.prototype = { + /** + * Spawn a functions once all previous functions enqueued are done running, + * and all promises passed to alsoWaitFor are no longer pending. + * + * @param func + * a function returning a promise. + * @return a promise that resolves once aFunc is done running. The promise + * "mirrors" the promise returned by aFunc. + */ + enqueue(func) { + // If a transaction awaits on a never resolved promise, or is mistakenly + // nested, it could hang the transactions queue forever. Thus we timeout + // the execution after a meaningful amount of time, to ensure in any case + // we'll proceed after a while. + let timeoutPromise = new Promise((resolve, reject) => { + setTimeout( + () => + reject( + new Error( + "PlacesTransaction timeout, most likely caused by unresolved pending work." + ) + ), + TRANSACTIONS_QUEUE_TIMEOUT_MS + ); + }); + let promise = this._promise.then(() => + Promise.race([func(), timeoutPromise]) + ); + + // Propagate exceptions to the caller, but dismiss them internally. + this._promise = promise.catch(console.error); + return promise; + }, + + /** + * Same as above, but for a promise returned by a function that already run. + * This is useful, for example, for serializing transact calls with undo calls, + * even though transact has its own Enqueuer. + * + * @param otherPromise + * any promise. + */ + alsoWaitFor(otherPromise) { + // We don't care if aPromise resolves or rejects, but just that is not + // pending anymore. + // If a transaction awaits on a never resolved promise, or is mistakenly + // nested, it could hang the transactions queue forever. Thus we timeout + // the execution after a meaningful amount of time, to ensure in any case + // we'll proceed after a while. + let timeoutPromise = new Promise((resolve, reject) => { + setTimeout( + () => + reject( + new Error( + "PlacesTransaction timeout, most likely caused by unresolved pending work." + ) + ), + TRANSACTIONS_QUEUE_TIMEOUT_MS + ); + }); + let promise = Promise.race([otherPromise, timeoutPromise]).catch( + console.error + ); + this._promise = Promise.all([this._promise, promise]); + }, + + /** + * The promise for this queue. + */ + get promise() { + return this._promise; + }, +}; + +var TransactionsManager = { + // See the documentation at the top of this file. |transact| calls are not + // serialized with |batch| calls. + _mainEnqueuer: new Enqueuer(), + _transactEnqueuer: new Enqueuer(), + + // Is a batch in progress? set when we enter a batch function and unset when + // it's execution is done. + _batching: false, + + // If a batch started, this indicates if we've already created an entry in the + // transactions history for the batch (i.e. if at least one transaction was + // executed successfully). + _createdBatchEntry: false, + + // Transactions object should never be recycled (that is, |execute| should + // only be called once (or not at all) after they're constructed. + // This keeps track of all transactions which were executed. + _executedTransactions: new WeakSet(), + + transact(txnProxy) { + let rawTxn = lazy.TransactionsHistory.getRawTransaction(txnProxy); + if (!rawTxn) { + throw new Error("|transact| was called with an unexpected object"); + } + + if (this._executedTransactions.has(rawTxn)) { + throw new Error("Transactions objects may not be recycled."); + } + + // Add it in advance so one doesn't accidentally do + // sameTxn.transact(); sameTxn.transact(); + this._executedTransactions.add(rawTxn); + + let promise = this._transactEnqueuer.enqueue(async () => { + // Don't try to catch exceptions. If execute fails, we better not add the + // transaction to the undo stack. + let retval = await rawTxn.execute(); + + let forceNewEntry = !this._batching || !this._createdBatchEntry; + lazy.TransactionsHistory.add(txnProxy, forceNewEntry); + if (this._batching) { + this._createdBatchEntry = true; + } + + this._updateCommandsOnActiveWindow(); + return retval; + }); + this._mainEnqueuer.alsoWaitFor(promise); + return promise; + }, + + batch(task) { + return this._mainEnqueuer.enqueue(async () => { + this._batching = true; + this._createdBatchEntry = false; + let rv; + try { + rv = await task(); + } finally { + // We must enqueue clearing batching mode to ensure that any existing + // transactions have completed before we clear the batching mode. + this._mainEnqueuer.enqueue(() => { + this._batching = false; + this._createdBatchEntry = false; + }); + } + return rv; + }); + }, + + /** + * Undo the top undo entry, if any, and update the undo position accordingly. + */ + undo() { + let promise = this._mainEnqueuer.enqueue(async () => { + let entry = lazy.TransactionsHistory.topUndoEntry; + if (!entry) { + return; + } + + for (let txnProxy of entry) { + try { + await lazy.TransactionsHistory.getRawTransaction(txnProxy).undo(); + } catch (ex) { + // If one transaction is broken, it's not safe to work with any other + // undo entry. Report the error and clear the undo history. + console.error(ex, "Can't undo a transaction, clearing undo entries."); + lazy.TransactionsHistory.clearUndoEntries(); + return; + } + } + lazy.TransactionsHistory._undoPosition++; + this._updateCommandsOnActiveWindow(); + }); + this._transactEnqueuer.alsoWaitFor(promise); + return promise; + }, + + /** + * Redo the top redo entry, if any, and update the undo position accordingly. + */ + redo() { + let promise = this._mainEnqueuer.enqueue(async () => { + let entry = lazy.TransactionsHistory.topRedoEntry; + if (!entry) { + return; + } + + for (let i = entry.length - 1; i >= 0; i--) { + let transaction = lazy.TransactionsHistory.getRawTransaction(entry[i]); + try { + if (transaction.redo) { + await transaction.redo(); + } else { + await transaction.execute(); + } + } catch (ex) { + // If one transaction is broken, it's not safe to work with any other + // redo entry. Report the error and clear the undo history. + console.error(ex, "Can't redo a transaction, clearing redo entries."); + lazy.TransactionsHistory.clearRedoEntries(); + return; + } + } + lazy.TransactionsHistory._undoPosition--; + this._updateCommandsOnActiveWindow(); + }); + + this._transactEnqueuer.alsoWaitFor(promise); + return promise; + }, + + clearTransactionsHistory(undoEntries, redoEntries) { + let promise = this._mainEnqueuer.enqueue(function () { + if (undoEntries && redoEntries) { + lazy.TransactionsHistory.clearAllEntries(); + } else if (undoEntries) { + lazy.TransactionsHistory.clearUndoEntries(); + } else if (redoEntries) { + lazy.TransactionsHistory.clearRedoEntries(); + } else { + throw new Error("either aUndoEntries or aRedoEntries should be true"); + } + }); + + this._transactEnqueuer.alsoWaitFor(promise); + return promise; + }, + + // Updates commands in the undo group of the active window commands. + // Inactive windows commands will be updated on focus. + _updateCommandsOnActiveWindow() { + // Updating "undo" will cause a group update including "redo". + try { + let win = Services.focus.activeWindow; + if (win) { + win.updateCommands("undo"); + } + } catch (ex) { + console.error(ex, "Couldn't update undo commands."); + } + }, +}; + +/** + * Internal helper for defining the standard transactions and their input. + * It takes the required and optional properties, and generates the public + * constructor (which takes the input in the form of a plain object) which, + * when called, creates the argument-less "public" |execute| method by binding + * the input properties to the function arguments (required properties first, + * then the optional properties). + * + * If this seems confusing, look at the consumers. + * + * This magic serves two purposes: + * (1) It completely hides the transactions' internals from the module + * consumers. + * (2) It keeps each transaction implementation to what is about, bypassing + * all this bureaucracy while still validating input appropriately. + */ +function DefineTransaction(requiredProps = [], optionalProps = []) { + for (let prop of [...requiredProps, ...optionalProps]) { + if (!DefineTransaction.inputProps.has(prop)) { + throw new Error("Property '" + prop + "' is not defined"); + } + } + + let ctor = function (input) { + // We want to support both syntaxes: + // let t = new PlacesTransactions.NewBookmark(), + // let t = PlacesTransactions.NewBookmark() + if (this == PlacesTransactions) { + return new ctor(input); + } + + if (requiredProps.length || optionalProps.length) { + // Bind the input properties to the arguments of execute. + input = DefineTransaction.verifyInput( + input, + requiredProps, + optionalProps + ); + this.execute = this.execute.bind(this, input); + } + return lazy.TransactionsHistory.proxifyTransaction(this); + }; + return ctor; +} + +function simpleValidateFunc(checkFn) { + return v => { + if (!checkFn(v)) { + throw new Error("Invalid value"); + } + return v; + }; +} + +DefineTransaction.strValidate = simpleValidateFunc(v => typeof v == "string"); +DefineTransaction.strOrNullValidate = simpleValidateFunc( + v => typeof v == "string" || v === null +); +DefineTransaction.indexValidate = simpleValidateFunc( + v => Number.isInteger(v) && v >= PlacesUtils.bookmarks.DEFAULT_INDEX +); +DefineTransaction.guidValidate = simpleValidateFunc(v => + /^[a-zA-Z0-9\-_]{12}$/.test(v) +); + +function isPrimitive(v) { + return v === null || (typeof v != "object" && typeof v != "function"); +} + +function checkProperty(obj, prop, required, checkFn) { + if (prop in obj) { + return checkFn(obj[prop]); + } + + return !required; +} + +DefineTransaction.childObjectValidate = function (obj) { + if ( + obj && + checkProperty(obj, "title", false, v => typeof v == "string") && + !("type" in obj && obj.type != PlacesUtils.bookmarks.TYPE_BOOKMARK) + ) { + obj.url = DefineTransaction.urlValidate(obj.url); + let validKeys = ["title", "url"]; + if (Object.keys(obj).every(k => validKeys.includes(k))) { + return obj; + } + } + throw new Error("Invalid child object"); +}; + +DefineTransaction.urlValidate = function (url) { + if (url instanceof Ci.nsIURI) { + return URL.fromURI(url); + } + return new URL(url); +}; + +DefineTransaction.inputProps = new Map(); +DefineTransaction.defineInputProps = function ( + names, + validateFn, + defaultValue +) { + for (let name of names) { + this.inputProps.set(name, { + validateValue(value) { + if (value === undefined) { + return defaultValue; + } + try { + return validateFn(value); + } catch (ex) { + throw new Error(`Invalid value for input property ${name}: ${ex}`); + } + }, + + validateInput(input, required) { + if (required && !(name in input)) { + throw new Error(`Required input property is missing: ${name}`); + } + return this.validateValue(input[name]); + }, + + isArrayProperty: false, + }); + } +}; + +DefineTransaction.defineArrayInputProp = function (name, basePropertyName) { + let baseProp = this.inputProps.get(basePropertyName); + if (!baseProp) { + throw new Error(`Unknown input property: ${basePropertyName}`); + } + + this.inputProps.set(name, { + validateValue(aValue) { + if (aValue == undefined) { + return []; + } + + if (!Array.isArray(aValue)) { + throw new Error(`${name} input property value must be an array`); + } + + // We must create a new array in the local scope to avoid a memory leak due + // to the array global object. We can't use Cu.cloneInto as that doesn't + // handle the URIs. Slice & map also aren't good enough, so we start off + // with a clean array and insert what we need into it. + let newArray = []; + for (let item of aValue) { + newArray.push(baseProp.validateValue(item)); + } + return newArray; + }, + + // We allow setting either the array property itself (e.g. urls), or a + // single element of it (url, in that example), that is then transformed + // into a single-element array. + validateInput(input, required) { + if (name in input) { + // It's not allowed to set both though. + if (basePropertyName in input) { + throw new Error(`It is not allowed to set both ${name} and + ${basePropertyName} as input properties`); + } + let array = this.validateValue(input[name]); + if (required && !array.length) { + throw new Error(`Empty array passed for required input property: + ${name}`); + } + return array; + } + // If the property is required and it's not set as is, check if the base + // property is set. + if (required && !(basePropertyName in input)) { + throw new Error(`Required input property is missing: ${name}`); + } + + if (basePropertyName in input) { + return [baseProp.validateValue(input[basePropertyName])]; + } + + return []; + }, + + isArrayProperty: true, + }); +}; + +DefineTransaction.validatePropertyValue = function (prop, input, required) { + return this.inputProps.get(prop).validateInput(input, required); +}; + +DefineTransaction.getInputObjectForSingleValue = function ( + input, + requiredProps, + optionalProps +) { + // The following input forms may be deduced from a single value: + // * a single required property with or without optional properties (the given + // value is set to the required property). + // * a single optional property with no required properties. + if ( + requiredProps.length > 1 || + (!requiredProps.length && optionalProps.length > 1) + ) { + throw new Error("Transaction input isn't an object"); + } + + let propName = + requiredProps.length == 1 ? requiredProps[0] : optionalProps[0]; + let propValue = + this.inputProps.get(propName).isArrayProperty && !Array.isArray(input) + ? [input] + : input; + return { [propName]: propValue }; +}; + +DefineTransaction.verifyInput = function ( + input, + requiredProps = [], + optionalProps = [] +) { + if (!requiredProps.length && !optionalProps.length) { + return {}; + } + + // If there's just a single required/optional property, we allow passing it + // as is, so, for example, one could do PlacesTransactions.Remove(myGuid) + // rather than PlacesTransactions.Remove({ guid: myGuid}). + // This shortcut isn't supported for "complex" properties, like objects (note + // there is no use case for this at the moment anyway). + let isSinglePropertyInput = + isPrimitive(input) || + Array.isArray(input) || + input instanceof Ci.nsISupports; + if (isSinglePropertyInput) { + input = this.getInputObjectForSingleValue( + input, + requiredProps, + optionalProps + ); + } + + let fixedInput = {}; + for (let prop of requiredProps) { + fixedInput[prop] = this.validatePropertyValue(prop, input, true); + } + for (let prop of optionalProps) { + fixedInput[prop] = this.validatePropertyValue(prop, input, false); + } + + return fixedInput; +}; + +// Update the documentation at the top of this module if you add or +// remove properties. +DefineTransaction.defineInputProps( + ["url"], + DefineTransaction.urlValidate, + null +); +DefineTransaction.defineInputProps( + ["guid", "parentGuid", "newParentGuid"], + DefineTransaction.guidValidate +); +DefineTransaction.defineInputProps( + ["title", "postData"], + DefineTransaction.strOrNullValidate, + null +); +DefineTransaction.defineInputProps( + ["keyword", "oldKeyword", "oldTag", "tag"], + DefineTransaction.strValidate, + "" +); +DefineTransaction.defineInputProps( + ["index", "newIndex"], + DefineTransaction.indexValidate, + PlacesUtils.bookmarks.DEFAULT_INDEX +); +DefineTransaction.defineInputProps( + ["child"], + DefineTransaction.childObjectValidate +); +DefineTransaction.defineArrayInputProp("guids", "guid"); +DefineTransaction.defineArrayInputProp("urls", "url"); +DefineTransaction.defineArrayInputProp("tags", "tag"); +DefineTransaction.defineArrayInputProp("children", "child"); + +/** + * Creates items (all types) from a bookmarks tree representation, as defined + * in PlacesUtils.promiseBookmarksTree. + * + * @param tree + * the bookmarks tree object. You may pass either a bookmarks tree + * returned by promiseBookmarksTree, or a manually defined one. + * @param [optional] restoring (default: false) + * Whether or not the items are restored. Only in restore mode, are + * the guid, dateAdded and lastModified properties honored. + * @note the id, root and charset properties of items in aBookmarksTree are + * always ignored. The index property is ignored for all items but the + * root one. + * @return {Promise} + * @resolves to the guid of the new item. + */ +// TODO: Replace most of this with insertTree. +function createItemsFromBookmarksTree(tree, restoring = false) { + async function createItem( + item, + parentGuid, + index = PlacesUtils.bookmarks.DEFAULT_INDEX + ) { + let guid; + let info = { parentGuid, index }; + if (restoring) { + info.guid = item.guid; + info.dateAdded = PlacesUtils.toDate(item.dateAdded); + info.lastModified = PlacesUtils.toDate(item.lastModified); + } + let shouldResetLastModified = false; + switch (item.type) { + case PlacesUtils.TYPE_X_MOZ_PLACE: { + info.url = item.uri; + if (typeof item.title == "string") { + info.title = item.title; + } + + guid = (await PlacesUtils.bookmarks.insert(info)).guid; + + if ("keyword" in item) { + let { uri: url, keyword, postData } = item; + await PlacesUtils.keywords.insert({ url, keyword, postData }); + } + if ("tags" in item) { + PlacesUtils.tagging.tagURI( + Services.io.newURI(item.uri), + item.tags.split(",") + ); + } + break; + } + case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: { + info.type = PlacesUtils.bookmarks.TYPE_FOLDER; + if (typeof item.title == "string") { + info.title = item.title; + } + guid = (await PlacesUtils.bookmarks.insert(info)).guid; + if ("children" in item) { + for (let child of item.children) { + await createItem(child, guid); + } + } + if (restoring) { + shouldResetLastModified = true; + } + break; + } + case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: { + info.type = PlacesUtils.bookmarks.TYPE_SEPARATOR; + guid = (await PlacesUtils.bookmarks.insert(info)).guid; + break; + } + } + + if (shouldResetLastModified) { + let lastModified = PlacesUtils.toDate(item.lastModified); + await PlacesUtils.bookmarks.update({ guid, lastModified }); + } + + return guid; + } + return createItem(tree, tree.parentGuid, tree.index); +} + +/** *************************************************************************** + * The Standard Places Transactions. + * + * See the documentation at the top of this file. The valid values for input + * are also documented there. + *****************************************************************************/ + +var PT = PlacesTransactions; + +/** + * Transaction for creating a bookmark. + * + * Required Input Properties: url, parentGuid. + * Optional Input Properties: index, title, keyword, tags. + * + * When this transaction is executed, it's resolved to the new bookmark's GUID. + */ +PT.NewBookmark = DefineTransaction( + ["parentGuid", "url"], + ["index", "title", "tags"] +); +PT.NewBookmark.prototype = Object.seal({ + async execute({ parentGuid, url, index, title, tags }) { + let info = { parentGuid, index, url, title }; + // Filter tags to exclude already existing ones. + if (tags.length) { + let currentTags = PlacesUtils.tagging.getTagsForURI(url.URI); + tags = tags.filter(t => !currentTags.includes(t)); + } + + async function createItem() { + info = await PlacesUtils.bookmarks.insert(info); + if (tags.length) { + PlacesUtils.tagging.tagURI(url.URI, tags); + } + } + + await createItem(); + + this.undo = async function () { + // Pick up the removed info so we have the accurate last-modified value. + await PlacesUtils.bookmarks.remove(info); + if (tags.length) { + PlacesUtils.tagging.untagURI(url.URI, tags); + } + }; + this.redo = async function () { + await createItem(); + }; + return info.guid; + }, +}); + +/** + * Transaction for creating a folder. + * + * Required Input Properties: title, parentGuid. + * Optional Input Properties: index, children + * + * When this transaction is executed, it's resolved to the new folder's GUID. + */ +PT.NewFolder = DefineTransaction( + ["parentGuid", "title"], + ["index", "children"] +); +PT.NewFolder.prototype = Object.seal({ + async execute({ parentGuid, title, index, children }) { + let folderGuid; + let info = { + children: [ + { + // Ensure to specify a guid to be restored on redo. + guid: PlacesUtils.history.makeGuid(), + title, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }, + ], + // insertTree uses guid as the parent for where it is being inserted + // into. + guid: parentGuid, + }; + + if (children && children.length) { + // Ensure to specify a guid for each child to be restored on redo. + info.children[0].children = children.map(c => { + c.guid = PlacesUtils.history.makeGuid(); + return c; + }); + } + + async function createItem() { + // Note, insertTree returns an array, rather than the folder/child structure. + // For simplicity, we only get the new folder id here. This means that + // an undo then redo won't retain exactly the same information for all + // the child bookmarks, but we believe that isn't important at the moment. + let bmInfo = await PlacesUtils.bookmarks.insertTree(info); + // insertTree returns an array, but we only need to deal with the folder guid. + folderGuid = bmInfo[0].guid; + + // Bug 1388097: insertTree doesn't handle inserting at a specific index for the folder, + // therefore we update the bookmark manually afterwards. + if (index != PlacesUtils.bookmarks.DEFAULT_INDEX) { + bmInfo[0].index = index; + bmInfo = await PlacesUtils.bookmarks.update(bmInfo[0]); + } + } + await createItem(); + + this.undo = async function () { + await PlacesUtils.bookmarks.remove(folderGuid); + }; + this.redo = async function () { + await createItem(); + }; + return folderGuid; + }, +}); + +/** + * Transaction for creating a separator. + * + * Required Input Properties: parentGuid. + * Optional Input Properties: index. + * + * When this transaction is executed, it's resolved to the new separator's + * GUID. + */ +PT.NewSeparator = DefineTransaction(["parentGuid"], ["index"]); +PT.NewSeparator.prototype = Object.seal({ + async execute(info) { + info.type = PlacesUtils.bookmarks.TYPE_SEPARATOR; + info = await PlacesUtils.bookmarks.insert(info); + this.undo = PlacesUtils.bookmarks.remove.bind(PlacesUtils.bookmarks, info); + this.redo = PlacesUtils.bookmarks.insert.bind(PlacesUtils.bookmarks, info); + return info.guid; + }, +}); + +/** + * Transaction for moving an item. + * + * Required Input Properties: guid, newParentGuid. + * Optional Input Properties newIndex. + */ +PT.Move = DefineTransaction(["guids", "newParentGuid"], ["newIndex"]); +PT.Move.prototype = Object.seal({ + async execute({ guids, newParentGuid, newIndex }) { + let originalInfos = []; + let index = newIndex; + + for (let guid of guids) { + // We need to save the original data for undo. + let originalInfo = await PlacesUtils.bookmarks.fetch(guid); + if (!originalInfo) { + throw new Error("Cannot move a non-existent item"); + } + + originalInfos.push(originalInfo); + } + + await PlacesUtils.bookmarks.moveToFolder(guids, newParentGuid, index); + + this.undo = async function () { + // Undo has the potential for moving multiple bookmarks to multiple different + // folders and positions, which is very complicated to manage. Therefore we do + // individual moves one at a time and hopefully everything is put back approximately + // where it should be. + for (let info of originalInfos) { + await PlacesUtils.bookmarks.update(info); + } + }; + this.redo = PlacesUtils.bookmarks.moveToFolder.bind( + PlacesUtils.bookmarks, + guids, + newParentGuid, + index + ); + return guids; + }, +}); + +/** + * Transaction for setting the title for an item. + * + * Required Input Properties: guid, title. + */ +PT.EditTitle = DefineTransaction(["guid", "title"]); +PT.EditTitle.prototype = Object.seal({ + async execute({ guid, title }) { + let originalInfo = await PlacesUtils.bookmarks.fetch(guid); + if (!originalInfo) { + throw new Error("cannot update a non-existent item"); + } + + let updateInfo = { guid, title }; + updateInfo = await PlacesUtils.bookmarks.update(updateInfo); + + this.undo = PlacesUtils.bookmarks.update.bind( + PlacesUtils.bookmarks, + originalInfo + ); + this.redo = PlacesUtils.bookmarks.update.bind( + PlacesUtils.bookmarks, + updateInfo + ); + }, +}); + +/** + * Transaction for setting the URI for an item. + * + * Required Input Properties: guid, url. + */ +PT.EditUrl = DefineTransaction(["guid", "url"]); +PT.EditUrl.prototype = Object.seal({ + async execute({ guid, url }) { + let originalInfo = await PlacesUtils.bookmarks.fetch(guid); + if (!originalInfo) { + throw new Error("cannot update a non-existent item"); + } + if (originalInfo.type != PlacesUtils.bookmarks.TYPE_BOOKMARK) { + throw new Error("Cannot edit url for non-bookmark items"); + } + + let uri = url.URI; + let originalURI = originalInfo.url.URI; + let originalTags = PlacesUtils.tagging.getTagsForURI(originalURI); + let updatedInfo = { guid, url }; + let newURIAdditionalTags = null; + + async function updateItem() { + updatedInfo = await PlacesUtils.bookmarks.update(updatedInfo); + // Move tags from the original URI to the new URI. + if (originalTags.length) { + // Untag the original URI only if this was the only bookmark. + if (!(await PlacesUtils.bookmarks.fetch({ url: originalInfo.url }))) { + PlacesUtils.tagging.untagURI(originalURI, originalTags); + } + let currentNewURITags = PlacesUtils.tagging.getTagsForURI(uri); + newURIAdditionalTags = originalTags.filter( + t => !currentNewURITags.includes(t) + ); + if (newURIAdditionalTags && newURIAdditionalTags.length) { + PlacesUtils.tagging.tagURI(uri, newURIAdditionalTags); + } + } + } + await updateItem(); + + this.undo = async function () { + await PlacesUtils.bookmarks.update(originalInfo); + // Move tags from new URI to original URI. + if (originalTags.length) { + // Only untag the new URI if this is the only bookmark. + if ( + newURIAdditionalTags && + !!newURIAdditionalTags.length && + !(await PlacesUtils.bookmarks.fetch({ url })) + ) { + PlacesUtils.tagging.untagURI(uri, newURIAdditionalTags); + } + PlacesUtils.tagging.tagURI(originalURI, originalTags); + } + }; + + this.redo = async function () { + updatedInfo = await updateItem(); + }; + }, +}); + +/** + * Transaction for setting the keyword for a bookmark. + * + * Required Input Properties: guid, keyword. + * Optional Input Properties: postData, oldKeyword. + */ +PT.EditKeyword = DefineTransaction( + ["guid", "keyword"], + ["postData", "oldKeyword"] +); +PT.EditKeyword.prototype = Object.seal({ + async execute({ guid, keyword, postData, oldKeyword }) { + let url; + let oldKeywordEntry; + if (oldKeyword) { + oldKeywordEntry = await PlacesUtils.keywords.fetch(oldKeyword); + url = oldKeywordEntry.url; + await PlacesUtils.keywords.remove(oldKeyword); + } + + if (keyword) { + if (!url) { + url = (await PlacesUtils.bookmarks.fetch(guid)).url; + } + await PlacesUtils.keywords.insert({ + url, + keyword, + postData: postData || (oldKeywordEntry ? oldKeywordEntry.postData : ""), + }); + } + + this.undo = async function () { + if (keyword) { + await PlacesUtils.keywords.remove(keyword); + } + if (oldKeywordEntry) { + await PlacesUtils.keywords.insert(oldKeywordEntry); + } + }; + }, +}); + +/** + * Transaction for sorting a folder by name. + * + * Required Input Properties: guid. + */ +PT.SortByName = DefineTransaction(["guid"]); +PT.SortByName.prototype = { + async execute({ guid }) { + let sortingMethod = (node_a, node_b) => { + if ( + PlacesUtils.nodeIsContainer(node_a) && + !PlacesUtils.nodeIsContainer(node_b) + ) { + return -1; + } + if ( + !PlacesUtils.nodeIsContainer(node_a) && + PlacesUtils.nodeIsContainer(node_b) + ) { + return 1; + } + return node_a.title.localeCompare(node_b.title); + }; + let oldOrderGuids = []; + let newOrderGuids = []; + let preSepNodes = []; + + // This is not great, since it does main-thread IO. + // PromiseBookmarksTree can't be used, since it' won't stop at the first level'. + let root = PlacesUtils.getFolderContents(guid, false, false).root; + for (let i = 0; i < root.childCount; ++i) { + let node = root.getChild(i); + oldOrderGuids.push(node.bookmarkGuid); + if (PlacesUtils.nodeIsSeparator(node)) { + if (preSepNodes.length) { + preSepNodes.sort(sortingMethod); + newOrderGuids.push(...preSepNodes.map(n => n.bookmarkGuid)); + preSepNodes = []; + } + newOrderGuids.push(node.bookmarkGuid); + } else { + preSepNodes.push(node); + } + } + root.containerOpen = false; + if (preSepNodes.length) { + preSepNodes.sort(sortingMethod); + newOrderGuids.push(...preSepNodes.map(n => n.bookmarkGuid)); + } + await PlacesUtils.bookmarks.reorder(guid, newOrderGuids); + + this.undo = async function () { + await PlacesUtils.bookmarks.reorder(guid, oldOrderGuids); + }; + this.redo = async function () { + await PlacesUtils.bookmarks.reorder(guid, newOrderGuids); + }; + }, +}; + +/** + * Transaction for removing an item (any type). + * + * Required Input Properties: guids. + */ +PT.Remove = DefineTransaction(["guids"]); +PT.Remove.prototype = { + async execute({ guids }) { + let removedItems = []; + + for (let guid of guids) { + try { + // Although we don't strictly need to get this information for the remove, + // we do need it for the possibility of undo(). + removedItems.push(await PlacesUtils.promiseBookmarksTree(guid)); + } catch (ex) { + if (!ex.becauseInvalidURL) { + throw new Error(`Failed to get info for the guid: ${guid}: ${ex}`); + } + removedItems.push({ guid }); + } + } + + let removeThem = async function () { + if (removedItems.length) { + // We have to pass just the guids as although remove() accepts full + // info items, promiseBookmarksTree returns dateAdded and lastModified + // as PRTime rather than date types. + await PlacesUtils.bookmarks.remove( + removedItems.map(info => ({ guid: info.guid })) + ); + } + }; + await removeThem(); + + this.undo = async function () { + for (let info of removedItems) { + try { + await createItemsFromBookmarksTree(info, true); + } catch (ex) { + console.error(`Unable to undo removal of ${info.guid}`); + } + } + }; + this.redo = removeThem; + }, +}; + +/** + * Transaction for tagging urls. + * + * Required Input Properties: urls, tags. + */ +PT.Tag = DefineTransaction(["urls", "tags"]); +PT.Tag.prototype = { + async execute({ urls, tags }) { + let onUndo = [], + onRedo = []; + for (let url of urls) { + if (!(await PlacesUtils.bookmarks.fetch({ url }))) { + // Tagging is only allowed for bookmarked URIs (but see 424160). + let createTxn = lazy.TransactionsHistory.getRawTransaction( + PT.NewBookmark({ + url, + tags, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }) + ); + await createTxn.execute(); + onUndo.unshift(createTxn.undo.bind(createTxn)); + onRedo.push(createTxn.redo.bind(createTxn)); + } else { + let uri = url.URI; + let currentTags = PlacesUtils.tagging.getTagsForURI(uri); + let newTags = tags.filter(t => !currentTags.includes(t)); + if (newTags.length) { + PlacesUtils.tagging.tagURI(uri, newTags); + onUndo.unshift(() => { + PlacesUtils.tagging.untagURI(uri, newTags); + }); + onRedo.push(() => { + PlacesUtils.tagging.tagURI(uri, newTags); + }); + } + } + } + this.undo = async function () { + for (let f of onUndo) { + await f(); + } + }; + this.redo = async function () { + for (let f of onRedo) { + await f(); + } + }; + }, +}; + +/** + * Transaction for removing tags from a URI. + * + * Required Input Properties: urls. + * Optional Input Properties: tags. + * + * If |tags| is not set, all tags set for |url| are removed. + */ +PT.Untag = DefineTransaction(["urls"], ["tags"]); +PT.Untag.prototype = { + execute({ urls, tags }) { + let onUndo = [], + onRedo = []; + for (let url of urls) { + let uri = url.URI; + let tagsToRemove; + let tagsSet = PlacesUtils.tagging.getTagsForURI(uri); + if (tags.length) { + tagsToRemove = tags.filter(t => tagsSet.includes(t)); + } else { + tagsToRemove = tagsSet; + } + if (tagsToRemove.length) { + PlacesUtils.tagging.untagURI(uri, tagsToRemove); + } + onUndo.unshift(() => { + if (tagsToRemove.length) { + PlacesUtils.tagging.tagURI(uri, tagsToRemove); + } + }); + onRedo.push(() => { + if (tagsToRemove.length) { + PlacesUtils.tagging.untagURI(uri, tagsToRemove); + } + }); + } + this.undo = async function () { + for (let f of onUndo) { + await f(); + } + }; + this.redo = async function () { + for (let f of onRedo) { + await f(); + } + }; + }, +}; + +/** + * Transaction for renaming a tag. + * + * Required Input Properties: oldTag, tag. + */ +PT.RenameTag = DefineTransaction(["oldTag", "tag"]); +PT.RenameTag.prototype = { + async execute({ oldTag, tag }) { + // For now this is implemented by untagging and tagging all the bookmarks. + // We should create a specialized bookmarking API to just rename the tag. + let onUndo = [], + onRedo = []; + let urls = new Set(); + await PlacesUtils.bookmarks.fetch({ tags: [oldTag] }, b => urls.add(b.url)); + if (urls.size > 0) { + urls = Array.from(urls); + let tagTxn = lazy.TransactionsHistory.getRawTransaction( + PT.Tag({ urls, tags: [tag] }) + ); + await tagTxn.execute(); + onUndo.unshift(tagTxn.undo.bind(tagTxn)); + onRedo.push(tagTxn.redo.bind(tagTxn)); + let untagTxn = lazy.TransactionsHistory.getRawTransaction( + PT.Untag({ urls, tags: [oldTag] }) + ); + await untagTxn.execute(); + onUndo.unshift(untagTxn.undo.bind(untagTxn)); + onRedo.push(untagTxn.redo.bind(untagTxn)); + + // Update all the place: queries that refer to this tag. + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + SELECT h.url, b.guid, b.title + FROM moz_places h + JOIN moz_bookmarks b ON b.fk = h.id + WHERE url_hash BETWEEN hash("place", "prefix_lo") + AND hash("place", "prefix_hi") + AND url LIKE :tagQuery + `, + { tagQuery: "%tag=%" } + ); + for (let row of rows) { + let url = row.getResultByName("url"); + try { + url = new URL(url); + let urlParams = new URLSearchParams(url.pathname); + let tags = urlParams.getAll("tag"); + if (!tags.includes(oldTag)) { + continue; + } + if (tags.length > 1) { + // URLSearchParams cannot set more than 1 same-named param. + urlParams.delete("tag"); + urlParams.set("tag", tag); + url = new URL( + url.protocol + + urlParams + + "&tag=" + + tags.filter(t => t != oldTag).join("&tag=") + ); + } else { + urlParams.set("tag", tag); + url = new URL(url.protocol + urlParams); + } + } catch (ex) { + console.error( + "Invalid bookmark url: " + row.getResultByName("url") + ": " + ex + ); + continue; + } + let guid = row.getResultByName("guid"); + let title = row.getResultByName("title"); + + let editUrlTxn = lazy.TransactionsHistory.getRawTransaction( + PT.EditUrl({ guid, url }) + ); + await editUrlTxn.execute(); + onUndo.unshift(editUrlTxn.undo.bind(editUrlTxn)); + onRedo.push(editUrlTxn.redo.bind(editUrlTxn)); + if (title == oldTag) { + let editTitleTxn = lazy.TransactionsHistory.getRawTransaction( + PT.EditTitle({ guid, title: tag }) + ); + await editTitleTxn.execute(); + onUndo.unshift(editTitleTxn.undo.bind(editTitleTxn)); + onRedo.push(editTitleTxn.redo.bind(editTitleTxn)); + } + } + } + this.undo = async function () { + for (let f of onUndo) { + await f(); + } + }; + this.redo = async function () { + for (let f of onRedo) { + await f(); + } + }; + }, +}; + +/** + * Transaction for copying an item. + * + * Required Input Properties: guid, newParentGuid + * Optional Input Properties: newIndex. + */ +PT.Copy = DefineTransaction(["guid", "newParentGuid"], ["newIndex"]); +PT.Copy.prototype = { + async execute({ guid, newParentGuid, newIndex }) { + let creationInfo = null; + try { + creationInfo = await PlacesUtils.promiseBookmarksTree(guid); + } catch (ex) { + throw new Error( + "Failed to get info for the specified item (guid: " + + guid + + "). Ex: " + + ex + ); + } + creationInfo.parentGuid = newParentGuid; + creationInfo.index = newIndex; + + let newItemGuid = await createItemsFromBookmarksTree(creationInfo, false); + let newItemInfo = null; + this.undo = async function () { + if (!newItemInfo) { + newItemInfo = await PlacesUtils.promiseBookmarksTree(newItemGuid); + } + await PlacesUtils.bookmarks.remove(newItemGuid); + }; + this.redo = async function () { + await createItemsFromBookmarksTree(newItemInfo, true); + }; + + return newItemGuid; + }, +}; diff --git a/toolkit/components/places/PlacesUtils.sys.mjs b/toolkit/components/places/PlacesUtils.sys.mjs new file mode 100644 index 0000000000..f840566f1f --- /dev/null +++ b/toolkit/components/places/PlacesUtils.sys.mjs @@ -0,0 +1,3083 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +import { PromiseUtils } from "resource://gre/modules/PromiseUtils.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Bookmarks: "resource://gre/modules/Bookmarks.sys.mjs", + History: "resource://gre/modules/History.sys.mjs", + Log: "resource://gre/modules/Log.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + Sqlite: "resource://gre/modules/Sqlite.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "MOZ_ACTION_REGEX", () => { + return /^moz-action:([^,]+),(.*)$/; +}); + +XPCOMUtils.defineLazyGetter(lazy, "gCryptoHash", () => { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); + +// On Mac OSX, the transferable system converts "\r\n" to "\n\n", where +// we really just want "\n". On other platforms, the transferable system +// converts "\r\n" to "\n". +const NEWLINE = AppConstants.platform == "macosx" ? "\n" : "\r\n"; + +// Timers resolution is not always good, it can have a 16ms precision on Win. +const TIMERS_RESOLUTION_SKEW_MS = 16; + +function QI_node(aNode, aIID) { + try { + return aNode.QueryInterface(aIID); + } catch (ex) {} + return null; +} +function asContainer(aNode) { + return QI_node(aNode, Ci.nsINavHistoryContainerResultNode); +} +function asQuery(aNode) { + return QI_node(aNode, Ci.nsINavHistoryQueryResultNode); +} + +/** + * Sends a keyword change notification. + * + * @param url + * the url to notify about. + * @param keyword + * The keyword to notify, or empty string if a keyword was removed. + */ +async function notifyKeywordChange(url, keyword, source) { + // Notify bookmarks about the removal. + let bookmarks = []; + await PlacesUtils.bookmarks.fetch({ url }, b => bookmarks.push(b)); + for (let bookmark of bookmarks) { + let ids = await PlacesUtils.promiseManyItemIds([ + bookmark.guid, + bookmark.parentGuid, + ]); + bookmark.id = ids.get(bookmark.guid); + bookmark.parentId = ids.get(bookmark.parentGuid); + } + + const notifications = bookmarks.map( + bookmark => + new PlacesBookmarkKeyword({ + id: bookmark.id, + itemType: bookmark.type, + url, + guid: bookmark.guid, + parentGuid: bookmark.parentGuid, + keyword, + lastModified: bookmark.lastModified, + source, + isTagging: false, + }) + ); + if (notifications.length) { + PlacesObservers.notifyListeners(notifications); + } +} + +/** + * Serializes the given node in JSON format. + * + * @param aNode + * An nsINavHistoryResultNode + */ +function serializeNode(aNode) { + let data = {}; + + data.title = aNode.title; + // The id is no longer used for copying within the same instance/session of + // Firefox as of at least 61. However, we keep the id for now to maintain + // backwards compat of drag and drop with older Firefox versions. + data.id = aNode.itemId; + data.itemGuid = aNode.bookmarkGuid; + // Add an instanceId so we can tell which instance of an FF session the data + // is coming from. + data.instanceId = PlacesUtils.instanceId; + + let guid = aNode.bookmarkGuid; + + // Some nodes, e.g. the unfiled/menu/toolbar ones can have a virtual guid, so + // we ignore any that are a folder shortcut. These will be handled below. + if ( + guid && + !PlacesUtils.bookmarks.isVirtualRootItem(guid) && + !PlacesUtils.isVirtualLeftPaneItem(guid) + ) { + if (aNode.parent) { + data.parent = aNode.parent.itemId; + data.parentGuid = aNode.parent.bookmarkGuid; + } + + data.dateAdded = aNode.dateAdded; + data.lastModified = aNode.lastModified; + } + + if (PlacesUtils.nodeIsURI(aNode)) { + // Check for url validity. + new URL(aNode.uri); + data.type = PlacesUtils.TYPE_X_MOZ_PLACE; + data.uri = aNode.uri; + if (aNode.tags) { + data.tags = aNode.tags; + } + } else if (PlacesUtils.nodeIsFolder(aNode)) { + if (aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER_SHORTCUT) { + data.type = PlacesUtils.TYPE_X_MOZ_PLACE; + data.uri = aNode.uri; + data.concreteGuid = PlacesUtils.getConcreteItemGuid(aNode); + } else { + data.type = PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER; + } + } else if (PlacesUtils.nodeIsQuery(aNode)) { + data.type = PlacesUtils.TYPE_X_MOZ_PLACE; + data.uri = aNode.uri; + } else if (PlacesUtils.nodeIsSeparator(aNode)) { + data.type = PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR; + } + + return JSON.stringify(data); +} + +// Imposed to limit database size. +const DB_URL_LENGTH_MAX = 65536; +const DB_TITLE_LENGTH_MAX = 4096; +const DB_DESCRIPTION_LENGTH_MAX = 256; +const DB_SITENAME_LENGTH_MAX = 50; + +/** + * Executes a boolean validate function, throwing if it returns false. + * + * @param boolValidateFn + * A boolean validate function. + * @return the input value. + * @throws if input doesn't pass the validate function. + */ +function simpleValidateFunc(boolValidateFn) { + return (v, input) => { + if (!boolValidateFn(v, input)) { + throw new Error("Invalid value"); + } + return v; + }; +} + +/** + * List of bookmark object validators, one per each known property. + * Validators must throw if the property value is invalid and return a fixed up + * version of the value, if needed. + */ +const BOOKMARK_VALIDATORS = Object.freeze({ + guid: simpleValidateFunc(v => PlacesUtils.isValidGuid(v)), + parentGuid: simpleValidateFunc(v => PlacesUtils.isValidGuid(v)), + guidPrefix: simpleValidateFunc(v => PlacesUtils.isValidGuidPrefix(v)), + index: simpleValidateFunc( + v => Number.isInteger(v) && v >= PlacesUtils.bookmarks.DEFAULT_INDEX + ), + dateAdded: simpleValidateFunc(v => v.constructor.name == "Date" && !isNaN(v)), + lastModified: simpleValidateFunc( + v => v.constructor.name == "Date" && !isNaN(v) + ), + type: simpleValidateFunc( + v => + Number.isInteger(v) && + [ + PlacesUtils.bookmarks.TYPE_BOOKMARK, + PlacesUtils.bookmarks.TYPE_FOLDER, + PlacesUtils.bookmarks.TYPE_SEPARATOR, + ].includes(v) + ), + title: v => { + if (v === null) { + return ""; + } + if (typeof v == "string") { + return v.slice(0, DB_TITLE_LENGTH_MAX); + } + throw new Error("Invalid title"); + }, + url: v => { + simpleValidateFunc( + val => + (typeof val == "string" && val.length <= DB_URL_LENGTH_MAX) || + (val instanceof Ci.nsIURI && val.spec.length <= DB_URL_LENGTH_MAX) || + (URL.isInstance(val) && val.href.length <= DB_URL_LENGTH_MAX) + )(v); + if (typeof v === "string") { + return new URL(v); + } + if (v instanceof Ci.nsIURI) { + return URL.fromURI(v); + } + return v; + }, + source: simpleValidateFunc( + v => + Number.isInteger(v) && + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v) + ), + keyword: simpleValidateFunc(v => typeof v == "string" && v.length), + charset: simpleValidateFunc(v => typeof v == "string" && v.length), + postData: simpleValidateFunc(v => typeof v == "string" && v.length), + tags: simpleValidateFunc( + v => + Array.isArray(v) && + v.length && + v.every(item => item && typeof item == "string") + ), +}); + +// Sync bookmark records can contain additional properties. +const SYNC_BOOKMARK_VALIDATORS = Object.freeze({ + // Sync uses Places GUIDs for all records except roots. + recordId: simpleValidateFunc( + v => + typeof v == "string" && + (lazy.PlacesSyncUtils.bookmarks.ROOTS.includes(v) || + PlacesUtils.isValidGuid(v)) + ), + parentRecordId: v => SYNC_BOOKMARK_VALIDATORS.recordId(v), + // Sync uses kinds instead of types. + kind: simpleValidateFunc( + v => + typeof v == "string" && + Object.values(lazy.PlacesSyncUtils.bookmarks.KINDS).includes(v) + ), + query: simpleValidateFunc(v => v === null || (typeof v == "string" && v)), + folder: simpleValidateFunc( + v => + typeof v == "string" && + v && + v.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH + ), + tags: v => { + if (v === null) { + return []; + } + if (!Array.isArray(v)) { + throw new Error("Invalid tag array"); + } + for (let tag of v) { + if ( + typeof tag != "string" || + !tag || + tag.length > PlacesUtils.bookmarks.MAX_TAG_LENGTH + ) { + throw new Error(`Invalid tag: ${tag}`); + } + } + return v; + }, + keyword: simpleValidateFunc(v => v === null || typeof v == "string"), + dateAdded: simpleValidateFunc( + v => + typeof v === "number" && + v > lazy.PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP + ), + feed: v => (v === null ? v : BOOKMARK_VALIDATORS.url(v)), + site: v => (v === null ? v : BOOKMARK_VALIDATORS.url(v)), + title: BOOKMARK_VALIDATORS.title, + url: BOOKMARK_VALIDATORS.url, +}); + +// Sync change records are passed between `PlacesSyncUtils` and the Sync +// bookmarks engine, and are used to update an item's sync status and change +// counter at the end of a sync. +const SYNC_CHANGE_RECORD_VALIDATORS = Object.freeze({ + modified: simpleValidateFunc(v => typeof v == "number" && v >= 0), + counter: simpleValidateFunc(v => typeof v == "number" && v >= 0), + status: simpleValidateFunc( + v => + typeof v == "number" && + Object.values(PlacesUtils.bookmarks.SYNC_STATUS).includes(v) + ), + tombstone: simpleValidateFunc(v => v === true || v === false), + synced: simpleValidateFunc(v => v === true || v === false), +}); +/** + * List PageInfo bookmark object validators. + */ +const PAGEINFO_VALIDATORS = Object.freeze({ + guid: BOOKMARK_VALIDATORS.guid, + url: BOOKMARK_VALIDATORS.url, + title: v => { + if (v == null || v == undefined) { + return undefined; + } else if (typeof v === "string") { + return v; + } + throw new TypeError( + `title property of PageInfo object: ${v} must be a string if provided` + ); + }, + previewImageURL: v => { + if (!v) { + return null; + } + return BOOKMARK_VALIDATORS.url(v); + }, + description: v => { + if (typeof v === "string" || v === null) { + return v ? v.slice(0, DB_DESCRIPTION_LENGTH_MAX) : null; + } + throw new TypeError( + `description property of pageInfo object: ${v} must be either a string or null if provided` + ); + }, + siteName: v => { + if (typeof v === "string" || v === null) { + return v ? v.slice(0, DB_SITENAME_LENGTH_MAX) : null; + } + throw new TypeError( + `siteName property of pageInfo object: ${v} must be either a string or null if provided` + ); + }, + annotations: v => { + if (typeof v != "object" || v.constructor.name != "Map") { + throw new TypeError("annotations must be a Map"); + } + + if (v.size == 0) { + throw new TypeError("there must be at least one annotation"); + } + + for (let [key, value] of v.entries()) { + if (typeof key != "string") { + throw new TypeError("all annotation keys must be strings"); + } + if ( + typeof value != "string" && + typeof value != "number" && + typeof value != "boolean" && + value !== null && + value !== undefined + ) { + throw new TypeError( + "all annotation values must be Boolean, Numbers or Strings" + ); + } + } + return v; + }, + visits: v => { + if (!Array.isArray(v) || !v.length) { + throw new TypeError("PageInfo object must have an array of visits"); + } + let visits = []; + for (let inVisit of v) { + let visit = { + date: new Date(), + transition: inVisit.transition || lazy.History.TRANSITIONS.LINK, + }; + + if (!PlacesUtils.history.isValidTransition(visit.transition)) { + throw new TypeError( + `transition: ${visit.transition} is not a valid transition type` + ); + } + + if (inVisit.date) { + PlacesUtils.history.ensureDate(inVisit.date); + if (inVisit.date > Date.now() + TIMERS_RESOLUTION_SKEW_MS) { + throw new TypeError(`date: ${inVisit.date} cannot be a future date`); + } + visit.date = inVisit.date; + } + + if (inVisit.referrer) { + visit.referrer = PlacesUtils.normalizeToURLOrGUID(inVisit.referrer); + } + visits.push(visit); + } + return visits; + }, +}); + +export var PlacesUtils = { + // Place entries that are containers, e.g. bookmark folders or queries. + TYPE_X_MOZ_PLACE_CONTAINER: "text/x-moz-place-container", + // Place entries that are bookmark separators. + TYPE_X_MOZ_PLACE_SEPARATOR: "text/x-moz-place-separator", + // Place entries that are not containers or separators + TYPE_X_MOZ_PLACE: "text/x-moz-place", + // Place entries in shortcut url format (url\ntitle) + TYPE_X_MOZ_URL: "text/x-moz-url", + // Place entries formatted as HTML anchors + TYPE_HTML: "text/html", + // Place entries as raw URL text + TYPE_PLAINTEXT: "text/plain", + // Used to track the action that populated the clipboard. + TYPE_X_MOZ_PLACE_ACTION: "text/x-moz-place-action", + + // Deprecated: Remaining only for supporting migration of old livemarks. + LMANNO_FEEDURI: "livemark/feedURI", + LMANNO_SITEURI: "livemark/siteURI", + CHARSET_ANNO: "URIProperties/characterSet", + // Deprecated: This is only used for supporting import from older datasets. + MOBILE_ROOT_ANNO: "mobile/bookmarksRoot", + + TOPIC_SHUTDOWN: "places-shutdown", + TOPIC_INIT_COMPLETE: "places-init-complete", + TOPIC_DATABASE_LOCKED: "places-database-locked", + TOPIC_EXPIRATION_FINISHED: "places-expiration-finished", + TOPIC_FAVICONS_EXPIRED: "places-favicons-expired", + TOPIC_VACUUM_STARTING: "places-vacuum-starting", + TOPIC_BOOKMARKS_RESTORE_BEGIN: "bookmarks-restore-begin", + TOPIC_BOOKMARKS_RESTORE_SUCCESS: "bookmarks-restore-success", + TOPIC_BOOKMARKS_RESTORE_FAILED: "bookmarks-restore-failed", + + observers: PlacesObservers, + + /** + * GUIDs associated with virtual queries that are used for displaying the + * top-level folders in the left pane. + */ + virtualAllBookmarksGuid: "allbms_____v", + virtualHistoryGuid: "history____v", + virtualDownloadsGuid: "downloads__v", + virtualTagsGuid: "tags_______v", + + /** + * Checks if a guid is a virtual left-pane root. + * + * @param {String} guid The guid of the item to look for. + * @returns {Boolean} true if guid is a virtual root, false otherwise. + */ + isVirtualLeftPaneItem(guid) { + return ( + guid == PlacesUtils.virtualAllBookmarksGuid || + guid == PlacesUtils.virtualHistoryGuid || + guid == PlacesUtils.virtualDownloadsGuid || + guid == PlacesUtils.virtualTagsGuid + ); + }, + + asContainer: aNode => asContainer(aNode), + asQuery: aNode => asQuery(aNode), + + endl: NEWLINE, + + /** + * Is a string a valid GUID? + * + * @param guid: (String) + * @return (Boolean) + */ + isValidGuid(guid) { + return typeof guid == "string" && guid && /^[a-zA-Z0-9\-_]{12}$/.test(guid); + }, + + /** + * Is a string a valid GUID prefix? + * + * @param guidPrefix: (String) + * @return (Boolean) + */ + isValidGuidPrefix(guidPrefix) { + return ( + typeof guidPrefix == "string" && + guidPrefix && + /^[a-zA-Z0-9\-_]{1,11}$/.test(guidPrefix) + ); + }, + + /** + * Generates a random GUID and replace its beginning with the given + * prefix. We do this instead of just prepending the prefix to keep + * the correct character length. + * + * @param prefix: (String) + * @return (String) + */ + generateGuidWithPrefix(prefix) { + return prefix + this.history.makeGuid().substring(prefix.length); + }, + + /** + * Converts a string or n URL object to an nsIURI. + * + * @param url (URL) or (String) + * the URL to convert. + * @return nsIURI for the given URL. + */ + toURI(url) { + if (url instanceof Ci.nsIURI) { + return url; + } + if (URL.isInstance(url)) { + return url.URI; + } + return Services.io.newURI(url); + }, + + /** + * Convert a Date object to a PRTime (microseconds). + * + * @param date + * the Date object to convert. + * @return microseconds from the epoch. + */ + toPRTime(date) { + if ( + (typeof date != "number" && date.constructor.name != "Date") || + isNaN(date) + ) { + throw new Error("Invalid value passed to toPRTime"); + } + return date * 1000; + }, + + /** + * Convert a PRTime to a Date object. + * + * @param time + * microseconds from the epoch. + * @return a Date object. + */ + toDate(time) { + if (typeof time != "number" || isNaN(time)) { + throw new Error("Invalid value passed to toDate"); + } + return new Date(parseInt(time / 1000)); + }, + + /** + * Wraps a string in a nsISupportsString wrapper. + * @param aString + * The string to wrap. + * @returns A nsISupportsString object containing a string. + */ + toISupportsString: function PU_toISupportsString(aString) { + let s = Cc["@mozilla.org/supports-string;1"].createInstance( + Ci.nsISupportsString + ); + s.data = aString; + return s; + }, + + getFormattedString: function PU_getFormattedString(key, params) { + return lazy.bundle.formatStringFromName(key, params); + }, + + getString: function PU_getString(key) { + return lazy.bundle.GetStringFromName(key); + }, + + /** + * Parses a moz-action URL and returns its parts. + * + * @param url A moz-action URI. + * @note URL is in the format moz-action:ACTION,JSON_ENCODED_PARAMS + */ + parseActionUrl(url) { + if (url instanceof Ci.nsIURI) { + url = url.spec; + } else if (URL.isInstance(url)) { + url = url.href; + } + // Faster bailout. + if (!url.startsWith("moz-action:")) { + return null; + } + + try { + let [, type, params] = url.match(lazy.MOZ_ACTION_REGEX); + let action = { + type, + params: JSON.parse(params), + }; + for (let key in action.params) { + action.params[key] = decodeURIComponent(action.params[key]); + } + return action; + } catch (ex) { + console.error(`Invalid action url "${url}"`); + return null; + } + }, + + /** + * Determines if a folder is generated from a query. + * @param aNode a result true. + * @returns true if the node is a folder generated from a query. + */ + isQueryGeneratedFolder(node) { + if (!node.parent) { + return false; + } + return this.nodeIsFolder(node) && this.nodeIsQuery(node.parent); + }, + + /** + * Determines whether or not a ResultNode is a Bookmark folder. + * @param aNode + * A result node + * @returns true if the node is a Bookmark folder, false otherwise + */ + nodeIsFolder: function PU_nodeIsFolder(aNode) { + return ( + aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER || + aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER_SHORTCUT + ); + }, + + /** + * Determines whether or not a ResultNode represents a bookmarked URI. + * @param aNode + * A result node + * @returns true if the node represents a bookmarked URI, false otherwise + */ + nodeIsBookmark: function PU_nodeIsBookmark(aNode) { + return ( + aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_URI && + aNode.itemId != -1 + ); + }, + + /** + * Determines whether or not a ResultNode is a Bookmark separator. + * @param aNode + * A result node + * @returns true if the node is a Bookmark separator, false otherwise + */ + nodeIsSeparator: function PU_nodeIsSeparator(aNode) { + return aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_SEPARATOR; + }, + + /** + * Determines whether or not a ResultNode is a URL item. + * @param aNode + * A result node + * @returns true if the node is a URL item, false otherwise + */ + nodeIsURI: function PU_nodeIsURI(aNode) { + return aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_URI; + }, + + /** + * Determines whether or not a ResultNode is a Query item. + * @param aNode + * A result node + * @returns true if the node is a Query item, false otherwise + */ + nodeIsQuery: function PU_nodeIsQuery(aNode) { + return aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_QUERY; + }, + + /** + * Generator for a node's ancestors. + * @param aNode + * A result node + */ + nodeAncestors: function* PU_nodeAncestors(aNode) { + let node = aNode.parent; + while (node) { + yield node; + node = node.parent; + } + }, + + /** + * Checks validity of an object, filling up default values for optional + * properties. + * + * @param {string} name + * The operation name. This is included in the error message if + * validation fails. + * @param validators (object) + * An object containing input validators. Keys should be field names; + * values should be validation functions. + * @param props (object) + * The object to validate. + * @param behavior (object) [optional] + * Object defining special behavior for some of the properties. + * The following behaviors may be optionally set: + * - required: this property is required. + * - replaceWith: this property will be overwritten with the value + * provided + * - requiredIf: if the provided condition is satisfied, then this + * property is required. + * - validIf: if the provided condition is not satisfied, then this + * property is invalid. + * - defaultValue: an undefined property should default to this value. + * - fixup: a function invoked when validation fails, takes the input + * object as argument and must fix the property. + * + * @return a validated and normalized item. + * @throws if the object contains invalid data. + * @note any unknown properties are pass-through. + */ + validateItemProperties(name, validators, props, behavior = {}) { + if (typeof props != "object" || !props) { + throw new Error(`${name}: Input should be a valid object`); + } + // Make a shallow copy of `props` to avoid mutating the original object + // when filling in defaults. + let input = Object.assign({}, props); + let normalizedInput = {}; + let required = new Set(); + for (let prop in behavior) { + if ( + behavior[prop].hasOwnProperty("required") && + behavior[prop].required + ) { + required.add(prop); + } + if ( + behavior[prop].hasOwnProperty("requiredIf") && + behavior[prop].requiredIf(input) + ) { + required.add(prop); + } + if ( + behavior[prop].hasOwnProperty("validIf") && + input[prop] !== undefined && + !behavior[prop].validIf(input) + ) { + if (behavior[prop].hasOwnProperty("fixup")) { + behavior[prop].fixup(input); + } else { + throw new Error( + `${name}: Invalid value for property '${prop}': ${JSON.stringify( + input[prop] + )}` + ); + } + } + if ( + behavior[prop].hasOwnProperty("defaultValue") && + input[prop] === undefined + ) { + input[prop] = behavior[prop].defaultValue; + } + if (behavior[prop].hasOwnProperty("replaceWith")) { + input[prop] = behavior[prop].replaceWith; + } + } + + for (let prop in input) { + if (required.has(prop)) { + required.delete(prop); + } else if (input[prop] === undefined) { + // Skip undefined properties that are not required. + continue; + } + if (validators.hasOwnProperty(prop)) { + try { + normalizedInput[prop] = validators[prop](input[prop], input); + } catch (ex) { + if ( + behavior.hasOwnProperty(prop) && + behavior[prop].hasOwnProperty("fixup") + ) { + behavior[prop].fixup(input); + normalizedInput[prop] = input[prop]; + } else { + throw new Error( + `${name}: Invalid value for property '${prop}': ${JSON.stringify( + input[prop] + )}` + ); + } + } + } + } + if (required.size > 0) { + throw new Error( + `${name}: The following properties were expected: ${[...required].join( + ", " + )}` + ); + } + return normalizedInput; + }, + + BOOKMARK_VALIDATORS, + PAGEINFO_VALIDATORS, + SYNC_BOOKMARK_VALIDATORS, + SYNC_CHANGE_RECORD_VALIDATORS, + + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), + + _shutdownFunctions: [], + registerShutdownFunction: function PU_registerShutdownFunction(aFunc) { + // If this is the first registered function, add the shutdown observer. + if (!this._shutdownFunctions.length) { + Services.obs.addObserver(this, this.TOPIC_SHUTDOWN); + } + this._shutdownFunctions.push(aFunc); + }, + + // nsIObserver + observe: function PU_observe(aSubject, aTopic, aData) { + switch (aTopic) { + case this.TOPIC_SHUTDOWN: + Services.obs.removeObserver(this, this.TOPIC_SHUTDOWN); + while (this._shutdownFunctions.length) { + this._shutdownFunctions.shift().apply(this); + } + break; + } + }, + + /** + * Determines whether or not a ResultNode is a host container. + * @param aNode + * A result node + * @returns true if the node is a host container, false otherwise + */ + nodeIsHost: function PU_nodeIsHost(aNode) { + return ( + aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_QUERY && + aNode.parent && + asQuery(aNode.parent).queryOptions.resultType == + Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY + ); + }, + + /** + * Determines whether or not a ResultNode is a day container. + * @param node + * A NavHistoryResultNode + * @returns true if the node is a day container, false otherwise + */ + nodeIsDay: function PU_nodeIsDay(aNode) { + var resultType; + return ( + aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_QUERY && + aNode.parent && + ((resultType = asQuery(aNode.parent).queryOptions.resultType) == + Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY || + resultType == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY) + ); + }, + + /** + * Determines whether or not a result-node is a tag container. + * @param aNode + * A result-node + * @returns true if the node is a tag container, false otherwise + */ + nodeIsTagQuery: function PU_nodeIsTagQuery(aNode) { + if (aNode.type != Ci.nsINavHistoryResultNode.RESULT_TYPE_QUERY) { + return false; + } + // Direct child of RESULTS_AS_TAGS_ROOT. + let parent = aNode.parent; + if ( + parent && + PlacesUtils.asQuery(parent).queryOptions.resultType == + Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAGS_ROOT + ) { + return true; + } + // We must also support the right pane of the Library, when the tag query + // is the root node. Unfortunately this is also valid for any tag query + // selected in the left pane that is not a direct child of RESULTS_AS_TAGS_ROOT. + if ( + !parent && + aNode == aNode.parentResult.root && + PlacesUtils.asQuery(aNode).query.tags.length == 1 + ) { + return true; + } + return false; + }, + + /** + * Determines whether or not a ResultNode is a container. + * @param aNode + * A result node + * @returns true if the node is a container item, false otherwise + */ + containerTypes: [ + Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER, + Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER_SHORTCUT, + Ci.nsINavHistoryResultNode.RESULT_TYPE_QUERY, + ], + nodeIsContainer: function PU_nodeIsContainer(aNode) { + return this.containerTypes.includes(aNode.type); + }, + + /** + * Determines whether or not a ResultNode is an history related container. + * @param node + * A result node + * @returns true if the node is an history related container, false otherwise + */ + nodeIsHistoryContainer: function PU_nodeIsHistoryContainer(aNode) { + var resultType; + return ( + this.nodeIsQuery(aNode) && + ((resultType = asQuery(aNode).queryOptions.resultType) == + Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY || + resultType == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY || + resultType == Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY || + this.nodeIsDay(aNode) || + this.nodeIsHost(aNode)) + ); + }, + + /** + * Gets the concrete item-guid for the given node. For everything but folder + * shortcuts, this is just node.bookmarkGuid. For folder shortcuts, this is + * node.targetFolderGuid (see nsINavHistoryService.idl for the semantics). + * + * @param aNode + * a result node. + * @return the concrete item-guid for aNode. + */ + getConcreteItemGuid(aNode) { + if (aNode.type == Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER_SHORTCUT) { + return asQuery(aNode).targetFolderGuid; + } + return aNode.bookmarkGuid; + }, + + /** + * Reverse a host based on the moz_places algorithm, that is reverse the host + * string and add a trailing period. For example "google.com" becomes + * "moc.elgoog.". + * + * @param url + * the URL to generate a rev host for. + * @return the reversed host string. + */ + getReversedHost(url) { + return url.host.split("").reverse().join("") + "."; + }, + + /** + * String-wraps a result node according to the rules of the specified + * content type for copy or move operations. + * + * @param aNode + * The Result node to wrap (serialize) + * @param aType + * The content type to serialize as + * @return A string serialization of the node + */ + wrapNode(aNode, aType) { + // when wrapping a node, we want all the items, even if the original + // query options are excluding them. + // This can happen when copying from the left hand pane of the bookmarks + // organizer. + // @return [node, shouldClose] + function gatherDataFromNode(node, gatherDataFunc) { + if ( + PlacesUtils.nodeIsFolder(node) && + node.type != Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER_SHORTCUT && + asQuery(node).queryOptions.excludeItems + ) { + let folderRoot = PlacesUtils.getFolderContents( + node.bookmarkGuid, + false, + true + ).root; + try { + return gatherDataFunc(folderRoot); + } finally { + folderRoot.containerOpen = false; + } + } + // If we didn't create our own query, do not alter the node's state. + return gatherDataFunc(node); + } + + function gatherDataHtml(node) { + let htmlEscape = s => + s + .replace(/&/g, "&") + .replace(/>/g, ">") + .replace(/" + NEWLINE; + let cc = node.childCount; + for (let i = 0; i < cc; ++i) { + childString += + "
      " + + NEWLINE + + gatherDataHtml(node.getChild(i)) + + "
      " + + NEWLINE; + } + node.containerOpen = wasOpen; + return childString + "" + NEWLINE; + } + if (PlacesUtils.nodeIsURI(node)) { + return `${escapedTitle}${NEWLINE}`; + } + if (PlacesUtils.nodeIsSeparator(node)) { + return "
      " + NEWLINE; + } + return ""; + } + + function gatherDataText(node) { + if (PlacesUtils.nodeIsContainer(node)) { + asContainer(node); + let wasOpen = node.containerOpen; + if (!wasOpen) { + node.containerOpen = true; + } + + let childString = node.title + NEWLINE; + let cc = node.childCount; + for (let i = 0; i < cc; ++i) { + let child = node.getChild(i); + let suffix = i < cc - 1 ? NEWLINE : ""; + childString += gatherDataText(child) + suffix; + } + node.containerOpen = wasOpen; + return childString; + } + if (PlacesUtils.nodeIsURI(node)) { + return node.uri; + } + if (PlacesUtils.nodeIsSeparator(node)) { + return "--------------------"; + } + return ""; + } + + switch (aType) { + case this.TYPE_X_MOZ_PLACE: + case this.TYPE_X_MOZ_PLACE_SEPARATOR: + case this.TYPE_X_MOZ_PLACE_CONTAINER: { + // Serialize the node to JSON. + return serializeNode(aNode); + } + case this.TYPE_X_MOZ_URL: { + if (PlacesUtils.nodeIsURI(aNode)) { + return aNode.uri + NEWLINE + aNode.title; + } + if (PlacesUtils.nodeIsContainer(aNode)) { + return PlacesUtils.getURLsForContainerNode(aNode) + .map(item => item.uri + "\n" + item.title) + .join("\n"); + } + return ""; + } + case this.TYPE_HTML: { + return gatherDataFromNode(aNode, gatherDataHtml); + } + } + + // Otherwise, we wrap as TYPE_PLAINTEXT. + return gatherDataFromNode(aNode, gatherDataText); + }, + + /** + * Unwraps data from the Clipboard or the current Drag Session. + * @param blob + * A blob (string) of data, in some format we potentially know how + * to parse. + * @param type + * The content type of the blob. + * @returns An array of objects representing each item contained by the source. + * @throws if the blob contains invalid data. + */ + unwrapNodes: function PU_unwrapNodes(blob, type) { + // We split on "\n" because the transferable system converts "\r\n" to "\n" + var nodes = []; + switch (type) { + case this.TYPE_X_MOZ_PLACE: + case this.TYPE_X_MOZ_PLACE_SEPARATOR: + case this.TYPE_X_MOZ_PLACE_CONTAINER: + nodes = JSON.parse("[" + blob + "]"); + break; + case this.TYPE_X_MOZ_URL: { + let parts = blob.split("\n"); + // data in this type has 2 parts per entry, so if there are fewer + // than 2 parts left, the blob is malformed and we should stop + // but drag and drop of files from the shell has parts.length = 1 + if (parts.length != 1 && parts.length % 2) { + break; + } + for (let i = 0; i < parts.length; i = i + 2) { + let uriString = parts[i]; + let titleString = ""; + if (parts.length > i + 1) { + titleString = parts[i + 1]; + } else { + // for drag and drop of files, try to use the leafName as title + try { + titleString = Services.io + .newURI(uriString) + .QueryInterface(Ci.nsIURL).fileName; + } catch (ex) {} + } + // note: Services.io.newURI() will throw if uriString is not a valid URI + let uri = Services.io.newURI(uriString); + if (Services.io.newURI(uriString) && uri.scheme != "place") { + nodes.push({ + uri: uriString, + title: titleString ? titleString : uriString, + type: this.TYPE_X_MOZ_URL, + }); + } + } + break; + } + case this.TYPE_PLAINTEXT: { + let parts = blob.split("\n"); + for (let i = 0; i < parts.length; i++) { + let uriString = parts[i]; + // text/uri-list is converted to TYPE_PLAINTEXT but it could contain + // comments line prepended by #, we should skip them, as well as + // empty uris. + if (uriString.substr(0, 1) == "\x23" || uriString == "") { + continue; + } + // note: Services.io.newURI) will throw if uriString is not a valid URI + let uri = Services.io.newURI(uriString); + if (uri.scheme != "place") { + nodes.push({ + uri: uriString, + title: uriString, + type: this.TYPE_X_MOZ_URL, + }); + } + } + break; + } + default: + throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); + } + return nodes; + }, + + /** + * Validate an input PageInfo object, returning a valid PageInfo object. + * + * @param pageInfo: (PageInfo) + * @return (PageInfo) + */ + validatePageInfo(pageInfo, validateVisits = true) { + return this.validateItemProperties( + "PageInfo", + PAGEINFO_VALIDATORS, + pageInfo, + { + url: { requiredIf: b => !b.guid }, + guid: { requiredIf: b => !b.url }, + visits: { requiredIf: b => validateVisits }, + } + ); + }, + /** + * Normalize a key to either a string (if it is a valid GUID) or an + * instance of `URL` (if it is a `URL`, `nsIURI`, or a string + * representing a valid url). + * + * @throws (TypeError) + * If the key is neither a valid guid nor a valid url. + */ + normalizeToURLOrGUID(key) { + if (typeof key === "string") { + // A string may be a URL or a guid + if (this.isValidGuid(key)) { + return key; + } + return new URL(key); + } + if (URL.isInstance(key)) { + return key; + } + if (key instanceof Ci.nsIURI) { + return URL.fromURI(key); + } + throw new TypeError("Invalid url or guid: " + key); + }, + + /** + * Generates a nsINavHistoryResult for the contents of a folder. + * @param aFolderGuid + * The folder to open + * @param [optional] excludeItems + * True to hide all items (individual bookmarks). This is used on + * the left places pane so you just get a folder hierarchy. + * @param [optional] expandQueries + * True to make query items expand as new containers. For managing, + * you want this to be false, for menus and such, you want this to + * be true. + * @returns A nsINavHistoryResult containing the contents of the + * folder. The result.root is guaranteed to be open. + */ + getFolderContents(aFolderGuid, aExcludeItems, aExpandQueries) { + if (!this.isValidGuid(aFolderGuid)) { + throw new Error("aFolderGuid should be a valid GUID."); + } + var query = this.history.getNewQuery(); + query.setParents([aFolderGuid]); + var options = this.history.getNewQueryOptions(); + options.excludeItems = aExcludeItems; + options.expandQueries = aExpandQueries; + + var result = this.history.executeQuery(query, options); + result.root.containerOpen = true; + return result; + }, + + // Identifier getters for special folders. + // You should use these everywhere PlacesUtils is available to avoid XPCOM + // traversal just to get roots' ids. + get tagsFolderId() { + delete this.tagsFolderId; + return (this.tagsFolderId = this.bookmarks.tagsFolder); + }, + + /** + * Checks if item is a root. + * + * @param {String} guid The guid of the item to look for. + * @returns {Boolean} true if guid is a root, false otherwise. + */ + isRootItem(guid) { + return ( + guid == PlacesUtils.bookmarks.menuGuid || + guid == PlacesUtils.bookmarks.toolbarGuid || + guid == PlacesUtils.bookmarks.unfiledGuid || + guid == PlacesUtils.bookmarks.tagsGuid || + guid == PlacesUtils.bookmarks.rootGuid || + guid == PlacesUtils.bookmarks.mobileGuid + ); + }, + + /** + * Returns a nsNavHistoryContainerResultNode with forced excludeItems and + * expandQueries. + * @param aNode + * The node to convert + * @param [optional] excludeItems + * True to hide all items (individual bookmarks). This is used on + * the left places pane so you just get a folder hierarchy. + * @param [optional] expandQueries + * True to make query items expand as new containers. For managing, + * you want this to be false, for menus and such, you want this to + * be true. + * @returns A nsINavHistoryContainerResultNode containing the unfiltered + * contents of the container. + * @note The returned container node could be open or closed, we don't + * guarantee its status. + */ + getContainerNodeWithOptions: function PU_getContainerNodeWithOptions( + aNode, + aExcludeItems, + aExpandQueries + ) { + if (!this.nodeIsContainer(aNode)) { + throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); + } + + // excludeItems is inherited by child containers in an excludeItems view. + var excludeItems = + asQuery(aNode).queryOptions.excludeItems || + asQuery(aNode.parentResult.root).queryOptions.excludeItems; + // expandQueries is inherited by child containers in an expandQueries view. + var expandQueries = + asQuery(aNode).queryOptions.expandQueries && + asQuery(aNode.parentResult.root).queryOptions.expandQueries; + + // If our options are exactly what we expect, directly return the node. + if (excludeItems == aExcludeItems && expandQueries == aExpandQueries) { + return aNode; + } + + // Otherwise, get contents manually. + var query = {}, + options = {}; + this.history.queryStringToQuery(aNode.uri, query, options); + options.value.excludeItems = aExcludeItems; + options.value.expandQueries = aExpandQueries; + return this.history.executeQuery(query.value, options.value).root; + }, + + /** + * Returns true if a container has uri nodes in its first level. + * Has better performance than (getURLsForContainerNode(node).length > 0). + * @param aNode + * The container node to search through. + * @returns true if the node contains uri nodes, false otherwise. + */ + hasChildURIs: function PU_hasChildURIs(aNode) { + if (!this.nodeIsContainer(aNode)) { + return false; + } + + let root = this.getContainerNodeWithOptions(aNode, false, true); + let result = root.parentResult; + let didSuppressNotifications = false; + let wasOpen = root.containerOpen; + if (!wasOpen) { + didSuppressNotifications = result.suppressNotifications; + if (!didSuppressNotifications) { + result.suppressNotifications = true; + } + + root.containerOpen = true; + } + + let found = false; + for (let i = 0; i < root.childCount && !found; i++) { + let child = root.getChild(i); + if (this.nodeIsURI(child)) { + found = true; + } + } + + if (!wasOpen) { + root.containerOpen = false; + if (!didSuppressNotifications) { + result.suppressNotifications = false; + } + } + return found; + }, + + /** + * Returns an array containing all the uris in the first level of the + * passed in container. + * If you only need to know if the node contains uris, use hasChildURIs. + * @param aNode + * The container node to search through + * @returns array of uris in the first level of the container. + */ + getURLsForContainerNode: function PU_getURLsForContainerNode(aNode) { + let urls = []; + if (!this.nodeIsContainer(aNode)) { + return urls; + } + + let root = this.getContainerNodeWithOptions(aNode, false, true); + let result = root.parentResult; + let wasOpen = root.containerOpen; + let didSuppressNotifications = false; + if (!wasOpen) { + didSuppressNotifications = result.suppressNotifications; + if (!didSuppressNotifications) { + result.suppressNotifications = true; + } + + root.containerOpen = true; + } + + for (let i = 0; i < root.childCount; ++i) { + let child = root.getChild(i); + if (this.nodeIsURI(child)) { + urls.push({ + uri: child.uri, + isBookmark: this.nodeIsBookmark(child), + title: child.title, + }); + } + } + + if (!wasOpen) { + root.containerOpen = false; + if (!didSuppressNotifications) { + result.suppressNotifications = false; + } + } + return urls; + }, + + /** + * Gets a shared Sqlite.sys.mjs readonly connection to the Places database, + * usable only for SELECT queries. + * + * This is intended to be used mostly internally, components outside of + * Places should, when possible, use API calls and file bugs to get proper + * APIs, where they are missing. + * Keep in mind the Places DB schema is by no means frozen or even stable. + * Your custom queries can - and will - break overtime. + * + * Example: + * let db = await PlacesUtils.promiseDBConnection(); + * let rows = await db.executeCached(sql, params); + */ + promiseDBConnection: () => lazy.gAsyncDBConnPromised, + + /** + * This is pretty much the same as promiseDBConnection, but with a larger + * page cache, useful for consumers doing large table scans, like the urlbar. + * @see promiseDBConnection + */ + promiseLargeCacheDBConnection: () => lazy.gAsyncDBLargeCacheConnPromised, + get largeCacheDBConnDeferred() { + return gAsyncDBLargeCacheConnDeferred; + }, + + /** + * Returns a Sqlite.sys.mjs wrapper for the main Places connection. Most callers + * should prefer `withConnectionWrapper`, which ensures that all database + * operations finish before the connection is closed. + */ + promiseUnsafeWritableDBConnection: () => lazy.gAsyncDBWrapperPromised, + + /** + * Performs a read/write operation on the Places database through a Sqlite.sys.mjs + * wrapped connection to the Places database. + * + * This is intended to be used only by Places itself, always use APIs if you + * need to modify the Places database. Use promiseDBConnection if you need to + * SELECT from the database and there's no covering API. + * Keep in mind the Places DB schema is by no means frozen or even stable. + * Your custom queries can - and will - break overtime. + * + * As all operations on the Places database are asynchronous, if shutdown + * is initiated while an operation is pending, this could cause dataloss. + * Using `withConnectionWrapper` ensures that shutdown waits until all + * operations are complete before proceeding. + * + * Example: + * await withConnectionWrapper("Bookmarks: Remove a bookmark", Task.async(function*(db) { + * // Proceed with the db, asynchronously. + * // Shutdown will not interrupt operations that take place here. + * })); + * + * @param {string} name The name of the operation. Used for debugging, logging + * and crash reporting. + * @param {function(db)} task A function that takes as argument a Sqlite.sys.mjs + * connection and returns a Promise. Shutdown is guaranteed to not interrupt + * execution of `task`. + */ + async withConnectionWrapper(name, task) { + if (!name) { + throw new TypeError("Expecting a user-readable name"); + } + let db = await lazy.gAsyncDBWrapperPromised; + return db.executeBeforeShutdown(name, task); + }, + + /** + * Gets favicon data for a given page url. + * + * @param {string | URL | nsIURI} aPageUrl + * url of the page to look favicon for. + * @param {number} preferredWidth + * The preferred width of the favicon in pixels. The default value of 0 + * returns the largest icon available. + * @resolves to an object representing a favicon entry, having the following + * properties: { uri, dataLen, data, mimeType } + * @rejects JavaScript exception if the given url has no associated favicon. + */ + promiseFaviconData(aPageUrl, preferredWidth = 0) { + return new Promise((resolve, reject) => { + if (!(aPageUrl instanceof Ci.nsIURI)) { + aPageUrl = PlacesUtils.toURI(aPageUrl); + } + PlacesUtils.favicons.getFaviconDataForPage( + aPageUrl, + function (uri, dataLen, data, mimeType, size) { + if (uri) { + resolve({ uri, dataLen, data, mimeType, size }); + } else { + reject(); + } + }, + preferredWidth + ); + }); + }, + + /** + * Returns the passed URL with a #size ref for the specified size and + * devicePixelRatio. + * + * @param window + * The window where the icon will appear. + * @param href + * The string href we should add the ref to. + * @param size + * The target image size + * @return The URL with the fragment at the end, in the same formar as input. + */ + urlWithSizeRef(window, href, size) { + return ( + href + + (href.includes("#") ? "&" : "#") + + "size=" + + Math.round(size) * window.devicePixelRatio + ); + }, + + /** + * Get the unique id for an item (a bookmark, a folder or a separator) given + * its item id. + * + * @param aItemId + * an item id + * @return {Promise} + * @resolves to the GUID. + * @rejects if aItemId is invalid. + */ + promiseItemGuid(aItemId) { + return GuidHelper.getItemGuid(aItemId); + }, + + /** + * Get the item id for an item (a bookmark, a folder or a separator) given + * its unique id. + * + * @param aGuid + * an item GUID + * @return {Promise} + * @resolves to the item id. + * @rejects if there's no item for the given GUID. + */ + promiseItemId(aGuid) { + return GuidHelper.getItemId(aGuid); + }, + + /** + * Get the item ids for multiple items (a bookmark, a folder or a separator) + * given the unique ids for each item. + * + * @param {Array} aGuids An array of item GUIDs. + * @return {Promise} + * @resolves to a Map of item ids. + * @rejects if not all of the GUIDs could be found. + */ + promiseManyItemIds(aGuids) { + return GuidHelper.getManyItemIds(aGuids); + }, + + /** + * Invalidate the GUID cache for the given itemId. + * + * @param aItemId + * an item id + */ + invalidateCachedGuidFor(aItemId) { + GuidHelper.invalidateCacheForItemId(aItemId); + }, + + /** + * Invalidates the entire GUID cache. + */ + invalidateCachedGuids() { + GuidHelper.invalidateCache(); + }, + + /** + * Asynchronously retrieve a JS-object representation of a places bookmarks + * item (a bookmark, a folder, or a separator) along with all of its + * descendants. + * + * @param [optional] aItemGuid + * the (topmost) item to be queried. If it's not passed, the places + * root is queried: that is, you get a representation of the entire + * bookmarks hierarchy. + * @param [optional] aOptions + * Options for customizing the query behavior, in the form of a JS + * object with any of the following properties: + * - excludeItemsCallback: a function for excluding items, along with + * their descendants. Given an item object (that has everything set + * apart its potential children data), it should return true if the + * item should be excluded. Once an item is excluded, the function + * isn't called for any of its descendants. This isn't called for + * the root item. + * WARNING: since the function may be called for each item, using + * this option can slow down the process significantly if the + * callback does anything that's not relatively trivial. It is + * highly recommended to avoid any synchronous I/O or DB queries. + * - includeItemIds: opt-in to include the deprecated id property. + * Use it if you must. It'll be removed once the switch to GUIDs is + * complete. + * + * @return {Promise} + * @resolves to a JS object that represents either a single item or a + * bookmarks tree. Each node in the tree has the following properties set: + * - guid (string): the item's GUID (same as aItemGuid for the top item). + * - [deprecated] id (number): the item's id. This is only if + * aOptions.includeItemIds is set. + * - type (string): the item's type. @see PlacesUtils.TYPE_X_* + * - typeCode (number): the item's type in numeric format. + * @see PlacesUtils.bookmarks.TYPE_* + * - title (string): the item's title. If it has no title, this property + * isn't set. + * - dateAdded (number, microseconds from the epoch): the date-added value of + * the item. + * - lastModified (number, microseconds from the epoch): the last-modified + * value of the item. + * - index: the item's index under it's parent. + * + * The root object (i.e. the one for aItemGuid) also has the following + * properties set: + * - parentGuid (string): the GUID of the root's parent. This isn't set if + * the root item is the places root. + * - itemsCount (number, not enumerable): the number of items, including the + * root item itself, which are represented in the resolved object. + * + * Bookmark items also have the following properties: + * - uri (string): the item's url. + * - tags (string): csv string of the bookmark's tags. + * - charset (string): the last known charset of the bookmark. + * - keyword (string): the bookmark's keyword (unset if none). + * - postData (string): the bookmark's keyword postData (unset if none). + * - iconUri (string): the bookmark's favicon url. + * The last four properties are not set at all if they're irrelevant (e.g. + * |charset| is not set if no charset was previously set for the bookmark + * url). + * + * Folders may also have the following properties: + * - children (array): the folder's children information, each of them + * having the same set of properties as above. + * + * @rejects if the query failed for any reason. + * @note if aItemGuid points to a non-existent item, the returned promise is + * resolved to null. + */ + async promiseBookmarksTree(aItemGuid = "", aOptions = {}) { + let createItemInfoObject = async function (aRow, aIncludeParentGuid) { + let item = {}; + let copyProps = (...props) => { + for (let prop of props) { + let val = aRow.getResultByName(prop); + if (val !== null) { + item[prop] = val; + } + } + }; + copyProps("guid", "title", "index", "dateAdded", "lastModified"); + if (aIncludeParentGuid) { + copyProps("parentGuid"); + } + + let itemId = aRow.getResultByName("id"); + if (aOptions.includeItemIds) { + item.id = itemId; + } + + // Cache it for promiseItemId consumers regardless. + GuidHelper.updateCache(itemId, item.guid); + + let type = aRow.getResultByName("type"); + item.typeCode = type; + if (type == Ci.nsINavBookmarksService.TYPE_BOOKMARK) { + copyProps("charset", "tags", "iconUri"); + } + + switch (type) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + item.type = PlacesUtils.TYPE_X_MOZ_PLACE; + // If this throws due to an invalid url, the item will be skipped. + try { + item.uri = new URL(aRow.getResultByName("url")).href; + } catch (ex) { + let error = new Error("Invalid bookmark URL"); + error.becauseInvalidURL = true; + throw error; + } + // Keywords are cached, so this should be decently fast. + let entry = await PlacesUtils.keywords.fetch({ url: item.uri }); + if (entry) { + item.keyword = entry.keyword; + item.postData = entry.postData; + } + break; + case PlacesUtils.bookmarks.TYPE_FOLDER: + item.type = PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER; + // Mark root folders. + if (item.guid == PlacesUtils.bookmarks.rootGuid) { + item.root = "placesRoot"; + } else if (item.guid == PlacesUtils.bookmarks.menuGuid) { + item.root = "bookmarksMenuFolder"; + } else if (item.guid == PlacesUtils.bookmarks.unfiledGuid) { + item.root = "unfiledBookmarksFolder"; + } else if (item.guid == PlacesUtils.bookmarks.toolbarGuid) { + item.root = "toolbarFolder"; + } else if (item.guid == PlacesUtils.bookmarks.mobileGuid) { + item.root = "mobileFolder"; + } + break; + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + item.type = PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR; + break; + default: + console.error(`Unexpected bookmark type ${type}`); + break; + } + return item; + }; + + const QUERY_STR = `/* do not warn (bug no): cannot use an index */ + WITH RECURSIVE + descendants(fk, level, type, id, guid, parent, parentGuid, position, + title, dateAdded, lastModified) AS ( + SELECT b1.fk, 0, b1.type, b1.id, b1.guid, b1.parent, + (SELECT guid FROM moz_bookmarks WHERE id = b1.parent), + b1.position, b1.title, b1.dateAdded, b1.lastModified + FROM moz_bookmarks b1 WHERE b1.guid=:item_guid + UNION ALL + SELECT b2.fk, level + 1, b2.type, b2.id, b2.guid, b2.parent, + descendants.guid, b2.position, b2.title, b2.dateAdded, + b2.lastModified + FROM moz_bookmarks b2 + JOIN descendants ON b2.parent = descendants.id AND b2.id <> :tags_folder) + SELECT d.level, d.id, d.guid, d.parent, d.parentGuid, d.type, + d.position AS [index], IFNULL(d.title, '') AS title, d.dateAdded, + d.lastModified, h.url, (SELECT icon_url FROM moz_icons i + JOIN moz_icons_to_pages ON icon_id = i.id + JOIN moz_pages_w_icons pi ON page_id = pi.id + WHERE pi.page_url_hash = hash(h.url) AND pi.page_url = h.url + ORDER BY width DESC LIMIT 1) AS iconUri, + (SELECT GROUP_CONCAT(t.title, ',') + FROM moz_bookmarks b2 + JOIN moz_bookmarks t ON t.id = +b2.parent AND t.parent = :tags_folder + WHERE b2.fk = h.id + ) AS tags, + (SELECT a.content FROM moz_annos a + JOIN moz_anno_attributes n ON a.anno_attribute_id = n.id + WHERE place_id = h.id AND n.name = :charset_anno + ) AS charset + FROM descendants d + LEFT JOIN moz_bookmarks b3 ON b3.id = d.parent + LEFT JOIN moz_places h ON h.id = d.fk + ORDER BY d.level, d.parent, d.position`; + + if (!aItemGuid) { + aItemGuid = this.bookmarks.rootGuid; + } + + let hasExcludeItemsCallback = aOptions.hasOwnProperty( + "excludeItemsCallback" + ); + let excludedParents = new Set(); + let shouldExcludeItem = (aItem, aParentGuid) => { + let exclude = + excludedParents.has(aParentGuid) || + aOptions.excludeItemsCallback(aItem); + if (exclude) { + if (aItem.type == this.TYPE_X_MOZ_PLACE_CONTAINER) { + excludedParents.add(aItem.guid); + } + } + return exclude; + }; + + let rootItem = null; + let parentsMap = new Map(); + let conn = await this.promiseDBConnection(); + let rows = await conn.executeCached(QUERY_STR, { + tags_folder: PlacesUtils.tagsFolderId, + charset_anno: PlacesUtils.CHARSET_ANNO, + item_guid: aItemGuid, + }); + let yieldCounter = 0; + for (let row of rows) { + let item; + if (!rootItem) { + try { + // This is the first row. + rootItem = item = await createItemInfoObject(row, true); + Object.defineProperty(rootItem, "itemsCount", { + value: 1, + writable: true, + enumerable: false, + configurable: false, + }); + } catch (ex) { + console.error("Failed to fetch the data for the root item"); + throw ex; + } + } else { + try { + // Our query guarantees that we always visit parents ahead of their + // children. + item = await createItemInfoObject(row, false); + let parentGuid = row.getResultByName("parentGuid"); + if (hasExcludeItemsCallback && shouldExcludeItem(item, parentGuid)) { + continue; + } + + let parentItem = parentsMap.get(parentGuid); + if ("children" in parentItem) { + parentItem.children.push(item); + } else { + parentItem.children = [item]; + } + + rootItem.itemsCount++; + } catch (ex) { + // This is a bogus child, report and skip it. + console.error("Failed to fetch the data for an item " + ex); + continue; + } + } + + if (item.type == this.TYPE_X_MOZ_PLACE_CONTAINER) { + parentsMap.set(item.guid, item); + } + + // With many bookmarks we end up stealing the CPU - even with yielding! + // So we let everyone else have a go every few items (bug 1186714). + if (++yieldCounter % 50 == 0) { + await new Promise(resolve => { + Services.tm.dispatchToMainThread(resolve); + }); + } + } + + return rootItem; + }, + + /** + * Returns a generator that iterates over `array` and yields slices of no + * more than `chunkLength` elements at a time. + * + * @param {Array} array An array containing zero or more elements. + * @param {number} chunkLength The maximum number of elements in each chunk. + * @yields {Array} A chunk of the array. + * @throws if `chunkLength` is negative or not an integer. + */ + *chunkArray(array, chunkLength) { + if (chunkLength <= 0 || !Number.isInteger(chunkLength)) { + throw new TypeError("Chunk length must be a positive integer"); + } + if (!array.length) { + return; + } + if (array.length <= chunkLength) { + yield array; + return; + } + let startIndex = 0; + while (startIndex < array.length) { + yield array.slice(startIndex, (startIndex += chunkLength)); + } + }, + + /** + * Returns SQL placeholders to bind multiple values into an IN clause. + * @param {Array|number} info + * Array or number of entries to create. + * @param {string} [prefix] + * String prefix to add before the SQL param. + * @param {string} [suffix] + * String suffix to add after the SQL param. + */ + sqlBindPlaceholders(info, prefix = "", suffix = "") { + let length = Array.isArray(info) ? info.length : info; + return new Array(length).fill(prefix + "?" + suffix).join(","); + }, + + /** + * Run some text through md5 and return the hash. + * @param {string} data The string to hash. + * @param {string} [format] Which format of the hash to return: + * - "ascii" for ascii format. + * - "hex" for hex format. + * @returns {string} md5 hash of the input string in the required format. + */ + md5(data, { format = "ascii" } = {}) { + lazy.gCryptoHash.init(lazy.gCryptoHash.MD5); + + // Convert the data to a byte array for hashing + lazy.gCryptoHash.update( + data.split("").map(c => c.charCodeAt(0)), + data.length + ); + switch (format) { + case "hex": + let hash = lazy.gCryptoHash.finish(false); + return Array.from(hash, (c, i) => + hash.charCodeAt(i).toString(16).padStart(2, "0") + ).join(""); + case "ascii": + default: + return lazy.gCryptoHash.finish(true); + } + }, + + /** + * Inserts a new place if one doesn't currently exist. + * + * This should only be used from an API that is connecting this new entry to + * some additional foreign table. Otherwise this will just create an orphan + * entry that could be expired at any time. + * + * @param db + * The database connection to use. + * @param url + * A valid URL object. + * @return {Promise} resolved when the operation is complete. + */ + async maybeInsertPlace(db, url) { + // The IGNORE conflict can trigger on `guid`. + await db.executeCached( + `INSERT OR IGNORE INTO moz_places (url, url_hash, rev_host, hidden, frecency, guid) + VALUES (:url, hash(:url), :rev_host, + (CASE WHEN :url BETWEEN 'place:' AND 'place:' || X'FFFF' THEN 1 ELSE 0 END), + :frecency, + IFNULL((SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url), + GENERATE_GUID())) + `, + { + url: url.href, + rev_host: this.getReversedHost(url), + frecency: url.protocol == "place:" ? 0 : -1, + } + ); + await db.executeCached("DELETE FROM moz_updateoriginsinsert_temp"); + }, + + /** + * Tries to insert a set of new places if they don't exist yet. + * + * This should only be used from an API that is connecting this new entry to + * some additional foreign table. Otherwise this will just create an orphan + * entry that could be expired at any time. + * + * @param db + * The database to use + * @param urls + * An array with all the url objects to insert. + * @return {Promise} resolved when the operation is complete. + */ + async maybeInsertManyPlaces(db, urls) { + await db.executeCached( + `INSERT OR IGNORE INTO moz_places (url, url_hash, rev_host, hidden, frecency, guid) VALUES + (:url, hash(:url), :rev_host, + (CASE WHEN :url BETWEEN 'place:' AND 'place:' || X'FFFF' THEN 1 ELSE 0 END), + :frecency, + IFNULL((SELECT guid FROM moz_places WHERE url_hash = hash(:url) AND url = :url), :maybeguid))`, + urls.map(url => ({ + url: url.href, + rev_host: this.getReversedHost(url), + frecency: url.protocol == "place:" ? 0 : -1, + maybeguid: this.history.makeGuid(), + })) + ); + await db.executeCached("DELETE FROM moz_updateoriginsinsert_temp"); + }, + + /** + * Can be used to detect being in automation to allow specific code paths + * that are normally disallowed. + */ + get isInAutomation() { + return ( + Cu.isInAutomation || Services.env.exists("XPCSHELL_TEST_PROFILE_DIR") + ); + }, + + /** + * Creates a logger. + * Logging level can be controlled through places.loglevel. + * + * @param {string} [prefix] Prefix to use for the logged messages, "::" will + * be appended automatically to the prefix. + * @returns {object} The logger. + */ + getLogger({ prefix = "" } = {}) { + if (!this._logger) { + this._logger = lazy.Log.repository.getLogger("places"); + this._logger.manageLevelFromPref("places.loglevel"); + this._logger.addAppender( + new lazy.Log.ConsoleAppender(new lazy.Log.BasicFormatter()) + ); + } + if (prefix) { + // This is not an early return because it is necessary to invoke getLogger + // at least once before getLoggerWithMessagePrefix; it replaces a + // method of the original logger, rather than using an actual Proxy. + return lazy.Log.repository.getLoggerWithMessagePrefix( + "places", + prefix + " :: " + ); + } + return this._logger; + }, +}; + +XPCOMUtils.defineLazyGetter(PlacesUtils, "history", function () { + let hs = Cc["@mozilla.org/browser/nav-history-service;1"].getService( + Ci.nsINavHistoryService + ); + return Object.freeze( + new Proxy(hs, { + get(target, name) { + let property, object; + if (name in target) { + property = target[name]; + object = target; + } else { + property = lazy.History[name]; + object = lazy.History; + } + if (typeof property == "function") { + return property.bind(object); + } + return property; + }, + set(target, name, val) { + // Forward to the XPCOM object, otherwise don't allow to set properties. + if (name in target) { + target[name] = val; + return true; + } + // This will throw in strict mode. + return false; + }, + }) + ); +}); + +XPCOMUtils.defineLazyServiceGetter( + PlacesUtils, + "favicons", + "@mozilla.org/browser/favicon-service;1", + "nsIFaviconService" +); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "bmsvc", + "@mozilla.org/browser/nav-bookmarks-service;1", + "nsINavBookmarksService" +); +XPCOMUtils.defineLazyGetter(PlacesUtils, "bookmarks", () => { + return Object.freeze( + new Proxy(lazy.Bookmarks, { + get: (target, name) => + lazy.Bookmarks.hasOwnProperty(name) + ? lazy.Bookmarks[name] + : lazy.bmsvc[name], + }) + ); +}); + +XPCOMUtils.defineLazyServiceGetter( + PlacesUtils, + "tagging", + "@mozilla.org/browser/tagging-service;1", + "nsITaggingService" +); + +XPCOMUtils.defineLazyGetter(lazy, "bundle", function () { + const PLACES_STRING_BUNDLE_URI = "chrome://places/locale/places.properties"; + return Services.strings.createBundle(PLACES_STRING_BUNDLE_URI); +}); + +// This is just used as a reasonably-random value for copy & paste / drag operations. +XPCOMUtils.defineLazyGetter(PlacesUtils, "instanceId", () => { + return PlacesUtils.history.makeGuid(); +}); + +/** + * Setup internal databases for closing properly during shutdown. + * + * 1. Places initiates shutdown. + * 2. Before places can move to the step where it closes the low-level connection, + * we need to make sure that we have closed `conn`. + * 3. Before we can close `conn`, we need to make sure that all external clients + * have stopped using `conn`. + * 4. Before we can close Sqlite, we need to close `conn`. + */ +function setupDbForShutdown(conn, name) { + try { + let state = "0. Not started."; + let promiseClosed = new Promise((resolve, reject) => { + // The service initiates shutdown. + // Before it can safely close its connection, we need to make sure + // that we have closed the high-level connection. + try { + PlacesUtils.history.connectionShutdownClient.jsclient.addBlocker( + `${name} closing as part of Places shutdown`, + async function () { + state = "1. Service has initiated shutdown"; + + // At this stage, all external clients have finished using the + // database. We just need to close the high-level connection. + try { + await conn.close(); + state = "2. Closed Sqlite.sys.mjs connection."; + resolve(); + } catch (ex) { + state = "2. Failed to closed Sqlite.sys.mjs connection: " + ex; + reject(ex); + } + }, + () => state + ); + } catch (ex) { + // It's too late to block shutdown, just close the connection. + conn.close(); + reject(ex); + } + }).catch(console.error); + + // Make sure that Sqlite.sys.mjs doesn't close until we are done + // with the high-level connection. + lazy.Sqlite.shutdown.addBlocker( + `${name} must be closed before Sqlite.sys.mjs`, + () => promiseClosed, + () => state + ); + } catch (ex) { + // It's too late to block shutdown, just close the connection. + conn.close(); + throw ex; + } +} + +XPCOMUtils.defineLazyGetter(lazy, "gAsyncDBConnPromised", () => + lazy.Sqlite.cloneStorageConnection({ + connection: PlacesUtils.history.DBConnection, + readOnly: true, + }) + .then(conn => { + setupDbForShutdown(conn, "PlacesUtils read-only connection"); + return conn; + }) + .catch(console.error) +); + +XPCOMUtils.defineLazyGetter(lazy, "gAsyncDBWrapperPromised", () => + lazy.Sqlite.wrapStorageConnection({ + connection: PlacesUtils.history.DBConnection, + }) + .then(conn => { + setupDbForShutdown(conn, "PlacesUtils wrapped connection"); + return conn; + }) + .catch(console.error) +); + +var gAsyncDBLargeCacheConnDeferred = PromiseUtils.defer(); +XPCOMUtils.defineLazyGetter(lazy, "gAsyncDBLargeCacheConnPromised", () => + lazy.Sqlite.cloneStorageConnection({ + connection: PlacesUtils.history.DBConnection, + readOnly: true, + }) + .then(async conn => { + setupDbForShutdown(conn, "PlacesUtils large cache read-only connection"); + // Components like the urlbar often fallback to a table scan due to lack + // of full text indices. A larger cache helps reducing IO and improves + // performance. This value is expected to be larger than the default + // mozStorage value defined as MAX_CACHE_SIZE_BYTES in + // storage/mozStorageConnection.cpp. + await conn.execute("PRAGMA cache_size = -6144"); // 6MiB + // These should be kept in sync with nsPlacesTables.h. + await conn.execute(` + CREATE TEMP TABLE IF NOT EXISTS moz_openpages_temp ( + url TEXT, + userContextId INTEGER, + open_count INTEGER, + PRIMARY KEY (url, userContextId) + )`); + await conn.execute(` + CREATE TEMP TRIGGER IF NOT EXISTS moz_openpages_temp_afterupdate_trigger + AFTER UPDATE OF open_count ON moz_openpages_temp FOR EACH ROW + WHEN NEW.open_count = 0 + BEGIN + DELETE FROM moz_openpages_temp + WHERE url = NEW.url + AND userContextId = NEW.userContextId; + END`); + gAsyncDBLargeCacheConnDeferred.resolve(conn); + return conn; + }) + .catch(console.error) +); + +/** + * The metadata API allows consumers to store simple key-value metadata in + * Places. Keys are strings, values can be any type that SQLite supports: + * numbers (integers and doubles), Booleans, strings, and blobs. Values are + * cached in memory for faster lookups. + * + * Since some consumers set metadata as part of an existing operation or active + * transaction, the API also exposes a `*withConnection` variant for each + * method that takes an open database connection. + */ +PlacesUtils.metadata = { + cache: new Map(), + jsonPrefix: "data:application/json;base64,", + + /** + * Returns the value associated with a metadata key. + * + * @param {String} key + * The metadata key to look up. + * @param {String|Object|Array} defaultValue + * Optional. The default value to return if the value is not present, + * or cannot be parsed. + * @resolves {*} + * The value associated with the key, or the defaultValue if there is one. + * @rejects + * Rejected if the value is not found or it cannot be parsed + * and there is no defaultValue. + */ + get(key, defaultValue) { + return PlacesUtils.withConnectionWrapper("PlacesUtils.metadata.get", db => + this.getWithConnection(db, key, defaultValue) + ); + }, + + /** + * Sets the value for a metadata key. + * + * @param {String} key + * The metadata key to update. + * @param {*} + * The value to associate with the key. + */ + set(key, value) { + return PlacesUtils.withConnectionWrapper("PlacesUtils.metadata.set", db => + this.setWithConnection(db, key, value) + ); + }, + + /** + * Removes the values for the given metadata keys. + * + * @param {String...} + * One or more metadata keys to remove. + */ + delete(...keys) { + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.metadata.delete", + db => this.deleteWithConnection(db, ...keys) + ); + }, + + async getWithConnection(db, key, defaultValue) { + key = this.canonicalizeKey(key); + if (this.cache.has(key)) { + return this.cache.get(key); + } + let rows = await db.executeCached( + ` + SELECT value FROM moz_meta WHERE key = :key`, + { key } + ); + let value = null; + if (rows.length) { + let row = rows[0]; + let rawValue = row.getResultByName("value"); + // Convert blobs back to `Uint8Array`s. + if (row.getTypeOfIndex(0) == row.VALUE_TYPE_BLOB) { + value = new Uint8Array(rawValue); + } else if ( + typeof rawValue == "string" && + rawValue.startsWith(this.jsonPrefix) + ) { + try { + value = JSON.parse( + this._base64Decode(rawValue.substr(this.jsonPrefix.length)) + ); + } catch (ex) { + if (defaultValue !== undefined) { + // We must create a new array in the local scope to avoid a memory + // leak due to the array global object. + value = Cu.cloneInto(defaultValue, {}); + } else { + throw ex; + } + } + } else { + value = rawValue; + } + } else if (defaultValue !== undefined) { + // We must create a new array in the local scope to avoid a memory leak due + // to the array global object. + value = Cu.cloneInto(defaultValue, {}); + } else { + throw new Error(`No data stored for key ${key}`); + } + this.cache.set(key, value); + return value; + }, + + async setWithConnection(db, key, value) { + if (value === null) { + await this.deleteWithConnection(db, key); + return; + } + + let cacheValue = value; + if ( + typeof value == "object" && + ChromeUtils.getClassName(value) != "Uint8Array" + ) { + value = this.jsonPrefix + this._base64Encode(JSON.stringify(value)); + } + + key = this.canonicalizeKey(key); + await db.executeCached( + ` + REPLACE INTO moz_meta (key, value) + VALUES (:key, :value)`, + { key, value } + ); + this.cache.set(key, cacheValue); + }, + + async deleteWithConnection(db, ...keys) { + keys = keys.map(this.canonicalizeKey); + if (!keys.length) { + return; + } + await db.execute( + ` + DELETE FROM moz_meta + WHERE key IN (${new Array(keys.length).fill("?").join(",")})`, + keys + ); + for (let key of keys) { + this.cache.delete(key); + } + }, + + canonicalizeKey(key) { + if (typeof key != "string" || !/^[a-zA-Z0-9\/_]+$/.test(key)) { + throw new TypeError("Invalid metadata key: " + key); + } + return key.toLowerCase(); + }, + + _base64Encode(str) { + return ChromeUtils.base64URLEncode(new TextEncoder().encode(str), { + pad: true, + }); + }, + + _base64Decode(str) { + return new TextDecoder("utf-8").decode( + ChromeUtils.base64URLDecode(str, { padding: "require" }) + ); + }, +}; + +/** + * Keywords management API. + * Sooner or later these keywords will merge with search aliases, this is an + * interim API that should then be replaced by a unified one. + * Keywords are associated with URLs and can have POST data. + * The relations between URLs and keywords are the following: + * - 1 keyword can only point to 1 URL + * - 1 URL can have multiple keywords, iff they differ by POST data (included the empty one). + */ +PlacesUtils.keywords = { + /** + * Fetches a keyword entry based on keyword or URL. + * + * @param keywordOrEntry + * Either the keyword to fetch or an entry providing keyword + * or url property to find keywords for. If both properties are set, + * this returns their intersection. + * @param onResult [optional] + * Callback invoked for each found entry. + * @return {Promise} + * @resolves to an object in the form: { keyword, url, postData }, + * or null if a keyword entry was not found. + */ + fetch(keywordOrEntry, onResult = null) { + if (typeof keywordOrEntry == "string") { + keywordOrEntry = { keyword: keywordOrEntry }; + } + + if ( + keywordOrEntry === null || + typeof keywordOrEntry != "object" || + ("keyword" in keywordOrEntry && typeof keywordOrEntry.keyword != "string") + ) { + throw new Error("Invalid keyword"); + } + + let hasKeyword = "keyword" in keywordOrEntry; + let hasUrl = "url" in keywordOrEntry; + + if (!hasKeyword && !hasUrl) { + throw new Error("At least keyword or url must be provided"); + } + if (onResult && typeof onResult != "function") { + throw new Error("onResult callback must be a valid function"); + } + + if (hasUrl) { + try { + keywordOrEntry.url = BOOKMARK_VALIDATORS.url(keywordOrEntry.url); + } catch (ex) { + throw new Error(keywordOrEntry.url + " is not a valid URL"); + } + } + if (hasKeyword) { + keywordOrEntry.keyword = keywordOrEntry.keyword.trim().toLowerCase(); + } + + let safeOnResult = entry => { + if (onResult) { + try { + onResult(entry); + } catch (ex) { + console.error(ex); + } + } + }; + + return promiseKeywordsCache().then(cache => { + let entries = []; + if (hasKeyword) { + let entry = cache.get(keywordOrEntry.keyword); + if (entry) { + entries.push(entry); + } + } + if (hasUrl) { + for (let entry of cache.values()) { + if (entry.url.href == keywordOrEntry.url.href) { + entries.push(entry); + } + } + } + + entries = entries.filter(e => { + return ( + (!hasUrl || e.url.href == keywordOrEntry.url.href) && + (!hasKeyword || e.keyword == keywordOrEntry.keyword) + ); + }); + + entries.forEach(safeOnResult); + return entries.length ? entries[0] : null; + }); + }, + + /** + * Adds a new keyword and postData for the given URL. + * + * @param keywordEntry + * An object describing the keyword to insert, in the form: + * { + * keyword: non-empty string, + * url: URL or href to associate to the keyword, + * postData: optional POST data to associate to the keyword + * source: The change source, forwarded to all bookmark observers. + * Defaults to nsINavBookmarksService::SOURCE_DEFAULT. + * } + * @note Do not define a postData property if there isn't any POST data. + * Defining an empty string for POST data is equivalent to not having it. + * @resolves when the addition is complete. + */ + insert(keywordEntry) { + if (!keywordEntry || typeof keywordEntry != "object") { + throw new Error("Input should be a valid object"); + } + + if ( + !("keyword" in keywordEntry) || + !keywordEntry.keyword || + typeof keywordEntry.keyword != "string" + ) { + throw new Error("Invalid keyword"); + } + if ( + "postData" in keywordEntry && + keywordEntry.postData && + typeof keywordEntry.postData != "string" + ) { + throw new Error("Invalid POST data"); + } + if (!("url" in keywordEntry)) { + throw new Error("undefined is not a valid URL"); + } + + if (!("source" in keywordEntry)) { + keywordEntry.source = PlacesUtils.bookmarks.SOURCES.DEFAULT; + } + let { keyword, url, source } = keywordEntry; + keyword = keyword.trim().toLowerCase(); + let postData = keywordEntry.postData || ""; + // This also checks href for validity + try { + url = BOOKMARK_VALIDATORS.url(url); + } catch (ex) { + throw new Error(url + " is not a valid URL"); + } + + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.keywords.insert", + async db => { + let cache = await promiseKeywordsCache(); + + // Trying to set the same keyword is a no-op. + let oldEntry = cache.get(keyword); + if ( + oldEntry && + oldEntry.url.href == url.href && + (oldEntry.postData || "") == postData + ) { + return; + } + + // A keyword can only be associated to a single page. + // If another page is using the new keyword, we must update the keyword + // entry. + // Note we cannot use INSERT OR REPLACE cause it wouldn't invoke the delete + // trigger. + if (oldEntry) { + await db.executeCached( + `UPDATE moz_keywords + SET place_id = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url), + post_data = :post_data + WHERE keyword = :keyword + `, + { url: url.href, keyword, post_data: postData } + ); + await notifyKeywordChange(oldEntry.url.href, "", source); + } else { + // An entry for the given page could be missing, in such a case we need to + // create it. The IGNORE conflict can trigger on `guid`. + await db.executeTransaction(async () => { + await PlacesUtils.maybeInsertPlace(db, url); + + // A new keyword could be assigned to an url that already has one, + // then we must replace the old keyword with the new one. + let oldKeywords = []; + for (let entry of cache.values()) { + if ( + entry.url.href == url.href && + (entry.postData || "") == postData + ) { + oldKeywords.push(entry.keyword); + } + } + if (oldKeywords.length) { + for (let oldKeyword of oldKeywords) { + await db.executeCached( + `DELETE FROM moz_keywords WHERE keyword = :oldKeyword`, + { oldKeyword } + ); + cache.delete(oldKeyword); + } + } + + await db.executeCached( + `INSERT INTO moz_keywords (keyword, place_id, post_data) + VALUES (:keyword, (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url), :post_data) + `, + { url: url.href, keyword, post_data: postData } + ); + + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + url, + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta(source) + ); + }); + } + + cache.set(keyword, { keyword, url, postData: postData || null }); + + // In any case, notify about the new keyword. + await notifyKeywordChange(url.href, keyword, source); + } + ); + }, + + /** + * Removes a keyword. + * + * @param keyword + * The keyword to remove. + * @return {Promise} + * @resolves when the removal is complete. + */ + remove(keywordOrEntry) { + if (typeof keywordOrEntry == "string") { + keywordOrEntry = { + keyword: keywordOrEntry, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + }; + } + + if ( + keywordOrEntry === null || + typeof keywordOrEntry != "object" || + !keywordOrEntry.keyword || + typeof keywordOrEntry.keyword != "string" + ) { + throw new Error("Invalid keyword"); + } + + let { keyword, source = Ci.nsINavBookmarksService.SOURCE_DEFAULT } = + keywordOrEntry; + keyword = keywordOrEntry.keyword.trim().toLowerCase(); + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.keywords.remove", + async db => { + let cache = await promiseKeywordsCache(); + if (!cache.has(keyword)) { + return; + } + let { url } = cache.get(keyword); + cache.delete(keyword); + + await db.executeTransaction(async function () { + await db.execute( + `DELETE FROM moz_keywords WHERE keyword = :keyword`, + { keyword } + ); + + await lazy.PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL( + db, + url, + lazy.PlacesSyncUtils.bookmarks.determineSyncChangeDelta(source) + ); + }); + + // Notify bookmarks about the removal. + await notifyKeywordChange(url.href, "", source); + } + ); + }, + + /** + * Moves all (keyword, POST data) pairs from one URL to another, and fires + * observer notifications for all affected bookmarks. If the destination URL + * already has keywords, they will be removed and replaced with the source + * URL's keywords. + * + * @param oldURL + * The source URL. + * @param newURL + * The destination URL. + * @param source + * The change source, forwarded to all bookmark observers. + * @return {Promise} + * @resolves when all keywords have been moved to the destination URL. + */ + reassign(oldURL, newURL, source = PlacesUtils.bookmarks.SOURCES.DEFAULT) { + try { + oldURL = BOOKMARK_VALIDATORS.url(oldURL); + } catch (ex) { + throw new Error(oldURL + " is not a valid source URL"); + } + try { + newURL = BOOKMARK_VALIDATORS.url(newURL); + } catch (ex) { + throw new Error(oldURL + " is not a valid destination URL"); + } + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.keywords.reassign", + async function (db) { + let keywordsToReassign = []; + let keywordsToRemove = []; + let cache = await promiseKeywordsCache(); + for (let [keyword, entry] of cache) { + if (entry.url.href == oldURL.href) { + keywordsToReassign.push(keyword); + } + if (entry.url.href == newURL.href) { + keywordsToRemove.push(keyword); + } + } + if (!keywordsToReassign.length) { + return; + } + + await db.executeTransaction(async function () { + // Remove existing keywords from the new URL. + await db.executeCached( + `DELETE FROM moz_keywords WHERE keyword = :keyword`, + keywordsToRemove.map(keyword => ({ keyword })) + ); + + // Move keywords from the old URL to the new URL. + await db.executeCached( + ` + UPDATE moz_keywords SET + place_id = (SELECT id FROM moz_places + WHERE url_hash = hash(:newURL) AND + url = :newURL) + WHERE place_id = (SELECT id FROM moz_places + WHERE url_hash = hash(:oldURL) AND + url = :oldURL)`, + { newURL: newURL.href, oldURL: oldURL.href } + ); + }); + for (let keyword of keywordsToReassign) { + let entry = cache.get(keyword); + entry.url = newURL; + } + for (let keyword of keywordsToRemove) { + cache.delete(keyword); + } + + if (keywordsToReassign.length) { + // If we moved any keywords, notify that we removed all keywords from + // the old and new URLs, then notify for each moved keyword. + await notifyKeywordChange(oldURL, "", source); + await notifyKeywordChange(newURL, "", source); + for (let keyword of keywordsToReassign) { + await notifyKeywordChange(newURL, keyword, source); + } + } else if (keywordsToRemove.length) { + // If the old URL didn't have any keywords, but the new URL did, just + // notify that we removed all keywords from the new URL. + await notifyKeywordChange(oldURL, "", source); + } + } + ); + }, + + /** + * Removes all orphaned keywords from the given URLs. Orphaned keywords are + * associated with URLs that are no longer bookmarked. If a given URL is still + * bookmarked, its keywords will not be removed. + * + * @param urls + * A list of URLs to check for orphaned keywords. + * @return {Promise} + * @resolves when all keywords have been removed from URLs that are no longer + * bookmarked. + */ + removeFromURLsIfNotBookmarked(urls) { + let hrefs = new Set(); + for (let url of urls) { + try { + url = BOOKMARK_VALIDATORS.url(url); + } catch (ex) { + throw new Error(url + " is not a valid URL"); + } + hrefs.add(url.href); + } + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.keywords.removeFromURLsIfNotBookmarked", + async function (db) { + let keywordsByHref = new Map(); + let cache = await promiseKeywordsCache(); + for (let [keyword, entry] of cache) { + let href = entry.url.href; + if (!hrefs.has(href)) { + continue; + } + if (!keywordsByHref.has(href)) { + keywordsByHref.set(href, [keyword]); + continue; + } + let existingKeywords = keywordsByHref.get(href); + existingKeywords.push(keyword); + } + if (!keywordsByHref.size) { + return; + } + + let placeInfosToRemove = []; + let rows = await db.execute( + ` + SELECT h.id, h.url + FROM moz_places h + JOIN moz_keywords k ON k.place_id = h.id + GROUP BY h.id + HAVING h.foreign_count = count(*) + + (SELECT count(*) + FROM moz_bookmarks b + JOIN moz_bookmarks p ON b.parent = p.id + WHERE p.parent = :tags_root AND b.fk = h.id) + `, + { tags_root: PlacesUtils.tagsFolderId } + ); + for (let row of rows) { + placeInfosToRemove.push({ + placeId: row.getResultByName("id"), + href: row.getResultByName("url"), + }); + } + if (!placeInfosToRemove.length) { + return; + } + + await db.execute( + `DELETE FROM moz_keywords WHERE place_id IN (${Array.from( + placeInfosToRemove.map(info => info.placeId) + ).join()})` + ); + for (let { href } of placeInfosToRemove) { + let keywords = keywordsByHref.get(href); + for (let keyword of keywords) { + cache.delete(keyword); + } + } + } + ); + }, + + /** + * Removes all keywords from all URLs. + * + * @return {Promise} + * @resolves when all keywords have been removed. + */ + eraseEverything() { + return PlacesUtils.withConnectionWrapper( + "PlacesUtils.keywords.eraseEverything", + async function (db) { + let cache = await promiseKeywordsCache(); + if (!cache.size) { + return; + } + await db.executeCached(`DELETE FROM moz_keywords`); + cache.clear(); + } + ); + }, + + /** + * Invalidates the keywords cache, leaving all existing keywords in place. + * The cache will be repopulated on the next `PlacesUtils.keywords.*` call. + * + * @return {Promise} + * @resolves when the cache has been cleared. + */ + invalidateCachedKeywords() { + gKeywordsCachePromise = gKeywordsCachePromise.then(_ => null); + return gKeywordsCachePromise; + }, +}; + +var gKeywordsCachePromise = Promise.resolve(); + +function promiseKeywordsCache() { + let promise = gKeywordsCachePromise.then(function (cache) { + if (cache) { + return cache; + } + return PlacesUtils.withConnectionWrapper( + "PlacesUtils: promiseKeywordsCache", + async db => { + let cache = new Map(); + let rows = await db.execute( + `SELECT keyword, url, post_data + FROM moz_keywords k + JOIN moz_places h ON h.id = k.place_id + ` + ); + let brokenKeywords = []; + for (let row of rows) { + let keyword = row.getResultByName("keyword"); + try { + let entry = { + keyword, + url: new URL(row.getResultByName("url")), + postData: row.getResultByName("post_data") || null, + }; + cache.set(keyword, entry); + } catch (ex) { + // The url is invalid, don't load the keyword and remove it, or it + // would break the whole keywords API. + brokenKeywords.push(keyword); + } + } + if (brokenKeywords.length) { + await db.execute( + `DELETE FROM moz_keywords + WHERE keyword IN (${brokenKeywords.map(JSON.stringify).join(",")}) + ` + ); + } + return cache; + } + ); + }); + gKeywordsCachePromise = promise.catch(_ => {}); + return promise; +} + +// Sometime soon, likely as part of the transition to mozIAsyncBookmarks, +// itemIds will be deprecated in favour of GUIDs, which play much better +// with multiple undo/redo operations. Because these GUIDs are already stored, +// and because we don't want to revise the transactions API once more when this +// happens, transactions are set to work with GUIDs exclusively, in the sense +// that they may never expose itemIds, nor do they accept them as input. +// More importantly, transactions which add or remove items guarantee to +// restore the GUIDs on undo/redo, so that the following transactions that may +// done or undo can assume the items they're interested in are stil accessible +// through the same GUID. +// The current bookmarks API, however, doesn't expose the necessary means for +// working with GUIDs. So, until it does, this helper object accesses the +// Places database directly in order to switch between GUIDs and itemIds, and +// "restore" GUIDs on items re-created items. +var GuidHelper = { + // Cache for GUID<->itemId paris. + guidsForIds: new Map(), + idsForGuids: new Map(), + + async getItemId(aGuid) { + let cached = this.idsForGuids.get(aGuid); + if (cached !== undefined) { + return cached; + } + + let itemId = await PlacesUtils.withConnectionWrapper( + "GuidHelper.getItemId", + async function (db) { + let rows = await db.executeCached( + "SELECT b.id, b.guid from moz_bookmarks b WHERE b.guid = :guid LIMIT 1", + { guid: aGuid } + ); + if (!rows.length) { + throw new Error("no item found for the given GUID"); + } + + return rows[0].getResultByName("id"); + } + ); + + this.updateCache(itemId, aGuid); + return itemId; + }, + + async getManyItemIds(aGuids) { + let uncachedGuids = aGuids.filter(guid => !this.idsForGuids.has(guid)); + if (uncachedGuids.length) { + await PlacesUtils.withConnectionWrapper( + "GuidHelper.getItemId", + async db => { + while (uncachedGuids.length) { + let chunk = uncachedGuids.splice(0, 100); + let rows = await db.executeCached( + `SELECT b.id, b.guid from moz_bookmarks b WHERE + b.guid IN (${"?,".repeat(chunk.length - 1) + "?"}) + LIMIT ${chunk.length}`, + chunk + ); + if (rows.length < chunk.length) { + throw new Error("Not all items were found!"); + } + for (let row of rows) { + this.updateCache( + row.getResultByIndex(0), + row.getResultByIndex(1) + ); + } + } + } + ); + } + return new Map(aGuids.map(guid => [guid, this.idsForGuids.get(guid)])); + }, + + async getItemGuid(aItemId) { + let cached = this.guidsForIds.get(aItemId); + if (cached !== undefined) { + return cached; + } + + let guid = await PlacesUtils.withConnectionWrapper( + "GuidHelper.getItemGuid", + async function (db) { + let rows = await db.executeCached( + "SELECT b.id, b.guid from moz_bookmarks b WHERE b.id = :id LIMIT 1", + { id: aItemId } + ); + if (!rows.length) { + throw new Error("no item found for the given itemId"); + } + + return rows[0].getResultByName("guid"); + } + ); + + this.updateCache(aItemId, guid); + return guid; + }, + + /** + * Updates the cache. + * + * @note This is the only place where the cache should be populated, + * invalidation relies on both Maps being populated at the same time. + */ + updateCache(aItemId, aGuid) { + if (typeof aItemId != "number" || aItemId <= 0) { + throw new Error( + "Trying to update the GUIDs cache with an invalid itemId" + ); + } + if (!PlacesUtils.isValidGuid(aGuid)) { + throw new Error("Trying to update the GUIDs cache with an invalid GUID"); + } + this.ensureObservingRemovedItems(); + this.guidsForIds.set(aItemId, aGuid); + this.idsForGuids.set(aGuid, aItemId); + }, + + invalidateCacheForItemId(aItemId) { + let guid = this.guidsForIds.get(aItemId); + this.guidsForIds.delete(aItemId); + this.idsForGuids.delete(guid); + }, + + invalidateCache() { + this.guidsForIds.clear(); + this.idsForGuids.clear(); + }, + + ensureObservingRemovedItems() { + if (this.addListeners) { + return; + } + /** + * This observers serves two purposes: + * (1) Invalidate cached id<->GUID paris on when items are removed. + * (2) Cache GUIDs given us free of charge by onItemAdded/onItemRemoved. + * So, for exmaple, when the NewBookmark needs the new GUID, we already + * have it cached. + */ + let listener = events => { + for (let event of events) { + switch (event.type) { + case "bookmark-added": + this.updateCache(event.id, event.guid); + this.updateCache(event.parentId, event.parentGuid); + break; + case "bookmark-removed": + this.guidsForIds.delete(event.id); + this.idsForGuids.delete(event.guid); + this.updateCache(event.parentId, event.parentGuid); + break; + } + } + }; + + this.addListeners = true; + PlacesUtils.observers.addListener( + ["bookmark-added", "bookmark-removed"], + listener + ); + PlacesUtils.registerShutdownFunction(() => { + PlacesUtils.observers.removeListener( + ["bookmark-added", "bookmark-removed"], + listener + ); + }); + }, +}; diff --git a/toolkit/components/places/SQLFunctions.cpp b/toolkit/components/places/SQLFunctions.cpp new file mode 100644 index 0000000000..8668950d1c --- /dev/null +++ b/toolkit/components/places/SQLFunctions.cpp @@ -0,0 +1,1481 @@ +/* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/storage.h" +#include "mozilla/StaticPrefs_places.h" +#include "nsString.h" +#include "nsFaviconService.h" +#include "nsNavBookmarks.h" +#include "nsUnicharUtils.h" +#include "nsWhitespaceTokenizer.h" +#include "nsEscape.h" +#include "mozIPlacesAutoComplete.h" +#include "SQLFunctions.h" +#include "nsMathUtils.h" +#include "nsUnicodeProperties.h" +#include "nsUTF8Utils.h" +#include "nsINavHistoryService.h" +#include "nsPrintfCString.h" +#include "nsNavHistory.h" +#include "mozilla/Likely.h" +#include "mozilla/Services.h" +#include "mozilla/Utf8.h" +#include "nsURLHelper.h" +#include "nsVariant.h" +#include "nsICryptoHash.h" + +// Maximum number of chars to search through. +// MatchAutoCompleteFunction won't look for matches over this threshold. +#define MAX_CHARS_TO_SEARCH_THROUGH 255 + +#define SECONDS_PER_DAY 86400 + +using namespace mozilla::storage; + +//////////////////////////////////////////////////////////////////////////////// +//// Anonymous Helpers + +namespace { + +using const_char_iterator = nsACString::const_char_iterator; +using size_type = nsACString::size_type; +using char_type = nsACString::char_type; + +/** + * Scan forward through UTF-8 text until the next potential character that + * could match a given codepoint when lower-cased (false positives are okay). + * This avoids having to actually parse the UTF-8 text, which is slow. + * + * @param aStart + * An iterator pointing to the first character position considered. + * It will be updated by this function. + * @param aEnd + * An interator pointing to past-the-end of the string. + */ +static MOZ_ALWAYS_INLINE void goToNextSearchCandidate( + const_char_iterator& aStart, const const_char_iterator& aEnd, + uint32_t aSearchFor) { + // If the character we search for is ASCII, then we can scan until we find + // it or its ASCII uppercase character, modulo the special cases + // U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE and U+212A KELVIN SIGN + // (which are the only non-ASCII characters that lower-case to ASCII ones). + // Since false positives are okay, we approximate ASCII lower-casing by + // bit-ORing with 0x20, for increased performance. + // + // If the character we search for is *not* ASCII, we can ignore everything + // that is, since all ASCII characters lower-case to ASCII. + // + // Because of how UTF-8 uses high-order bits, this will never land us + // in the middle of a codepoint. + // + // The assumptions about Unicode made here are verified in the test_casing + // gtest. + if (aSearchFor < 128) { + // When searching for I or K, we pick out the first byte of the UTF-8 + // encoding of the corresponding special case character, and look for it + // in the loop below. For other characters we fall back to 0xff, which + // is not a valid UTF-8 byte. + unsigned char target = (unsigned char)(aSearchFor | 0x20); + unsigned char special = 0xff; + if (target == 'i' || target == 'k') { + special = (target == 'i' ? 0xc4 : 0xe2); + } + + while (aStart < aEnd && (unsigned char)(*aStart | 0x20) != target && + (unsigned char)*aStart != special) { + aStart++; + } + } else { + while (aStart < aEnd && (unsigned char)(*aStart) < 128) { + aStart++; + } + } +} + +/** + * Check whether a character position is on a word boundary of a UTF-8 string + * (rather than within a word). We define "within word" to be any position + * between [a-zA-Z] and [a-z] -- this lets us match CamelCase words. + * TODO: support non-latin alphabets. + * + * @param aPos + * An iterator pointing to the character position considered. It must + * *not* be the first byte of a string. + * + * @return true if boundary, false otherwise. + */ +static MOZ_ALWAYS_INLINE bool isOnBoundary(const_char_iterator aPos) { + if ('a' <= *aPos && *aPos <= 'z') { + char prev = static_cast(*(aPos - 1) | 0x20); + return !('a' <= prev && prev <= 'z'); + } + return true; +} + +/** + * Check whether a token string matches a particular position of a source + * string, case insensitively (or optionally, case and diacritic insensitively). + * + * @param aTokenStart + * An iterator pointing to the start of the token string. + * @param aTokenEnd + * An iterator pointing past-the-end of the token string. + * @param aSourceStart + * An iterator pointing to the position of source string to start + * matching at. + * @param aSourceEnd + * An iterator pointing past-the-end of the source string. + * @param aMatchDiacritics + * Whether or not the match is diacritic-sensitive. + * + * @return true if the string [aTokenStart, aTokenEnd) matches the start of + * the string [aSourceStart, aSourceEnd, false otherwise. + */ +static MOZ_ALWAYS_INLINE bool stringMatch(const_char_iterator aTokenStart, + const_char_iterator aTokenEnd, + const_char_iterator aSourceStart, + const_char_iterator aSourceEnd, + bool aMatchDiacritics) { + const_char_iterator tokenCur = aTokenStart, sourceCur = aSourceStart; + + while (tokenCur < aTokenEnd) { + if (sourceCur >= aSourceEnd) { + return false; + } + + bool error; + if (!CaseInsensitiveUTF8CharsEqual(sourceCur, tokenCur, aSourceEnd, + aTokenEnd, &sourceCur, &tokenCur, &error, + aMatchDiacritics)) { + return false; + } + } + + return true; +} + +enum FindInStringBehavior { eFindOnBoundary, eFindAnywhere }; + +/** + * Common implementation for findAnywhere and findOnBoundary. + * + * @param aToken + * The token we're searching for + * @param aSourceString + * The string in which we're searching + * @param aBehavior + * eFindOnBoundary if we should only consider matchines which occur on + * word boundaries, or eFindAnywhere if we should consider matches + * which appear anywhere. + * + * @return true if aToken was found in aSourceString, false otherwise. + */ +static bool findInString(const nsDependentCSubstring& aToken, + const nsACString& aSourceString, + FindInStringBehavior aBehavior) { + // GetLowerUTF8Codepoint assumes that there's at least one byte in + // the string, so don't pass an empty token here. + MOZ_ASSERT(!aToken.IsEmpty(), "Don't search for an empty token!"); + + // We cannot match anything if there is nothing to search. + if (aSourceString.IsEmpty()) { + return false; + } + + const nsNavHistory* history = nsNavHistory::GetConstHistoryService(); + bool matchDiacritics = history && history->MatchDiacritics(); + + const_char_iterator tokenStart(aToken.BeginReading()), + tokenEnd(aToken.EndReading()), tokenNext, + sourceStart(aSourceString.BeginReading()), + sourceEnd(aSourceString.EndReading()), sourceCur(sourceStart), sourceNext; + + uint32_t tokenFirstChar = + GetLowerUTF8Codepoint(tokenStart, tokenEnd, &tokenNext); + if (tokenFirstChar == uint32_t(-1)) { + return false; + } + if (!matchDiacritics) { + tokenFirstChar = ToNaked(tokenFirstChar); + } + + for (;;) { + if (matchDiacritics) { + // Scan forward to the next viable candidate (if any). + goToNextSearchCandidate(sourceCur, sourceEnd, tokenFirstChar); + } + if (sourceCur == sourceEnd) { + break; + } + + // Check whether the first character in the token matches the character + // at sourceCur. At the same time, get a pointer to the next character + // in the source. + uint32_t sourceFirstChar = + GetLowerUTF8Codepoint(sourceCur, sourceEnd, &sourceNext); + if (sourceFirstChar == uint32_t(-1)) { + return false; + } + if (!matchDiacritics) { + sourceFirstChar = ToNaked(sourceFirstChar); + } + + if (sourceFirstChar == tokenFirstChar && + (aBehavior != eFindOnBoundary || sourceCur == sourceStart || + isOnBoundary(sourceCur)) && + stringMatch(tokenNext, tokenEnd, sourceNext, sourceEnd, + matchDiacritics)) { + return true; + } + + sourceCur = sourceNext; + } + + return false; +} + +static MOZ_ALWAYS_INLINE nsDependentCString +getSharedUTF8String(mozIStorageValueArray* aValues, uint32_t aIndex) { + uint32_t len; + const char* str = aValues->AsSharedUTF8String(aIndex, &len); + if (!str) { + return nsDependentCString("", (size_t)0); + } + return nsDependentCString(str, len); +} + +/** + * Gets the length of the prefix in a URI spec. "Prefix" is defined to be the + * scheme, colon, and, if present, two slashes. + * + * Examples: + * + * http://example.com + * ~~~~~~~ + * => length == 7 + * + * foo:example + * ~~~~ + * => length == 4 + * + * not a spec + * => length == 0 + * + * @param aSpec + * A URI spec, or a string that may be a URI spec. + * @return The length of the prefix in the spec. If there isn't a prefix, + * returns 0. + */ +static MOZ_ALWAYS_INLINE size_type getPrefixLength(const nsACString& aSpec) { + // To keep the search bounded, look at 64 characters at most. The longest + // IANA schemes are ~30, so double that and round up to a nice number. + size_type length = std::min(static_cast(64), aSpec.Length()); + for (size_type i = 0; i < length; ++i) { + if (aSpec[i] == static_cast(':')) { + // Found the ':'. Now skip past "//", if present. + if (i + 2 < aSpec.Length() && + aSpec[i + 1] == static_cast('/') && + aSpec[i + 2] == static_cast('/')) { + i += 2; + } + return i + 1; + } + } + return 0; +} + +/** + * Gets the index in a URI spec of the host and port substring and optionally + * its length. + * + * Examples: + * + * http://example.com/ + * ~~~~~~~~~~~ + * => index == 7, length == 11 + * + * http://example.com:8888/ + * ~~~~~~~~~~~~~~~~ + * => index == 7, length == 16 + * + * http://user:pass@example.com/ + * ~~~~~~~~~~~ + * => index == 17, length == 11 + * + * foo:example + * ~~~~~~~ + * => index == 4, length == 7 + * + * not a spec + * ~~~~~~~~~~ + * => index == 0, length == 10 + * + * @param aSpec + * A URI spec, or a string that may be a URI spec. + * @param _hostAndPortLength + * The length of the host and port substring is returned through this + * param. Pass null if you don't care. + * @return The length of the host and port substring in the spec. If aSpec + * doesn't look like a URI, then the entire aSpec is assumed to be a + * "host and port", and this returns 0, and _hostAndPortLength will be + * the length of aSpec. + */ +static MOZ_ALWAYS_INLINE size_type +indexOfHostAndPort(const nsACString& aSpec, size_type* _hostAndPortLength) { + size_type index = getPrefixLength(aSpec); + size_type i = index; + for (; i < aSpec.Length(); ++i) { + // RFC 3986 (URIs): The origin ("authority") is terminated by '/', '?', or + // '#' (or the end of the URI). + if (aSpec[i] == static_cast('/') || + aSpec[i] == static_cast('?') || + aSpec[i] == static_cast('#')) { + break; + } + // RFC 3986: '@' marks the end of the userinfo component. + if (aSpec[i] == static_cast('@')) { + index = i + 1; + } + } + if (_hostAndPortLength) { + *_hostAndPortLength = i - index; + } + return index; +} + +} // End anonymous namespace + +namespace mozilla::places { + +//////////////////////////////////////////////////////////////////////////////// +//// AutoComplete Matching Function + +/* static */ +nsresult MatchAutoCompleteFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new MatchAutoCompleteFunction(); + + nsresult rv = aDBConn->CreateFunction("autocomplete_match"_ns, + kArgIndexLength, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/* static */ +nsDependentCSubstring MatchAutoCompleteFunction::fixupURISpec( + const nsACString& aURISpec, int32_t aMatchBehavior, nsACString& aSpecBuf) { + nsDependentCSubstring fixedSpec; + + // Try to unescape the string. If that succeeds and yields a different + // string which is also valid UTF-8, we'll use it. + // Otherwise, we will simply use our original string. + bool unescaped = + NS_UnescapeURL(aURISpec.BeginReading(), (int32_t)aURISpec.Length(), + esc_SkipControl, aSpecBuf); + if (unescaped && IsUtf8(aSpecBuf)) { + fixedSpec.Rebind(aSpecBuf, 0); + } else { + fixedSpec.Rebind(aURISpec, 0); + } + + if (aMatchBehavior == mozIPlacesAutoComplete::MATCH_ANYWHERE_UNMODIFIED) { + return fixedSpec; + } + + if (StringBeginsWith(fixedSpec, "http://"_ns)) { + fixedSpec.Rebind(fixedSpec, 7); + } else if (StringBeginsWith(fixedSpec, "https://"_ns)) { + fixedSpec.Rebind(fixedSpec, 8); + } else if (StringBeginsWith(fixedSpec, "ftp://"_ns)) { + fixedSpec.Rebind(fixedSpec, 6); + } + + return fixedSpec; +} + +/* static */ +bool MatchAutoCompleteFunction::findAnywhere( + const nsDependentCSubstring& aToken, const nsACString& aSourceString) { + // We can't use FindInReadable here; it works only for ASCII. + + return findInString(aToken, aSourceString, eFindAnywhere); +} + +/* static */ +bool MatchAutoCompleteFunction::findOnBoundary( + const nsDependentCSubstring& aToken, const nsACString& aSourceString) { + return findInString(aToken, aSourceString, eFindOnBoundary); +} + +/* static */ +MatchAutoCompleteFunction::searchFunctionPtr +MatchAutoCompleteFunction::getSearchFunction(int32_t aBehavior) { + switch (aBehavior) { + case mozIPlacesAutoComplete::MATCH_ANYWHERE: + case mozIPlacesAutoComplete::MATCH_ANYWHERE_UNMODIFIED: + return findAnywhere; + case mozIPlacesAutoComplete::MATCH_BOUNDARY: + default: + return findOnBoundary; + }; +} + +NS_IMPL_ISUPPORTS(MatchAutoCompleteFunction, mozIStorageFunction) + +MatchAutoCompleteFunction::MatchAutoCompleteFunction() + : mCachedZero(new IntegerVariant(0)), mCachedOne(new IntegerVariant(1)) { + static_assert(IntegerVariant::HasThreadSafeRefCnt::value, + "Caching assumes that variants have thread-safe refcounting"); +} + +NS_IMETHODIMP +MatchAutoCompleteFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Macro to make the code a bit cleaner and easier to read. Operates on + // searchBehavior. + int32_t searchBehavior = aArguments->AsInt32(kArgIndexSearchBehavior); +#define HAS_BEHAVIOR(aBitName) \ + (searchBehavior & mozIPlacesAutoComplete::BEHAVIOR_##aBitName) + + nsDependentCString searchString = + getSharedUTF8String(aArguments, kArgSearchString); + nsDependentCString url = getSharedUTF8String(aArguments, kArgIndexURL); + + int32_t matchBehavior = aArguments->AsInt32(kArgIndexMatchBehavior); + + // We only want to filter javascript: URLs if we are not supposed to search + // for them, and the search does not start with "javascript:". + if (matchBehavior != mozIPlacesAutoComplete::MATCH_ANYWHERE_UNMODIFIED && + StringBeginsWith(url, "javascript:"_ns) && !HAS_BEHAVIOR(JAVASCRIPT) && + !StringBeginsWith(searchString, "javascript:"_ns)) { + *_result = do_AddRef(mCachedZero).take(); + return NS_OK; + } + + int32_t visitCount = aArguments->AsInt32(kArgIndexVisitCount); + // Filtering on typed is no more used by Firefox, it is still being used by + // comm-central clients. + bool typed = aArguments->AsInt32(kArgIndexTyped) != 0; + bool bookmark = aArguments->AsInt32(kArgIndexBookmark) != 0; + nsDependentCString tags = getSharedUTF8String(aArguments, kArgIndexTags); + int32_t openPageCount = aArguments->AsInt32(kArgIndexOpenPageCount); + bool matches = false; + if (HAS_BEHAVIOR(RESTRICT)) { + // Make sure we match all the filter requirements. If a given restriction + // is active, make sure the corresponding condition is not true. + matches = (!HAS_BEHAVIOR(HISTORY) || visitCount > 0) && + (!HAS_BEHAVIOR(TYPED) || typed) && + (!HAS_BEHAVIOR(BOOKMARK) || bookmark) && + (!HAS_BEHAVIOR(TAG) || !tags.IsVoid()) && + (!HAS_BEHAVIOR(OPENPAGE) || openPageCount > 0); + } else { + // Make sure that we match all the filter requirements and that the + // corresponding condition is true if at least a given restriction is + // active. + matches = (HAS_BEHAVIOR(HISTORY) && visitCount > 0) || + (HAS_BEHAVIOR(TYPED) && typed) || + (HAS_BEHAVIOR(BOOKMARK) && bookmark) || + (HAS_BEHAVIOR(TAG) && !tags.IsVoid()) || + (HAS_BEHAVIOR(OPENPAGE) && openPageCount > 0); + } + + if (!matches) { + *_result = do_AddRef(mCachedZero).take(); + return NS_OK; + } + + // Obtain our search function. + searchFunctionPtr searchFunction = getSearchFunction(matchBehavior); + + // Clean up our URI spec and prepare it for searching. + nsCString fixedUrlBuf; + nsDependentCSubstring fixedUrl = + fixupURISpec(url, matchBehavior, fixedUrlBuf); + // Limit the number of chars we search through. + const nsDependentCSubstring& trimmedUrl = + Substring(fixedUrl, 0, MAX_CHARS_TO_SEARCH_THROUGH); + + nsDependentCString title = getSharedUTF8String(aArguments, kArgIndexTitle); + // Limit the number of chars we search through. + const nsDependentCSubstring& trimmedTitle = + Substring(title, 0, MAX_CHARS_TO_SEARCH_THROUGH); + + // Caller may pass a fallback title, for example in case of bookmarks or + // snapshots, one may want to search both the user provided title and the + // history one. + nsDependentCString fallbackTitle = + getSharedUTF8String(aArguments, kArgIndexFallbackTitle); + // Limit the number of chars we search through. + const nsDependentCSubstring& trimmedFallbackTitle = + Substring(fallbackTitle, 0, MAX_CHARS_TO_SEARCH_THROUGH); + + // Determine if every token matches either the bookmark title, tags, page + // title, or page URL. + nsCWhitespaceTokenizer tokenizer(searchString); + while (matches && tokenizer.hasMoreTokens()) { + const nsDependentCSubstring& token = tokenizer.nextToken(); + + if (HAS_BEHAVIOR(TITLE) && HAS_BEHAVIOR(URL)) { + matches = (searchFunction(token, trimmedTitle) || + searchFunction(token, trimmedFallbackTitle) || + searchFunction(token, tags)) && + searchFunction(token, trimmedUrl); + } else if (HAS_BEHAVIOR(TITLE)) { + matches = searchFunction(token, trimmedTitle) || + searchFunction(token, trimmedFallbackTitle) || + searchFunction(token, tags); + } else if (HAS_BEHAVIOR(URL)) { + matches = searchFunction(token, trimmedUrl); + } else { + matches = searchFunction(token, trimmedTitle) || + searchFunction(token, trimmedFallbackTitle) || + searchFunction(token, tags) || + searchFunction(token, trimmedUrl); + } + } + + *_result = do_AddRef(matches ? mCachedOne : mCachedZero).take(); + return NS_OK; +#undef HAS_BEHAVIOR +} + +//////////////////////////////////////////////////////////////////////////////// +//// Frecency Calculation Function + +/* static */ +nsresult CalculateFrecencyFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new CalculateFrecencyFunction(); + + nsresult rv = aDBConn->CreateFunction("calculate_frecency"_ns, -1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(CalculateFrecencyFunction, mozIStorageFunction) + +NS_IMETHODIMP +CalculateFrecencyFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Fetch arguments. Use default values if they were omitted. + uint32_t numEntries; + nsresult rv = aArguments->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numEntries <= 2, "unexpected number of arguments"); + + int64_t pageId = aArguments->AsInt64(0); + MOZ_ASSERT(pageId > 0, "Should always pass a valid page id"); + if (pageId <= 0) { + *_result = MakeAndAddRef(0).take(); + return NS_OK; + } + + enum RedirectBonus { eUnknown, eRedirect, eNormal }; + + RedirectBonus mostRecentVisitBonus = eUnknown; + + if (numEntries > 1) { + mostRecentVisitBonus = aArguments->AsInt32(1) ? eRedirect : eNormal; + } + + int32_t typed = 0; + int32_t visitCount = 0; + PRTime mostRecentBookmarkTime = 0; + int32_t isQuery = 0; + float pointsForSampledVisits = 0.0f; + int32_t numSampledVisits = 0; + int32_t bonus = 0; + + // This is a const version of the history object for thread-safety. + const nsNavHistory* history = nsNavHistory::GetConstHistoryService(); + NS_ENSURE_STATE(history); + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + + // Fetch the page stats from the database. + { + nsCOMPtr getPageInfo = DB->GetStatement( + "SELECT typed, visit_count, MAX(dateAdded), " + "(substr(url, 0, 7) = 'place:') " + "FROM moz_places h " + "LEFT JOIN moz_bookmarks ON fk = h.id " + "WHERE h.id = :page_id"); + NS_ENSURE_STATE(getPageInfo); + mozStorageStatementScoper infoScoper(getPageInfo); + + rv = getPageInfo->BindInt64ByName("page_id"_ns, pageId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult = false; + rv = getPageInfo->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_UNEXPECTED); + + rv = getPageInfo->GetInt32(0, &typed); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(1, &visitCount); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt64(2, &mostRecentBookmarkTime); + NS_ENSURE_SUCCESS(rv, rv); + rv = getPageInfo->GetInt32(3, &isQuery); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (visitCount > 0) { + // Get a sample of the last visits to the page, to calculate its weight. + // In case the visit is a redirect target, calculate the frecency + // as if the original page was visited. + // If it's a redirect source, we may want to use a lower bonus. + nsCString redirectsTransitionFragment = nsPrintfCString( + "%d AND %d ", nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT, + nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY); + nsCOMPtr getVisits = DB->GetStatement( + nsLiteralCString( + "/* do not warn (bug 659740 - SQLite may ignore index if few " + "visits exist) */" + "SELECT " + "IFNULL(origin.visit_type, v.visit_type) AS visit_type, " + "target.visit_type AS target_visit_type, " + "ROUND((strftime('%s','now','localtime','utc') - " + "v.visit_date/1000000)/86400) AS age_in_days, " + "v.source AS visit_source " + "FROM moz_historyvisits v " + "LEFT JOIN moz_historyvisits origin ON origin.id = v.from_visit " + "AND v.visit_type BETWEEN ") + + redirectsTransitionFragment + + nsLiteralCString( + "LEFT JOIN moz_historyvisits target ON v.id = target.from_visit " + "AND target.visit_type BETWEEN ") + + redirectsTransitionFragment + + nsLiteralCString("WHERE v.place_id = :page_id " + "ORDER BY v.visit_date DESC " + "LIMIT :max_visits ")); + NS_ENSURE_STATE(getVisits); + mozStorageStatementScoper visitsScoper(getVisits); + rv = getVisits->BindInt64ByName("page_id"_ns, pageId); + NS_ENSURE_SUCCESS(rv, rv); + rv = getVisits->BindInt32ByName("max_visits"_ns, + history->GetNumVisitsForFrecency()); + NS_ENSURE_SUCCESS(rv, rv); + + // Fetch only a limited number of recent visits. + bool hasResult = false; + while (NS_SUCCEEDED(getVisits->ExecuteStep(&hasResult)) && hasResult) { + // If this is a redirect target, we'll use the visitType of the source, + // otherwise the actual visitType. + int32_t visitType = getVisits->AsInt32(0); + + // When adding a new visit, we should haved passed-in whether we should + // use the redirect bonus. We can't fetch this information from the + // database, because we only store redirect targets. + // For older visits we extract the value from the database. + bool useRedirectBonus = mostRecentVisitBonus == eRedirect; + if (mostRecentVisitBonus == eUnknown || numSampledVisits > 0) { + int32_t targetVisitType = getVisits->AsInt32(1); + useRedirectBonus = + targetVisitType == + nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT || + (targetVisitType == + nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY && + visitType != nsINavHistoryService::TRANSITION_TYPED); + } + + uint32_t visitSource = getVisits->AsInt32(3); + if (mostRecentBookmarkTime) { + // For bookmarked visit, add full bonus. + bonus = history->GetFrecencyTransitionBonus(visitType, true, + useRedirectBonus); + bonus += history->GetFrecencyTransitionBonus( + nsINavHistoryService::TRANSITION_BOOKMARK, true); + } else if (visitSource == nsINavHistoryService::VISIT_SOURCE_ORGANIC) { + bonus = history->GetFrecencyTransitionBonus(visitType, true, + useRedirectBonus); + } else if (visitSource == nsINavHistoryService::VISIT_SOURCE_SEARCHED) { + bonus = history->GetFrecencyTransitionBonus( + nsINavHistoryService::TRANSITION_LINK, true, useRedirectBonus); + } + + // If bonus was zero, we can skip the work to determine the weight. + if (bonus) { + int32_t ageInDays = getVisits->AsInt32(2); + int32_t weight = history->GetFrecencyAgedWeight(ageInDays); + pointsForSampledVisits += ((float)weight * ((float)bonus / 100.0f)); + } + + numSampledVisits++; + } + } + + // If we sampled some visits for this page, use the calculated weight. + if (numSampledVisits) { + // We were unable to calculate points, maybe cause all the visits in the + // sample had a zero bonus. Though, we know the page has some past valid + // visit, or visit_count would be zero. Thus we set the frecency to + // -1, so they are still shown in autocomplete. + if (pointsForSampledVisits == 0.0f) { + *_result = MakeAndAddRef(-1).take(); + } else { + // Estimate frecency using the sampled visits. + // Use ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete. + *_result = + MakeAndAddRef( + (int32_t)ceilf((float)visitCount * ceilf(pointsForSampledVisits) / + (float)numSampledVisits)) + .take(); + } + return NS_OK; + } + + // Otherwise this page has no visits, it may be bookmarked. + if (!mostRecentBookmarkTime || isQuery) { + *_result = MakeAndAddRef(0).take(); + return NS_OK; + } + + MOZ_ASSERT(bonus == 0, "Pages should arrive here with 0 bonus"); + MOZ_ASSERT(mostRecentBookmarkTime > 0, "This should be a bookmarked page"); + + // For unvisited bookmarks, produce a non-zero frecency, so that they show + // up in URL bar autocomplete. + // Make it so something bookmarked and typed will have a higher frecency + // than something just typed or just bookmarked. + bonus += history->GetFrecencyTransitionBonus( + nsINavHistoryService::TRANSITION_BOOKMARK, false); + if (typed) { + bonus += history->GetFrecencyTransitionBonus( + nsINavHistoryService::TRANSITION_TYPED, false); + } + + // Use an appropriate bucket depending on the bookmark creation date. + int32_t bookmarkAgeInDays = + static_cast((PR_Now() - mostRecentBookmarkTime) / + ((PRTime)SECONDS_PER_DAY * (PRTime)PR_USEC_PER_SEC)); + + pointsForSampledVisits = + (float)history->GetFrecencyAgedWeight(bookmarkAgeInDays) * + ((float)bonus / 100.0f); + + // use ceilf() so that we don't round down to 0, which + // would cause us to completely ignore the place during autocomplete + *_result = + MakeAndAddRef((int32_t)ceilf(pointsForSampledVisits)) + .take(); + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Frecency Calculation Function + +/* static */ +nsresult CalculateAltFrecencyFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = + new CalculateAltFrecencyFunction(); + + nsresult rv = + aDBConn->CreateFunction("calculate_alt_frecency"_ns, -1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(CalculateAltFrecencyFunction, mozIStorageFunction) + +NS_IMETHODIMP +CalculateAltFrecencyFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Fetch arguments. Use default values if they were omitted. + uint32_t numEntries; + nsresult rv = aArguments->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numEntries <= 2, "unexpected number of arguments"); + + int64_t pageId = aArguments->AsInt64(0); + MOZ_ASSERT(pageId > 0, "Should always pass a valid page id"); + if (pageId <= 0) { + *_result = MakeAndAddRef(0).take(); + return NS_OK; + } + + int32_t isRedirect = 0; + if (numEntries > 1) { + isRedirect = aArguments->AsInt32(1); + } + // This is a const version of the history object for thread-safety. + const nsNavHistory* history = nsNavHistory::GetConstHistoryService(); + NS_ENSURE_STATE(history); + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + + /* + Exponentially decay each visit with an half-life of halfLifeDays. + Score per each visit is a weight exponentially decayed depending on how + far away is from a reference date, that is the most recent visit date. + The weight for each visit is assigned depending on the visit type and other + information (bookmarked, a redirect, a typed entry). + If a page has no visits, consider a single visit with an high weight and + decay its score using the bookmark date as reference time. + Frecency is the sum of all the scores / number of samples. + The final score is further decayed using the same half-life. + To avoid having to decay the score manually, the stored value is the number + of days after which the score would become 1. + + TODO: Add reference link to source docs here. + */ + nsCOMPtr stmt = DB->GetStatement( + "WITH " + "lambda (lambda) AS ( " + " SELECT ln(2) / :halfLifeDays " + "), " + "visits (days, weight) AS ( " + " SELECT " + " v.visit_date / 86400000000, " + " (SELECT CASE " + " WHEN IFNULL(s.visit_type, v.visit_type) = 3 " // is a bookmark + " OR ( v.source <> 3 " // is a search + " AND IFNULL(s.visit_type, v.visit_type) = 2 " // is typed + " AND t.id IS NULL AND NOT :isRedirect " // not a redirect + " ) " + " THEN :highWeight " + " WHEN t.id IS NULL AND NOT :isRedirect " // not a redirect + " AND IFNULL(s.visit_type, v.visit_type) NOT IN (4, 8, 9) " + " THEN :mediumWeight " + " ELSE :lowWeight " + " END) " + " FROM moz_historyvisits v " + // If it's a redirect target, use the visit_type of the source. + " LEFT JOIN moz_historyvisits s ON s.id = v.from_visit " + " AND v.visit_type IN (5,6) " + // If it's a redirect, use a low weight. + " LEFT JOIN moz_historyvisits t ON t.from_visit = v.id " + " AND t.visit_type IN (5,6) " + " WHERE v.place_id = :pageId " + " ORDER BY v.visit_date DESC " + " LIMIT :numSampledVisits " + "), " + "bookmark (days, weight) AS ( " + " SELECT dateAdded / 86400000000, 100 " + " FROM moz_bookmarks " + " WHERE fk = :pageId " + " ORDER BY dateAdded DESC " + " LIMIT 1 " + "), " + "samples (days, weight) AS ( " + " SELECT * FROM bookmark WHERE (SELECT count(*) FROM visits) = 0 " + " UNION ALL " + " SELECT * FROM visits " + "), " + "reference (days, samples_count) AS ( " + " SELECT max(samples.days), count(*) FROM samples " + "), " + "scores (score) AS ( " + " SELECT (weight * exp(-lambda * (samples.days - reference.days))) " + " FROM samples, reference, lambda " + ") " + "SELECT CASE " + "WHEN (substr(url, 0, 7) = 'place:') THEN 0 " + "ELSE " + " reference.days + CAST (( " + " ln( " + " (sum(score) / samples_count * MAX(visit_count, samples_count)) * " + " exp(-lambda) " + " ) / lambda " + " ) AS INTEGER) " + "END " + "FROM moz_places h, reference, lambda, scores " + "WHERE h.id = :pageId"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper infoScoper(stmt); + + rv = stmt->BindInt64ByName("pageId"_ns, pageId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("isRedirect"_ns, isRedirect); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName( + "halfLifeDays"_ns, + StaticPrefs::places_frecency_pages_alternative_halfLifeDays_AtStartup()); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName( + "numSampledVisits"_ns, + StaticPrefs:: + places_frecency_pages_alternative_numSampledVisits_AtStartup()); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName( + "lowWeight"_ns, + StaticPrefs::places_frecency_pages_alternative_lowWeight_AtStartup()); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName( + "mediumWeight"_ns, + StaticPrefs::places_frecency_pages_alternative_mediumWeight_AtStartup()); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName( + "highWeight"_ns, + StaticPrefs::places_frecency_pages_alternative_highWeight_AtStartup()); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult = false; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_UNEXPECTED); + + bool isNull; + if (NS_SUCCEEDED(stmt->GetIsNull(0, &isNull)) && isNull) { + *_result = MakeAndAddRef().take(); + } else { + int32_t score; + rv = stmt->GetInt32(0, &score); + NS_ENSURE_SUCCESS(rv, rv); + *_result = MakeAndAddRef(score).take(); + } + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// GUID Creation Function + +/* static */ +nsresult GenerateGUIDFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new GenerateGUIDFunction(); + nsresult rv = aDBConn->CreateFunction("generate_guid"_ns, 0, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(GenerateGUIDFunction, mozIStorageFunction) + +NS_IMETHODIMP +GenerateGUIDFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + nsAutoCString guid; + nsresult rv = GenerateGUID(guid); + NS_ENSURE_SUCCESS(rv, rv); + + *_result = MakeAndAddRef(guid).take(); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// GUID Validation Function + +/* static */ +nsresult IsValidGUIDFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new IsValidGUIDFunction(); + return aDBConn->CreateFunction("is_valid_guid"_ns, 1, function); +} + +NS_IMPL_ISUPPORTS(IsValidGUIDFunction, mozIStorageFunction) + +NS_IMETHODIMP +IsValidGUIDFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + nsAutoCString guid; + aArguments->GetUTF8String(0, guid); + + RefPtr result = new nsVariant(); + result->SetAsBool(IsValidGUID(guid)); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Get Unreversed Host Function + +/* static */ +nsresult GetUnreversedHostFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new GetUnreversedHostFunction(); + nsresult rv = aDBConn->CreateFunction("get_unreversed_host"_ns, 1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(GetUnreversedHostFunction, mozIStorageFunction) + +NS_IMETHODIMP +GetUnreversedHostFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + nsAutoString src; + aArguments->GetString(0, src); + + RefPtr result = new nsVariant(); + + if (src.Length() > 1) { + src.Truncate(src.Length() - 1); + nsAutoString dest; + ReverseString(src, dest); + result->SetAsAString(dest); + } else { + result->SetAsAString(u""_ns); + } + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Fixup URL Function + +/* static */ +nsresult FixupURLFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new FixupURLFunction(); + nsresult rv = aDBConn->CreateFunction("fixup_url"_ns, 1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(FixupURLFunction, mozIStorageFunction) + +NS_IMETHODIMP +FixupURLFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + nsAutoString src; + aArguments->GetString(0, src); + + RefPtr result = new nsVariant(); + + if (StringBeginsWith(src, u"http://"_ns)) { + src.Cut(0, 7); + } else if (StringBeginsWith(src, u"https://"_ns)) { + src.Cut(0, 8); + } else if (StringBeginsWith(src, u"ftp://"_ns)) { + src.Cut(0, 6); + } + + // Remove common URL hostname prefixes + if (StringBeginsWith(src, u"www."_ns)) { + src.Cut(0, 4); + } + + result->SetAsAString(src); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Store Last Inserted Id Function + +/* static */ +nsresult StoreLastInsertedIdFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = + new StoreLastInsertedIdFunction(); + nsresult rv = + aDBConn->CreateFunction("store_last_inserted_id"_ns, 2, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(StoreLastInsertedIdFunction, mozIStorageFunction) + +NS_IMETHODIMP +StoreLastInsertedIdFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + uint32_t numArgs; + nsresult rv = aArgs->GetNumEntries(&numArgs); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numArgs == 2); + + nsAutoCString table; + rv = aArgs->GetUTF8String(0, table); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t lastInsertedId = aArgs->AsInt64(1); + + MOZ_ASSERT(table.EqualsLiteral("moz_places") || + table.EqualsLiteral("moz_historyvisits") || + table.EqualsLiteral("moz_bookmarks") || + table.EqualsLiteral("moz_icons")); + + if (table.EqualsLiteral("moz_bookmarks")) { + nsNavBookmarks::StoreLastInsertedId(table, lastInsertedId); + } else if (table.EqualsLiteral("moz_icons")) { + nsFaviconService::StoreLastInsertedId(table, lastInsertedId); + } else { + nsNavHistory::StoreLastInsertedId(table, lastInsertedId); + } + + RefPtr result = new nsVariant(); + rv = result->SetAsInt64(lastInsertedId); + NS_ENSURE_SUCCESS(rv, rv); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Get Query Param Function + +/* static */ +nsresult GetQueryParamFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new GetQueryParamFunction(); + return aDBConn->CreateFunction("get_query_param"_ns, 2, function); +} + +NS_IMPL_ISUPPORTS(GetQueryParamFunction, mozIStorageFunction) + +NS_IMETHODIMP +GetQueryParamFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + nsDependentCString queryString = getSharedUTF8String(aArguments, 0); + nsDependentCString paramName = getSharedUTF8String(aArguments, 1); + + RefPtr result = new nsVariant(); + if (!queryString.IsEmpty() && !paramName.IsEmpty()) { + URLParams::Parse( + queryString, + [¶mName, &result](const nsAString& aName, const nsAString& aValue) { + NS_ConvertUTF16toUTF8 name(aName); + if (!paramName.Equals(name)) { + return true; + } + result->SetAsAString(aValue); + return false; + }); + } + + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Hash Function + +/* static */ +nsresult HashFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new HashFunction(); + return aDBConn->CreateFunction("hash"_ns, -1, function); +} + +NS_IMPL_ISUPPORTS(HashFunction, mozIStorageFunction) + +NS_IMETHODIMP +HashFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + // Fetch arguments. Use default values if they were omitted. + uint32_t numEntries; + nsresult rv = aArguments->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(numEntries >= 1 && numEntries <= 2, NS_ERROR_FAILURE); + + nsDependentCString str = getSharedUTF8String(aArguments, 0); + nsAutoCString mode; + if (numEntries > 1) { + aArguments->GetUTF8String(1, mode); + } + + RefPtr result = new nsVariant(); + uint64_t hash; + rv = mozilla::places::HashURL(str, mode, &hash); + NS_ENSURE_SUCCESS(rv, rv); + rv = result->SetAsInt64((int64_t)hash); + NS_ENSURE_SUCCESS(rv, rv); + + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// MD5 Function + +/* static */ +nsresult MD5HexFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new MD5HexFunction(); + return aDBConn->CreateFunction("md5hex"_ns, -1, function); +} + +NS_IMPL_ISUPPORTS(MD5HexFunction, mozIStorageFunction) + +NS_IMETHODIMP +MD5HexFunction::OnFunctionCall(mozIStorageValueArray* aArguments, + nsIVariant** _result) { + // Must have non-null function arguments. + MOZ_ASSERT(aArguments); + + // Fetch arguments. + uint32_t numEntries; + nsresult rv = aArguments->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(numEntries == 1, NS_ERROR_FAILURE); + nsDependentCString str = getSharedUTF8String(aArguments, 0); + + nsCOMPtr hasher = + do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + // MD5 is not a secure hash function, but it's ok for this use. + rv = hasher->Init(nsICryptoHash::MD5); + NS_ENSURE_SUCCESS(rv, rv); + + rv = hasher->Update(reinterpret_cast(str.BeginReading()), + str.Length()); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString binaryHash, hashString; + rv = hasher->Finish(false, binaryHash); + NS_ENSURE_SUCCESS(rv, rv); + + // Convert to HEX. + static const char* const hex = "0123456789abcdef"; + hashString.SetCapacity(2 * binaryHash.Length()); + for (size_t i = 0; i < binaryHash.Length(); ++i) { + auto c = static_cast(binaryHash[i]); + hashString.Append(hex[(c >> 4) & 0x0F]); + hashString.Append(hex[c & 0x0F]); + } + + RefPtr result = new nsVariant(); + result->SetAsACString(hashString); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Get prefix function + +/* static */ +nsresult GetPrefixFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new GetPrefixFunction(); + nsresult rv = aDBConn->CreateFunction("get_prefix"_ns, 1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(GetPrefixFunction, mozIStorageFunction) + +NS_IMETHODIMP +GetPrefixFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + MOZ_ASSERT(aArgs); + + uint32_t numArgs; + nsresult rv = aArgs->GetNumEntries(&numArgs); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numArgs == 1); + + nsDependentCString spec(getSharedUTF8String(aArgs, 0)); + + RefPtr result = new nsVariant(); + result->SetAsACString(Substring(spec, 0, getPrefixLength(spec))); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Get host and port function + +/* static */ +nsresult GetHostAndPortFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new GetHostAndPortFunction(); + nsresult rv = aDBConn->CreateFunction("get_host_and_port"_ns, 1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(GetHostAndPortFunction, mozIStorageFunction) + +NS_IMETHODIMP +GetHostAndPortFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + MOZ_ASSERT(aArgs); + + uint32_t numArgs; + nsresult rv = aArgs->GetNumEntries(&numArgs); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numArgs == 1); + + nsDependentCString spec(getSharedUTF8String(aArgs, 0)); + + RefPtr result = new nsVariant(); + + size_type length; + size_type index = indexOfHostAndPort(spec, &length); + result->SetAsACString(Substring(spec, index, length)); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Strip prefix and userinfo function + +/* static */ +nsresult StripPrefixAndUserinfoFunction::create( + mozIStorageConnection* aDBConn) { + RefPtr function = + new StripPrefixAndUserinfoFunction(); + nsresult rv = + aDBConn->CreateFunction("strip_prefix_and_userinfo"_ns, 1, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(StripPrefixAndUserinfoFunction, mozIStorageFunction) + +NS_IMETHODIMP +StripPrefixAndUserinfoFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + MOZ_ASSERT(aArgs); + + uint32_t numArgs; + nsresult rv = aArgs->GetNumEntries(&numArgs); + NS_ENSURE_SUCCESS(rv, rv); + MOZ_ASSERT(numArgs == 1); + + nsDependentCString spec(getSharedUTF8String(aArgs, 0)); + + RefPtr result = new nsVariant(); + + size_type index = indexOfHostAndPort(spec, nullptr); + result->SetAsACString(Substring(spec, index, spec.Length() - index)); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Is frecency decaying function + +/* static */ +nsresult IsFrecencyDecayingFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = + new IsFrecencyDecayingFunction(); + nsresult rv = aDBConn->CreateFunction("is_frecency_decaying"_ns, 0, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(IsFrecencyDecayingFunction, mozIStorageFunction) + +NS_IMETHODIMP +IsFrecencyDecayingFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + MOZ_ASSERT(aArgs); + +#ifdef DEBUG + uint32_t numArgs; + MOZ_ASSERT(NS_SUCCEEDED(aArgs->GetNumEntries(&numArgs)) && numArgs == 0); +#endif + + RefPtr result = new nsVariant(); + nsresult rv = result->SetAsBool(nsNavHistory::sIsFrecencyDecaying); + NS_ENSURE_SUCCESS(rv, rv); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Should start frecency recalculation function + +/* static */ +nsresult SetShouldStartFrecencyRecalculationFunction::create( + mozIStorageConnection* aDBConn) { + RefPtr function = + new SetShouldStartFrecencyRecalculationFunction(); + nsresult rv = aDBConn->CreateFunction( + "set_should_start_frecency_recalculation"_ns, 0, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(SetShouldStartFrecencyRecalculationFunction, + mozIStorageFunction) + +NS_IMETHODIMP +SetShouldStartFrecencyRecalculationFunction::OnFunctionCall( + mozIStorageValueArray* aArgs, nsIVariant** _result) { + MOZ_ASSERT(aArgs); + +#ifdef DEBUG + uint32_t numArgs; + MOZ_ASSERT(NS_SUCCEEDED(aArgs->GetNumEntries(&numArgs)) && numArgs == 0); +#endif + + // When changing from false to true, dispatch a runnable to the main-thread + // to start a recalculation. Once there's nothing left to recalculathe this + // boolean will be set back to false. Note this means there will be a short + // interval between completing a recalculation and setting this back to false + // where we could potentially lose a recalculation request. That should not be + // a big deal, since the recalculation will just happen at the next operation + // changing frecency or, in the worst case, at the next session. + if (!nsNavHistory::sShouldStartFrecencyRecalculation.exchange(true)) { + mozilla::Unused << NS_DispatchToMainThread(NS_NewRunnableFunction( + "SetShouldStartFrecencyRecalculationFunction::Notify", [] { + nsCOMPtr os = services::GetObserverService(); + if (os) { + mozilla::Unused << os->NotifyObservers( + nullptr, "frecency-recalculation-needed", nullptr); + } + })); + } + + RefPtr result = new nsVariant(); + nsresult rv = result->SetAsBool(true); + NS_ENSURE_SUCCESS(rv, rv); + result.forget(_result); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Note Sync Change Function + +/* static */ +nsresult NoteSyncChangeFunction::create(mozIStorageConnection* aDBConn) { + RefPtr function = new NoteSyncChangeFunction(); + nsresult rv = aDBConn->CreateFunction("note_sync_change"_ns, 0, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(NoteSyncChangeFunction, mozIStorageFunction) + +NS_IMETHODIMP +NoteSyncChangeFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + nsNavBookmarks::NoteSyncChange(); + *_result = nullptr; + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// Invalidate days of history Function + +/* static */ +nsresult InvalidateDaysOfHistoryFunction::create( + mozIStorageConnection* aDBConn) { + RefPtr function = + new InvalidateDaysOfHistoryFunction(); + nsresult rv = + aDBConn->CreateFunction("invalidate_days_of_history"_ns, 0, function); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(InvalidateDaysOfHistoryFunction, mozIStorageFunction) + +NS_IMETHODIMP +InvalidateDaysOfHistoryFunction::OnFunctionCall(mozIStorageValueArray* aArgs, + nsIVariant** _result) { + nsNavHistory::InvalidateDaysOfHistory(); + return NS_OK; +} + +} // namespace mozilla::places diff --git a/toolkit/components/places/SQLFunctions.h b/toolkit/components/places/SQLFunctions.h new file mode 100644 index 0000000000..ce99885378 --- /dev/null +++ b/toolkit/components/places/SQLFunctions.h @@ -0,0 +1,663 @@ +/* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_SQLFunctions_h_ +#define mozilla_places_SQLFunctions_h_ + +/** + * This file contains functions that Places adds to the database handle that can + * be accessed by SQL queries. + * + * Keep the GUID-related parts of this file in sync with + * toolkit/downloads/SQLFunctions.[h|cpp]! + */ + +#include "mozIStorageFunction.h" +#include "mozilla/Attributes.h" + +class mozIStorageConnection; + +namespace mozilla { +namespace places { + +//////////////////////////////////////////////////////////////////////////////// +//// AutoComplete Matching Function + +/** + * This function is used to determine if a given set of data should match an + * AutoComplete query. + * + * In SQL, you'd use it in the WHERE clause like so: + * WHERE AUTOCOMPLETE_MATCH(aSearchString, aURL, aTitle, aTags, aVisitCount, + * aTyped, aBookmark, aOpenPageCount, aMatchBehavior, + * aSearchBehavior, aFallbackTitle) + * + * @param aSearchString + * The string to compare against. + * @param aURL + * The URL to test for an AutoComplete match. + * @param aTitle + * The title to test for an AutoComplete match. + * @param aTags + * The tags to test for an AutoComplete match. + * @param aVisitCount + * The number of visits aURL has. + * @param aTyped + * Indicates if aURL is a typed URL or not. Treated as a boolean. + * @param aBookmark + * Indicates if aURL is a bookmark or not. Treated as a boolean. + * @param aOpenPageCount + * The number of times aURL has been registered as being open. (See + * UrlbarProviderOpenTabs::registerOpenTab.) + * @param aMatchBehavior + * The match behavior to use for this search. + * @param aSearchBehavior + * A bitfield dictating the search behavior. + * @param aFallbackTitle + * The title may come from a bookmark or a snapshot, in that case the + * caller can provide the original history title to match on both. + */ +class MatchAutoCompleteFunction final : public mozIStorageFunction { + public: + MatchAutoCompleteFunction(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~MatchAutoCompleteFunction() = default; + + /** + * IntegerVariants for 0 and 1 are frequently used in awesomebar queries, + * so we cache them to avoid allocating memory repeatedly. + */ + nsCOMPtr mCachedZero; + nsCOMPtr mCachedOne; + + /** + * Argument Indexes + */ + static const uint32_t kArgSearchString = 0; + static const uint32_t kArgIndexURL = 1; + static const uint32_t kArgIndexTitle = 2; + static const uint32_t kArgIndexTags = 3; + static const uint32_t kArgIndexVisitCount = 4; + static const uint32_t kArgIndexTyped = 5; + static const uint32_t kArgIndexBookmark = 6; + static const uint32_t kArgIndexOpenPageCount = 7; + static const uint32_t kArgIndexMatchBehavior = 8; + static const uint32_t kArgIndexSearchBehavior = 9; + static const uint32_t kArgIndexFallbackTitle = 10; + static const uint32_t kArgIndexLength = 11; + + /** + * Typedefs + */ + typedef bool (*searchFunctionPtr)(const nsDependentCSubstring& aToken, + const nsACString& aSourceString); + + typedef nsACString::const_char_iterator const_char_iterator; + + /** + * Obtains the search function to match on. + * + * @param aBehavior + * The matching behavior to use defined by one of the + * mozIPlacesAutoComplete::MATCH_* values. + * @return a pointer to the function that will perform the proper search. + */ + static searchFunctionPtr getSearchFunction(int32_t aBehavior); + + /** + * Tests if aSourceString starts with aToken. + * + * @param aToken + * The string to search for. + * @param aSourceString + * The string to search. + * @return true if found, false otherwise. + */ + static bool findBeginning(const nsDependentCSubstring& aToken, + const nsACString& aSourceString); + + /** + * Tests if aSourceString starts with aToken in a case sensitive way. + * + * @param aToken + * The string to search for. + * @param aSourceString + * The string to search. + * @return true if found, false otherwise. + */ + static bool findBeginningCaseSensitive(const nsDependentCSubstring& aToken, + const nsACString& aSourceString); + + /** + * Searches aSourceString for aToken anywhere in the string in a case- + * insensitive way. + * + * @param aToken + * The string to search for. + * @param aSourceString + * The string to search. + * @return true if found, false otherwise. + */ + static bool findAnywhere(const nsDependentCSubstring& aToken, + const nsACString& aSourceString); + + /** + * Tests if aToken is found on a word boundary in aSourceString. + * + * @param aToken + * The string to search for. + * @param aSourceString + * The string to search. + * @return true if found, false otherwise. + */ + static bool findOnBoundary(const nsDependentCSubstring& aToken, + const nsACString& aSourceString); + + /** + * Fixes a URI's spec such that it is ready to be searched. This includes + * unescaping escaped characters and removing certain specs that we do not + * care to search for. + * + * @param aURISpec + * The spec of the URI to prepare for searching. + * @param aMatchBehavior + * The matching behavior to use defined by one of the + * mozIPlacesAutoComplete::MATCH_* values. + * @param aSpecBuf + * A string buffer that the returned slice can point into, if needed. + * @return the fixed up string. + */ + static nsDependentCSubstring fixupURISpec(const nsACString& aURISpec, + int32_t aMatchBehavior, + nsACString& aSpecBuf); +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Frecency Calculation Function + +/** + * This function is used to calculate frecency for a page. + * + * In SQL, you'd use it in when setting frecency like: + * SET frecency = CALCULATE_FRECENCY(place_id). + * Optional parameters must be passed in if the page is not yet in the database, + * otherwise they will be fetched from it automatically. + * + * @param {int64_t} pageId + * The id of the page. Pass -1 if the page is being added right now. + * @param [optional] {int32_t} useRedirectBonus + * Whether we should use the lower redirect bonus for the most recent + * page visit. If not passed in, it will use a database guess. + */ +class CalculateFrecencyFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~CalculateFrecencyFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Alternative Frecency Calculation Function + +/** + * This function is used to calculate alternative frecency for a page. + * + * In SQL, you'd use it in when setting frecency like: + * SET alt_frecency = CALCULATE_ALT_FRECENCY(place_id). + * Optional parameters must be passed in if the page is not yet in the database, + * otherwise they will be fetched from it automatically. + * + * @param {int64_t} pageId + * The id of the page. Pass -1 if the page is being added right now. + * @param {int32_t} [useRedirectBonus] + * Whether we should use the lower redirect bonus for the most recent + * page visit. If not passed in, it will use a database guess. + */ +class CalculateAltFrecencyFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~CalculateAltFrecencyFunction() = default; +}; + +/** + * SQL function to generate a GUID for a place or bookmark item. This is just + * a wrapper around GenerateGUID in Helpers.h. + * + * @return a guid for the item. + */ +class GenerateGUIDFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~GenerateGUIDFunction() = default; +}; + +/** + * SQL function to check if a GUID is valid. This is just a wrapper around + * IsValidGUID in Helpers.h. + * + * @return true if valid, false otherwise. + */ +class IsValidGUIDFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~IsValidGUIDFunction() = default; +}; + +/** + * SQL function to unreverse the rev_host of a page. + * + * @param rev_host + * The rev_host value of the page. + * + * @return the unreversed host of the page. + */ +class GetUnreversedHostFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~GetUnreversedHostFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Fixup URL Function + +/** + * Make a given URL more suitable for searches, by removing common prefixes + * such as "www." + * + * @param url + * A URL. + * @return + * The same URL, with redundant parts removed. + */ +class FixupURLFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~FixupURLFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Store Last Inserted Id Function + +/** + * Store the last inserted id for reference purpose. + * + * @param tableName + * The table name. + * @param id + * The last inserted id. + * @return null + */ +class StoreLastInsertedIdFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~StoreLastInsertedIdFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Hash Function + +/** + * Calculates hash for a given string using the mfbt AddToHash function. + * + * @param string + * A string. + * @return + * The hash for the string. + */ +class HashFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~HashFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// MD5 Function + +/** + * Calculates md5 hash for a given string. + * + * @param string + * A string. + * @return + * The hash for the string. + */ +class MD5HexFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~MD5HexFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Get Query Param Function + +/** + * Extracts and returns the value of a parameter from a query string. + * + * @param string + * A string. + * @return + * The value of the query parameter as a string, or NULL if not set. + */ +class GetQueryParamFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~GetQueryParamFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Get prefix function + +/** + * Gets the length of the prefix in a URL. "Prefix" is defined to be the + * scheme, colon, and, if present, two slashes. + * + * @param url + * A URL, or a string that may be a URL. + * @return + * If `url` is actually a URL and has a prefix, then this returns the + * prefix. Otherwise this returns an empty string. + */ +class GetPrefixFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~GetPrefixFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Get host and port function + +/** + * Gets the host and port substring of a URL. + * + * @param url + * A URL, or a string that may be a URL. + * @return + * If `url` is actually a URL, or if it's a URL without the prefix, then + * this returns the host and port substring of the URL. Otherwise, this + * returns `url` itself. + */ +class GetHostAndPortFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~GetHostAndPortFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Strip prefix function + +/** + * Gets the part of a URL after its prefix and userinfo; i.e., the substring + * starting at the host. + * + * @param url + * A URL, or a string that may be a URL. + * @return + * If `url` is actually a URL, or if it's a URL without the prefix, then + * this returns the substring starting at the host. Otherwise, this + * returns `url` itself. + */ +class StripPrefixAndUserinfoFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~StripPrefixAndUserinfoFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Is frecency decaying function + +/** + * Returns nsNavHistory::IsFrecencyDecaying(). + * + * @return + * True if frecency is currently decaying and false otherwise. + */ +class IsFrecencyDecayingFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~IsFrecencyDecayingFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Should start frecency recalculation function + +/** + * sets nsNavHistory::sShouldStartFrecencyRecalculation to true. + * @returns {boolean} true + */ +class SetShouldStartFrecencyRecalculationFunction final + : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~SetShouldStartFrecencyRecalculationFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Note Sync Change Function + +/** + * Bumps the global Sync change counter. See the comment above + * `totalSyncChanges` in `nsINavBookmarksService` for a more detailed + * explanation. + */ +class NoteSyncChangeFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~NoteSyncChangeFunction() = default; +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Invalidate days of history Function + +/** + * Invalidate the days of history in nsNavHistory. + */ +class InvalidateDaysOfHistoryFunction final : public mozIStorageFunction { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + /** + * Registers the function with the specified database connection. + * + * @param aDBConn + * The database connection to register with. + */ + static nsresult create(mozIStorageConnection* aDBConn); + + private: + ~InvalidateDaysOfHistoryFunction() = default; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_SQLFunctions_h_ diff --git a/toolkit/components/places/Shutdown.cpp b/toolkit/components/places/Shutdown.cpp new file mode 100644 index 0000000000..8327784b86 --- /dev/null +++ b/toolkit/components/places/Shutdown.cpp @@ -0,0 +1,217 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Shutdown.h" +#include "mozilla/Unused.h" +#include "mozilla/Services.h" +#include "mozilla/SimpleEnumerator.h" +#include "nsComponentManagerUtils.h" +#include "nsIProperty.h" +#include "nsIObserverService.h" +#include "nsIWritablePropertyBag.h" +#include "nsVariant.h" +#include "Database.h" + +namespace mozilla { +namespace places { + +uint16_t PlacesShutdownBlocker::sCounter = 0; +Atomic PlacesShutdownBlocker::sIsStarted(false); + +PlacesShutdownBlocker::PlacesShutdownBlocker(const nsString& aName) + : mName(aName), mState(NOT_STARTED), mCounter(sCounter++) { + MOZ_ASSERT(NS_IsMainThread()); + // During tests, we can end up with the Database singleton being resurrected. + // Make sure that each instance of DatabaseShutdown has a unique name. + if (mCounter > 1) { + mName.AppendInt(mCounter); + } + // Create a barrier that will be exposed to clients through GetClient(), so + // they can block Places shutdown. + nsCOMPtr asyncShutdown = + services::GetAsyncShutdownService(); + MOZ_ASSERT(asyncShutdown); + if (asyncShutdown) { + nsCOMPtr barrier; + nsresult rv = asyncShutdown->MakeBarrier(mName, getter_AddRefs(barrier)); + MOZ_ALWAYS_SUCCEEDS(rv); + if (NS_SUCCEEDED(rv) && barrier) { + mBarrier = new nsMainThreadPtrHolder( + "PlacesShutdownBlocker::mBarrier", barrier); + } + } +} + +// nsIAsyncShutdownBlocker +NS_IMETHODIMP +PlacesShutdownBlocker::GetName(nsAString& aName) { + aName = mName; + return NS_OK; +} + +// nsIAsyncShutdownBlocker +NS_IMETHODIMP +PlacesShutdownBlocker::GetState(nsIPropertyBag** _state) { + NS_ENSURE_ARG_POINTER(_state); + + nsCOMPtr bag = + do_CreateInstance("@mozilla.org/hash-property-bag;1"); + NS_ENSURE_TRUE(bag, NS_ERROR_OUT_OF_MEMORY); + + RefPtr progress = new nsVariant(); + Unused << NS_WARN_IF(NS_FAILED(progress->SetAsUint8(mState))); + Unused << NS_WARN_IF( + NS_FAILED(bag->SetProperty(u"PlacesShutdownProgress"_ns, progress))); + + if (mBarrier) { + nsCOMPtr barrierState; + if (NS_SUCCEEDED(mBarrier->GetState(getter_AddRefs(barrierState))) && + barrierState) { + nsCOMPtr enumerator; + if (NS_SUCCEEDED( + barrierState->GetEnumerator(getter_AddRefs(enumerator))) && + enumerator) { + for (const auto& property : SimpleEnumerator(enumerator)) { + nsAutoString prefix(u"Barrier: "_ns); + nsAutoString name; + Unused << NS_WARN_IF(NS_FAILED(property->GetName(name))); + prefix.Append(name); + nsCOMPtr value; + Unused << NS_WARN_IF( + NS_FAILED(property->GetValue(getter_AddRefs(value)))); + Unused << NS_WARN_IF(NS_FAILED(bag->SetProperty(prefix, value))); + } + } + } + } + bag.forget(_state); + return NS_OK; +} + +already_AddRefed PlacesShutdownBlocker::GetClient() { + nsCOMPtr client; + if (mBarrier) { + MOZ_ALWAYS_SUCCEEDS(mBarrier->GetClient(getter_AddRefs(client))); + } + return client.forget(); +} + +// nsIAsyncShutdownBlocker +NS_IMETHODIMP +PlacesShutdownBlocker::BlockShutdown(nsIAsyncShutdownClient* aParentClient) { + MOZ_ASSERT(NS_IsMainThread()); + mParentClient = new nsMainThreadPtrHolder( + "ClientsShutdownBlocker::mParentClient", aParentClient); + mState = RECEIVED_BLOCK_SHUTDOWN; + + if (NS_WARN_IF(!mBarrier)) { + return NS_ERROR_NOT_AVAILABLE; + } + + // Wait until all the clients have removed their blockers. + MOZ_ALWAYS_SUCCEEDS(mBarrier->Wait(this)); + + mState = CALLED_WAIT_CLIENTS; + return NS_OK; +} + +// nsIAsyncShutdownCompletionCallback +NS_IMETHODIMP +PlacesShutdownBlocker::Done() { + MOZ_ASSERT(false, "Should always be overridden"); + return NS_OK; +} + +NS_IMPL_ISUPPORTS(PlacesShutdownBlocker, nsIAsyncShutdownBlocker, + nsIAsyncShutdownCompletionCallback) + +//////////////////////////////////////////////////////////////////////////////// + +ClientsShutdownBlocker::ClientsShutdownBlocker() + : PlacesShutdownBlocker(u"Places Clients shutdown"_ns) { + // Do nothing. +} + +// nsIAsyncShutdownCompletionCallback +NS_IMETHODIMP +ClientsShutdownBlocker::Done() { + // At this point all the clients are done, we can stop blocking the shutdown + // phase. + mState = RECEIVED_DONE; + + // mParentClient is nullptr in tests. + if (mParentClient) { + nsresult rv = mParentClient->RemoveBlocker(this); + if (NS_WARN_IF(NS_FAILED(rv))) return rv; + mParentClient = nullptr; + } + mBarrier = nullptr; + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// + +ConnectionShutdownBlocker::ConnectionShutdownBlocker(Database* aDatabase) + : PlacesShutdownBlocker(u"Places Connection shutdown"_ns), + mDatabase(aDatabase) { + // Do nothing. +} + +// nsIAsyncShutdownCompletionCallback +NS_IMETHODIMP +ConnectionShutdownBlocker::Done() { + // At this point all the clients are done, we can stop blocking the shutdown + // phase. + mState = RECEIVED_DONE; + + // Annotate that Database shutdown started. + sIsStarted = true; + + // At this stage, any use of this database is forbidden. Get rid of + // `gDatabase`. Note, however, that the database could be + // resurrected. This can happen in particular during tests. + MOZ_ASSERT(Database::gDatabase == nullptr || + Database::gDatabase == mDatabase); + Database::gDatabase = nullptr; + + // Database::Shutdown will invoke Complete once the connection is closed. + mDatabase->Shutdown(); + mState = CALLED_STORAGESHUTDOWN; + mBarrier = nullptr; + return NS_OK; +} + +// mozIStorageCompletionCallback +NS_IMETHODIMP +ConnectionShutdownBlocker::Complete(nsresult, nsISupports*) { + MOZ_ASSERT(NS_IsMainThread()); + mState = RECEIVED_STORAGESHUTDOWN_COMPLETE; + + // The connection is closed, the Database has no more use, so we can break + // possible cycles. + mDatabase = nullptr; + + // Notify the connection has gone. + nsCOMPtr os = mozilla::services::GetObserverService(); + MOZ_ASSERT(os); + if (os) { + MOZ_ALWAYS_SUCCEEDS( + os->NotifyObservers(nullptr, TOPIC_PLACES_CONNECTION_CLOSED, nullptr)); + } + mState = NOTIFIED_OBSERVERS_PLACES_CONNECTION_CLOSED; + + // mParentClient is nullptr in tests + if (mParentClient) { + nsresult rv = mParentClient->RemoveBlocker(this); + if (NS_WARN_IF(NS_FAILED(rv))) return rv; + mParentClient = nullptr; + } + return NS_OK; +} + +NS_IMPL_ISUPPORTS_INHERITED(ConnectionShutdownBlocker, PlacesShutdownBlocker, + mozIStorageCompletionCallback) + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/Shutdown.h b/toolkit/components/places/Shutdown.h new file mode 100644 index 0000000000..3e5612a454 --- /dev/null +++ b/toolkit/components/places/Shutdown.h @@ -0,0 +1,155 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_Shutdown_h_ +#define mozilla_places_Shutdown_h_ + +#include "mozIStorageCompletionCallback.h" +#include "nsIAsyncShutdown.h" +#include "nsProxyRelease.h" + +namespace mozilla { +namespace places { + +class Database; + +/** + * This is most of the code responsible for Places shutdown. + * + * PHASE 1 (Legacy clients shutdown) + * The shutdown procedure begins when the Database singleton receives + * profile-change-teardown (note that tests will instead notify nsNavHistory, + * that forwards the notification to the Database instance). + * Database::Observe first of all checks if initialization was completed + * properly, to avoid race conditions, then it notifies "places-shutdown" to + * legacy clients. Legacy clients are supposed to start and complete any + * shutdown critical work in the same tick, since we won't wait for them. + + * PHASE 2 (Modern clients shutdown) + * Modern clients should instead register as a blocker by passing a promise to + * nsINavHistoryService::shutdownClient (for example see Sanitizer.jsm), so they + * block Places shutdown until the promise is resolved. + * When profile-change-teardown is observed by async shutdown, it calls + * ClientsShutdownBlocker::BlockShutdown. This class is registered as a teardown + * phase blocker in Database::Init (see Database::mClientsShutdown). + * ClientsShutdownBlocker::BlockShudown waits for all the clients registered + * through nsINavHistoryService::shutdownClient. When all the clients are done, + * its `Done` method is invoked, and it stops blocking the shutdown phase, so + * that it can continue. + * + * PHASE 3 (Connection shutdown) + * ConnectionBlocker is registered as a profile-before-change blocker in + * Database::Init (see Database::mConnectionShutdown). + * When profile-before-change is observer by async shutdown, it calls + * ConnectionShutdownBlocker::BlockShutdown. + * Then the control is passed to Database::Shutdown, that executes some sanity + * checks, clears cached statements and proceeds with asyncClose. + * Once the connection is definitely closed, Database will call back + * ConnectionBlocker::Complete. At this point a final + * places-connection-closed notification is sent, for testing purposes. + */ + +/** + * A base AsyncShutdown blocker in charge of shutting down Places. + */ +class PlacesShutdownBlocker : public nsIAsyncShutdownBlocker, + public nsIAsyncShutdownCompletionCallback { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIASYNCSHUTDOWNBLOCKER + NS_DECL_NSIASYNCSHUTDOWNCOMPLETIONCALLBACK + + explicit PlacesShutdownBlocker(const nsString& aName); + + already_AddRefed GetClient(); + + // The current state, used internally and for forensics/debugging purposes. + // Not all the states make sense for all the derived classes. + enum States { + NOT_STARTED, + // Execution of `BlockShutdown` in progress. + RECEIVED_BLOCK_SHUTDOWN, + + // Values specific to ClientsShutdownBlocker + // a. Set while we are waiting for clients to do their job and unblock us. + CALLED_WAIT_CLIENTS, + // b. Set when all the clients are done. + RECEIVED_DONE, + + // Values specific to ConnectionShutdownBlocker + // a. Set after we notified observers that Places is closing the connection. + NOTIFIED_OBSERVERS_PLACES_WILL_CLOSE_CONNECTION, + // b. Set after we pass control to Database::Shutdown, and wait for it to + // close the connection and call our `Complete` method when done. + CALLED_STORAGESHUTDOWN, + // c. Set when Database has closed the connection and passed control to + // us through `Complete`. + RECEIVED_STORAGESHUTDOWN_COMPLETE, + // d. We have notified observers that Places has closed the connection. + NOTIFIED_OBSERVERS_PLACES_CONNECTION_CLOSED, + }; + States State() { return mState; } + + static Atomic sIsStarted; + + protected: + // The blocker name, also used as barrier name. + nsString mName; + // The current state, see States. + States mState; + // The barrier optionally used to wait for clients. + nsMainThreadPtrHandle mBarrier; + // The parent object who registered this as a blocker. + nsMainThreadPtrHandle mParentClient; + + // As tests may resurrect a dead `Database`, we use a counter to + // give the instances of `PlacesShutdownBlocker` unique names. + uint16_t mCounter; + static uint16_t sCounter; + + virtual ~PlacesShutdownBlocker() = default; +}; + +/** + * Blocker also used to wait for clients, through an owned barrier. + */ +class ClientsShutdownBlocker final : public PlacesShutdownBlocker { + public: + NS_INLINE_DECL_REFCOUNTING_INHERITED(ClientsShutdownBlocker, + PlacesShutdownBlocker) + + explicit ClientsShutdownBlocker(); + + NS_IMETHOD Done() override; + + private: + ~ClientsShutdownBlocker() = default; +}; + +/** + * Blocker used to wait when closing the database connection. + */ +class ConnectionShutdownBlocker final : public PlacesShutdownBlocker, + public mozIStorageCompletionCallback { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_MOZISTORAGECOMPLETIONCALLBACK + + explicit ConnectionShutdownBlocker(mozilla::places::Database* aDatabase); + + NS_IMETHOD Done() override; + + private: + ~ConnectionShutdownBlocker() = default; + + // The owning database. + // The cycle is broken in method Complete(), once the connection + // has been closed by mozStorage. + RefPtr mDatabase; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_Shutdown_h_ diff --git a/toolkit/components/places/SyncedBookmarksMirror.h b/toolkit/components/places/SyncedBookmarksMirror.h new file mode 100644 index 0000000000..4839896841 --- /dev/null +++ b/toolkit/components/places/SyncedBookmarksMirror.h @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_SyncedBookmarksMirror_h_ +#define mozilla_places_SyncedBookmarksMirror_h_ + +#include "mozISyncedBookmarksMirror.h" +#include "nsCOMPtr.h" + +extern "C" { + +// Implemented in Rust, in the `bookmark_sync` crate. +void NS_NewSyncedBookmarksMerger(mozISyncedBookmarksMerger** aResult); + +} // extern "C" + +namespace mozilla { +namespace places { + +already_AddRefed NewSyncedBookmarksMerger() { + nsCOMPtr merger; + NS_NewSyncedBookmarksMerger(getter_AddRefs(merger)); + return merger.forget(); +} + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_SyncedBookmarksMirror_h_ diff --git a/toolkit/components/places/SyncedBookmarksMirror.sys.mjs b/toolkit/components/places/SyncedBookmarksMirror.sys.mjs new file mode 100644 index 0000000000..2e197d2964 --- /dev/null +++ b/toolkit/components/places/SyncedBookmarksMirror.sys.mjs @@ -0,0 +1,2473 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file implements a mirror and two-way merger for synced bookmarks. The + * mirror matches the complete tree stored on the Sync server, and stages new + * bookmarks changed on the server since the last sync. The merger walks the + * local tree in Places and the mirrored remote tree, produces a new merged + * tree, then updates the local tree to reflect the merged tree. + * + * Let's start with an overview of the different classes, and how they fit + * together. + * + * - `SyncedBookmarksMirror` sets up the database, validates and upserts new + * incoming records, attaches to Places, and applies the changed records. + * During application, we fetch the local and remote bookmark trees, merge + * them, and update Places to match. Merging and application happen in a + * single transaction, so applying the merged tree won't collide with local + * changes. A failure at this point aborts the merge and leaves Places + * unchanged. + * + * - A `BookmarkTree` is a fully rooted tree that also notes deletions. A + * `BookmarkNode` represents a local item in Places, or a remote item in the + * mirror. + * + * - A `MergedBookmarkNode` holds a local node, a remote node, and a + * `MergeState` that indicates which node to prefer when updating Places and + * the server to match the merged tree. + * + * - `BookmarkObserverRecorder` records all changes made to Places during the + * merge, then dispatches `PlacesObservers` notifications. Places uses + * these notifications to update the UI and internal caches. We can't dispatch + * during the merge because observers won't see the changes until the merge + * transaction commits and the database is consistent again. + * + * - After application, we flag all applied incoming items as merged, create + * Sync records for the locally new and updated items in Places, and upload + * the records to the server. At this point, all outgoing items are flagged as + * changed in Places, so the next sync can resume cleanly if the upload is + * interrupted or fails. + * + * - Once upload succeeds, we update the mirror with the uploaded records, so + * that the mirror matches the server again. An interruption or error here + * will leave the uploaded items flagged as changed in Places, so we'll merge + * them again on the next sync. This is redundant work, but shouldn't cause + * issues. + */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Async: "resource://services-common/async.sys.mjs", + Log: "resource://gre/modules/Log.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "MirrorLog", () => + lazy.Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror") +); + +const SyncedBookmarksMerger = Components.Constructor( + "@mozilla.org/browser/synced-bookmarks-merger;1", + "mozISyncedBookmarksMerger" +); + +// These can be removed once they're exposed in a central location (bug +// 1375896). +const DB_URL_LENGTH_MAX = 65536; +const DB_TITLE_LENGTH_MAX = 4096; + +// The current mirror database schema version. Bump for migrations, then add +// migration code to `migrateMirrorSchema`. +const MIRROR_SCHEMA_VERSION = 8; + +// Use a shared jankYielder in these functions +XPCOMUtils.defineLazyGetter(lazy, "yieldState", () => lazy.Async.yieldState()); + +/** Adapts a `Log.sys.mjs` logger to a `mozIServicesLogSink`. */ +class LogAdapter { + constructor(log) { + this.log = log; + } + + get maxLevel() { + let level = this.log.level; + if (level <= lazy.Log.Level.All) { + return Ci.mozIServicesLogSink.LEVEL_TRACE; + } + if (level <= lazy.Log.Level.Info) { + return Ci.mozIServicesLogSink.LEVEL_DEBUG; + } + if (level <= lazy.Log.Level.Warn) { + return Ci.mozIServicesLogSink.LEVEL_WARN; + } + if (level <= lazy.Log.Level.Error) { + return Ci.mozIServicesLogSink.LEVEL_ERROR; + } + return Ci.mozIServicesLogSink.LEVEL_OFF; + } + + trace(message) { + this.log.trace(message); + } + + debug(message) { + this.log.debug(message); + } + + warn(message) { + this.log.warn(message); + } + + error(message) { + this.log.error(message); + } +} + +/** + * A helper to track the progress of a merge for telemetry and shutdown hang + * reporting. + */ +class ProgressTracker { + constructor(recordStepTelemetry) { + this.recordStepTelemetry = recordStepTelemetry; + this.steps = []; + } + + /** + * Records a merge step, updating the shutdown blocker state. + * + * @param {String} name A step name from `ProgressTracker.STEPS`. This is + * included in shutdown hang crash reports, along with the timestamp + * the step was recorded. + * @param {Number} [took] The time taken, in milliseconds. + * @param {Array} [counts] An array of additional counts to report in the + * shutdown blocker state. + */ + step(name, took = -1, counts = null) { + let info = { step: name, at: Date.now() }; + if (took > -1) { + info.took = took; + } + if (counts) { + info.counts = counts; + } + this.steps.push(info); + } + + /** + * Records a merge step with timings and counts for telemetry. + * + * @param {String} name The step name. + * @param {Number} took The time taken, in milliseconds. + * @param {Array} [counts] An array of additional `{ name, count }` tuples to + * record in telemetry for this step. + */ + stepWithTelemetry(name, took, counts = null) { + this.step(name, took, counts); + this.recordStepTelemetry(name, took, counts); + } + + /** + * Records a merge step with the time taken and item count. + * + * @param {String} name The step name. + * @param {Number} took The time taken, in milliseconds. + * @param {Number} count The number of items handled in this step. + */ + stepWithItemCount(name, took, count) { + this.stepWithTelemetry(name, took, [{ name: "items", count }]); + } + + /** + * Clears all recorded merge steps. + */ + reset() { + this.steps = []; + } + + /** + * Returns the shutdown blocker state. This is included in shutdown hang + * crash reports, in the `AsyncShutdownTimeout` annotation. + * + * @see `fetchState` in `AsyncShutdown` for more details. + * @return {Object} A stringifiable object with the recorded steps. + */ + fetchState() { + return { steps: this.steps }; + } +} + +/** Merge steps for which we record progress. */ +ProgressTracker.STEPS = { + FETCH_LOCAL_TREE: "fetchLocalTree", + FETCH_REMOTE_TREE: "fetchRemoteTree", + MERGE: "merge", + APPLY: "apply", + NOTIFY_OBSERVERS: "notifyObservers", + FETCH_LOCAL_CHANGE_RECORDS: "fetchLocalChangeRecords", + FINALIZE: "finalize", +}; + +/** + * A mirror maintains a copy of the complete tree as stored on the Sync server. + * It is persistent. + * + * The mirror schema is a hybrid of how Sync and Places represent bookmarks. + * The `items` table contains item attributes (title, kind, URL, etc.), while + * the `structure` table stores parent-child relationships and position. + * This is similar to how iOS encodes "value" and "structure" state, + * though we handle these differently when merging. See `BookmarkMerger` for + * details. + * + * There's no guarantee that the remote state is consistent. We might be missing + * parents or children, or a bookmark and its parent might disagree about where + * it belongs. This means we need a strategy to handle missing parents and + * children. + * + * We treat the `children` of the last parent we see as canonical, and ignore + * the child's `parentid` entirely. We also ignore missing children, and + * temporarily reparent bookmarks with missing parents to "unfiled". When we + * eventually see the missing items, either during a later sync or as part of + * repair, we'll fill in the mirror's gaps and fix up the local tree. + * + * During merging, we won't intentionally try to fix inconsistencies on the + * server, and opt to build as complete a tree as we can from the remote state, + * even if we diverge from what's in the mirror. See bug 1433512 for context. + * + * If a sync is interrupted, we resume downloading from the server collection + * last modified time, or the server last modified time of the most recent + * record if newer. New incoming records always replace existing records in the + * mirror. + * + * We delete the mirror database on client reset, including when the sync ID + * changes on the server, and when the user is node reassigned, disables the + * bookmarks engine, or signs out. + */ +export class SyncedBookmarksMirror { + constructor( + db, + wasCorrupt = false, + { + recordStepTelemetry, + recordValidationTelemetry, + finalizeAt = lazy.PlacesUtils.history.shutdownClient.jsclient, + } = {} + ) { + this.db = db; + this.wasCorrupt = wasCorrupt; + this.recordValidationTelemetry = recordValidationTelemetry; + + this.merger = new SyncedBookmarksMerger(); + this.merger.db = db.unsafeRawConnection.QueryInterface( + Ci.mozIStorageConnection + ); + this.merger.logger = new LogAdapter(lazy.MirrorLog); + + // Automatically close the database connection on shutdown. `progress` + // tracks state for shutdown hang reporting. + this.progress = new ProgressTracker(recordStepTelemetry); + this.finalizeController = new AbortController(); + this.finalizeAt = finalizeAt; + this.finalizeBound = () => this.finalize({ alsoCleanup: false }); + this.finalizeAt.addBlocker( + "SyncedBookmarksMirror: finalize", + this.finalizeBound, + { fetchState: () => this.progress } + ); + } + + /** + * Sets up the mirror database connection and upgrades the mirror to the + * newest schema version. Automatically recreates the mirror if it's corrupt; + * throws on failure. + * + * @param {String} options.path + * The path to the mirror database file, either absolute or relative + * to the profile path. + * @param {Function} options.recordStepTelemetry + * A function with the signature `(name: String, took: Number, + * counts: Array?)`, where `name` is the name of the merge step, + * `took` is the time taken in milliseconds, and `counts` is an + * array of named counts (`{ name, count }` tuples) with additional + * counts for the step to record in the telemetry ping. + * @param {Function} options.recordValidationTelemetry + * A function with the signature `(took: Number, checked: Number, + * problems: Array)`, where `took` is the time taken to run + * validation in milliseconds, `checked` is the number of items + * checked, and `problems` is an array of named problem counts. + * @param {AsyncShutdown.Barrier} [options.finalizeAt] + * A shutdown phase, barrier, or barrier client that should + * automatically finalize the mirror when triggered. Exposed for + * testing. + * @return {SyncedBookmarksMirror} + * A mirror ready for use. + */ + static async open(options) { + let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection(); + if (!db) { + throw new TypeError("Can't open mirror without Places connection"); + } + let path; + if (PathUtils.isAbsolute(options.path)) { + path = options.path; + } else { + path = PathUtils.join(PathUtils.profileDir, options.path); + } + let wasCorrupt = false; + try { + await attachAndInitMirrorDatabase(db, path); + } catch (ex) { + if (isDatabaseCorrupt(ex)) { + lazy.MirrorLog.warn( + "Error attaching mirror to Places; removing and " + + "recreating mirror", + ex + ); + wasCorrupt = true; + await IOUtils.remove(path); + await attachAndInitMirrorDatabase(db, path); + } else { + lazy.MirrorLog.error( + "Unrecoverable error attaching mirror to Places", + ex + ); + throw ex; + } + } + return new SyncedBookmarksMirror(db, wasCorrupt, options); + } + + /** + * Returns the newer of the bookmarks collection last modified time, or the + * server modified time of the newest record. The bookmarks engine uses this + * timestamp as the "high water mark" for all downloaded records. Each sync + * downloads and stores records that are strictly newer than this time. + * + * @return {Number} + * The high water mark time, in seconds. + */ + async getCollectionHighWaterMark() { + // The first case, where we have records with server modified times newer + // than the collection last modified time, occurs when a sync is interrupted + // before we call `setCollectionLastModified`. We subtract one second, the + // maximum time precision guaranteed by the server, so that we don't miss + // other records with the same time as the newest one we downloaded. + let rows = await this.db.executeCached( + ` + SELECT MAX( + IFNULL((SELECT MAX(serverModified) - 1000 FROM items), 0), + IFNULL((SELECT CAST(value AS INTEGER) FROM meta + WHERE key = :modifiedKey), 0) + ) AS highWaterMark`, + { modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED } + ); + let highWaterMark = rows[0].getResultByName("highWaterMark"); + return highWaterMark / 1000; + } + + /** + * Updates the bookmarks collection last modified time. Note that this may + * be newer than the modified time of the most recent record. + * + * @param {Number|String} lastModifiedSeconds + * The collection last modified time, in seconds. + */ + async setCollectionLastModified(lastModifiedSeconds) { + let lastModified = Math.floor(lastModifiedSeconds * 1000); + if (!Number.isInteger(lastModified)) { + throw new TypeError("Invalid collection last modified time"); + } + await this.db.executeBeforeShutdown( + "SyncedBookmarksMirror: setCollectionLastModified", + db => + db.executeCached( + ` + REPLACE INTO meta(key, value) + VALUES(:modifiedKey, :lastModified)`, + { + modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED, + lastModified, + } + ) + ); + } + + /** + * Returns the bookmarks collection sync ID. This corresponds to + * `PlacesSyncUtils.bookmarks.getSyncId`. + * + * @return {String} + * The sync ID, or `""` if one isn't set. + */ + async getSyncId() { + let rows = await this.db.executeCached( + ` + SELECT value FROM meta WHERE key = :syncIdKey`, + { syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID } + ); + return rows.length ? rows[0].getResultByName("value") : ""; + } + + /** + * Ensures that the sync ID in the mirror is up-to-date with the server and + * Places, and discards the mirror on mismatch. + * + * The bookmarks engine store the same sync ID in Places and the mirror to + * "tie" the two together. This allows Sync to do the right thing if the + * database files are copied between profiles connected to different accounts. + * + * See `PlacesSyncUtils.bookmarks.ensureCurrentSyncId` for an explanation of + * how Places handles sync ID mismatches. + * + * @param {String} newSyncId + * The server's sync ID. + */ + async ensureCurrentSyncId(newSyncId) { + if (!newSyncId || typeof newSyncId != "string") { + throw new TypeError("Invalid new bookmarks sync ID"); + } + let existingSyncId = await this.getSyncId(); + if (existingSyncId == newSyncId) { + lazy.MirrorLog.trace("Sync ID up-to-date in mirror", { existingSyncId }); + return; + } + lazy.MirrorLog.info( + "Sync ID changed from ${existingSyncId} to " + + "${newSyncId}; resetting mirror", + { existingSyncId, newSyncId } + ); + await this.db.executeBeforeShutdown( + "SyncedBookmarksMirror: ensureCurrentSyncId", + db => + db.executeTransaction(async function () { + await resetMirror(db); + await db.execute( + ` + REPLACE INTO meta(key, value) + VALUES(:syncIdKey, :newSyncId)`, + { syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID, newSyncId } + ); + }) + ); + } + + /** + * Stores incoming or uploaded Sync records in the mirror. Rejects if any + * records are invalid. + * + * @param {PlacesItem[]} records + * Sync records to store in the mirror. + * @param {Boolean} [options.needsMerge] + * Indicates if the records were changed remotely since the last sync, + * and should be merged into the local tree. This option is set to + * `true` for incoming records, and `false` for successfully uploaded + * records. Tests can also pass `false` to set up an existing mirror. + * @param {AbortSignal} [options.signal] + * An abort signal that can be used to interrupt the operation. If + * omitted, storing incoming items can still be interrupted when the + * mirror is finalized. + */ + async store(records, { needsMerge = true, signal = null } = {}) { + let options = { + needsMerge, + signal: anyAborted(this.finalizeController.signal, signal), + }; + await this.db.executeBeforeShutdown("SyncedBookmarksMirror: store", db => + db.executeTransaction(async () => { + for (let record of records) { + if (options.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while storing incoming items" + ); + } + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + if (guid == lazy.PlacesUtils.bookmarks.rootGuid) { + // The engine should hard DELETE Places roots from the server. + throw new TypeError("Can't store Places root"); + } + if (lazy.MirrorLog.level <= lazy.Log.Level.Trace) { + lazy.MirrorLog.trace( + `Storing in mirror: ${record.cleartextToString()}` + ); + } + switch (record.type) { + case "bookmark": + await this.storeRemoteBookmark(record, options); + continue; + + case "query": + await this.storeRemoteQuery(record, options); + continue; + + case "folder": + await this.storeRemoteFolder(record, options); + continue; + + case "livemark": + await this.storeRemoteLivemark(record, options); + continue; + + case "separator": + await this.storeRemoteSeparator(record, options); + continue; + + default: + if (record.deleted) { + await this.storeRemoteTombstone(record, options); + continue; + } + } + lazy.MirrorLog.warn("Ignoring record with unknown type", record.type); + } + }) + ); + } + + /** + * Builds a complete merged tree from the local and remote trees, resolves + * value and structure conflicts, dedupes local items, applies the merged + * tree back to Places, and notifies observers about the changes. + * + * Merging and application happen in a transaction, meaning code that uses the + * main Places connection, including the UI, will fail to write to the + * database until the transaction commits. Asynchronous consumers will retry + * on `SQLITE_BUSY`; synchronous consumers will fail after waiting for 100ms. + * See bug 1305563, comment 122 for details. + * + * @param {Number} [options.localTimeSeconds] + * The current local time, in seconds. + * @param {Number} [options.remoteTimeSeconds] + * The current server time, in seconds. + * @param {Boolean} [options.notifyInStableOrder] + * If `true`, fire observer notifications for items in the same folder + * in a stable order. This is disabled by default, to avoid the cost + * of sorting the notifications, but enabled in some tests to simplify + * their checks. + * @param {AbortSignal} [options.signal] + * An abort signal that can be used to interrupt a merge when its + * associated `AbortController` is aborted. If omitted, the merge can + * still be interrupted when the mirror is finalized. + * @return {Object.} + * A changeset containing locally changed and reconciled records to + * upload to the server, and to store in the mirror once upload + * succeeds. + */ + async apply({ + localTimeSeconds, + remoteTimeSeconds, + notifyInStableOrder, + signal = null, + } = {}) { + // We intentionally don't use `executeBeforeShutdown` in this function, + // since merging can take a while for large trees, and we don't want to + // block shutdown. Since all new items are in the mirror, we'll just try + // to merge again on the next sync. + + let finalizeOrInterruptSignal = anyAborted( + this.finalizeController.signal, + signal + ); + + let changeRecords; + try { + changeRecords = await this.tryApply( + finalizeOrInterruptSignal, + localTimeSeconds, + remoteTimeSeconds, + notifyInStableOrder + ); + } finally { + this.progress.reset(); + } + + return changeRecords; + } + + async tryApply( + signal, + localTimeSeconds, + remoteTimeSeconds, + notifyInStableOrder = false + ) { + let wasMerged = await withTiming("Merging bookmarks in Rust", () => + this.merge(signal, localTimeSeconds, remoteTimeSeconds) + ); + + if (!wasMerged) { + lazy.MirrorLog.debug("No changes detected in both mirror and Places"); + return {}; + } + + // At this point, the database is consistent, so we can notify observers and + // inflate records for outgoing items. + + let observersToNotify = new BookmarkObserverRecorder(this.db, { + signal, + notifyInStableOrder, + }); + + await withTiming( + "Notifying Places observers", + async () => { + try { + // Note that we don't use a transaction when fetching info for + // observers, so it's possible we might notify with stale info if the + // main connection changes Places between the time we finish merging, + // and the time we notify observers. + await observersToNotify.notifyAll(); + } catch (ex) { + // Places relies on observer notifications to update internal caches. + // If notifying observers failed, these caches may be inconsistent, + // so we invalidate them just in case. + lazy.PlacesUtils.invalidateCachedGuids(); + await lazy.PlacesUtils.keywords.invalidateCachedKeywords(); + lazy.MirrorLog.warn("Error notifying Places observers", ex); + } finally { + await this.db.executeTransaction(async () => { + await this.db.execute(`DELETE FROM itemsAdded`); + await this.db.execute(`DELETE FROM guidsChanged`); + await this.db.execute(`DELETE FROM itemsChanged`); + await this.db.execute(`DELETE FROM itemsRemoved`); + await this.db.execute(`DELETE FROM itemsMoved`); + }); + } + }, + time => + this.progress.stepWithTelemetry( + ProgressTracker.STEPS.NOTIFY_OBSERVERS, + time + ) + ); + + let { changeRecords } = await withTiming( + "Fetching records for local items to upload", + async () => { + try { + let result = await this.fetchLocalChangeRecords(signal); + return result; + } finally { + await this.db.execute(`DELETE FROM itemsToUpload`); + } + }, + (time, result) => + this.progress.stepWithItemCount( + ProgressTracker.STEPS.FETCH_LOCAL_CHANGE_RECORDS, + time, + result.count + ) + ); + + return changeRecords; + } + + merge(signal, localTimeSeconds = Date.now() / 1000, remoteTimeSeconds = 0) { + return new Promise((resolve, reject) => { + let op = null; + function onAbort() { + signal.removeEventListener("abort", onAbort); + op.cancel(); + } + let callback = { + QueryInterface: ChromeUtils.generateQI([ + "mozISyncedBookmarksMirrorProgressListener", + "mozISyncedBookmarksMirrorCallback", + ]), + // `mozISyncedBookmarksMirrorProgressListener` methods. + onFetchLocalTree: (took, itemCount, deleteCount, problemsBag) => { + let counts = [ + { + name: "items", + count: itemCount, + }, + { + name: "deletions", + count: deleteCount, + }, + ]; + this.progress.stepWithTelemetry( + ProgressTracker.STEPS.FETCH_LOCAL_TREE, + took, + counts + ); + // We don't record local tree problems in validation telemetry. + }, + onFetchRemoteTree: (took, itemCount, deleteCount, problemsBag) => { + let counts = [ + { + name: "items", + count: itemCount, + }, + { + name: "deletions", + count: deleteCount, + }, + ]; + this.progress.stepWithTelemetry( + ProgressTracker.STEPS.FETCH_REMOTE_TREE, + took, + counts + ); + // Record validation telemetry for problems in the remote tree. + let problems = bagToNamedCounts(problemsBag, [ + "orphans", + "misparentedRoots", + "multipleParents", + "nonFolderParents", + "parentChildDisagreements", + "missingChildren", + ]); + let checked = itemCount + deleteCount; + this.recordValidationTelemetry(took, checked, problems); + }, + onMerge: (took, countsBag) => { + let counts = bagToNamedCounts(countsBag, [ + "items", + "dupes", + "remoteRevives", + "localDeletes", + "localRevives", + "remoteDeletes", + ]); + this.progress.stepWithTelemetry( + ProgressTracker.STEPS.MERGE, + took, + counts + ); + }, + onApply: took => { + this.progress.stepWithTelemetry(ProgressTracker.STEPS.APPLY, took); + }, + // `mozISyncedBookmarksMirrorCallback` methods. + handleSuccess(result) { + signal.removeEventListener("abort", onAbort); + resolve(result); + }, + handleError(code, message) { + signal.removeEventListener("abort", onAbort); + switch (code) { + case Cr.NS_ERROR_STORAGE_BUSY: + reject(new SyncedBookmarksMirror.MergeConflictError(message)); + break; + + case Cr.NS_ERROR_ABORT: + reject(new SyncedBookmarksMirror.InterruptedError(message)); + break; + + default: + reject(new SyncedBookmarksMirror.MergeError(message)); + } + }, + }; + op = this.merger.merge(localTimeSeconds, remoteTimeSeconds, callback); + if (signal.aborted) { + op.cancel(); + } else { + signal.addEventListener("abort", onAbort); + } + }); + } + + /** + * Discards the mirror contents. This is called when the user is node + * reassigned, disables the bookmarks engine, or signs out. + */ + async reset() { + await this.db.executeBeforeShutdown("SyncedBookmarksMirror: reset", db => + db.executeTransaction(() => resetMirror(db)) + ); + } + + /** + * Fetches the GUIDs of all items in the remote tree that need to be merged + * into the local tree. + * + * @return {String[]} + * Remotely changed GUIDs that need to be merged into Places. + */ + async fetchUnmergedGuids() { + let rows = await this.db.execute(` + SELECT guid FROM items + WHERE needsMerge + ORDER BY guid`); + return rows.map(row => row.getResultByName("guid")); + } + + async storeRemoteBookmark(record, { needsMerge, signal }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + + let url = validateURL(record.bmkUri); + if (url) { + await this.maybeStoreRemoteURL(url); + } + + let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid( + record.parentid + ); + let serverModified = determineServerModified(record); + let dateAdded = determineDateAdded(record); + let title = validateTitle(record.title); + let keyword = validateKeyword(record.keyword); + let validity = url + ? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID + : Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE; + + await this.db.executeCached( + ` + REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind, + dateAdded, title, keyword, validity, + urlId) + VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind, + :dateAdded, NULLIF(:title, ''), :keyword, :validity, + (SELECT id FROM urls + WHERE hash = hash(:url) AND + url = :url))`, + { + guid, + parentGuid, + serverModified, + needsMerge, + kind: Ci.mozISyncedBookmarksMerger.KIND_BOOKMARK, + dateAdded, + title, + keyword, + url: url ? url.href : null, + validity, + } + ); + + let tags = record.tags; + if (tags && Array.isArray(tags)) { + for (let rawTag of tags) { + if (signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while storing tags for incoming bookmark" + ); + } + let tag = validateTag(rawTag); + if (!tag) { + continue; + } + await this.db.executeCached( + ` + INSERT INTO tags(itemId, tag) + SELECT id, :tag FROM items + WHERE guid = :guid`, + { tag, guid } + ); + } + } + } + + async storeRemoteQuery(record, { needsMerge }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + + let validity = Ci.mozISyncedBookmarksMerger.VALIDITY_VALID; + + let url = validateURL(record.bmkUri); + if (url) { + // The query has a valid URL. Determine if we need to rewrite and reupload + // it. + let params = new URLSearchParams(url.href.slice(url.protocol.length)); + let type = +params.get("type"); + if (type == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) { + // Legacy tag queries with this type use a `place:` URL with a `folder` + // param that points to the tag folder ID. Rewrite the query to directly + // reference the tag stored in its `folderName`, then flag the rewritten + // query for reupload. + let tagFolderName = validateTag(record.folderName); + if (tagFolderName) { + try { + url.href = `place:tag=${tagFolderName}`; + validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD; + } catch (ex) { + // The tag folder name isn't URL-encoded (bug 1449939), so we might + // produce an invalid URL. However, invalid URLs are already likely + // to cause other issues, and it's better to replace or delete the + // query than break syncing or the Firefox UI. + url = null; + } + } else { + // The tag folder name is invalid, so replace or delete the remote + // copy. + url = null; + } + } else { + let folder = params.get("folder"); + if (folder && !params.has("excludeItems")) { + // We don't sync enough information to rewrite other queries with a + // `folder` param (bug 1377175). Referencing a nonexistent folder ID + // causes the query to return all items in the database, so we add + // `excludeItems=1` to stop it from doing so. We also flag the + // rewritten query for reupload. + try { + url.href = `${url.href}&excludeItems=1`; + validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD; + } catch (ex) { + url = null; + } + } + } + + // Other queries are implicitly valid, and don't need to be reuploaded + // or replaced. + } + + if (url) { + await this.maybeStoreRemoteURL(url); + } else { + // If the query doesn't have a valid URL, we must replace the remote copy + // with either a valid local copy, or a tombstone if the query doesn't + // exist locally. + validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE; + } + + let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid( + record.parentid + ); + let serverModified = determineServerModified(record); + let dateAdded = determineDateAdded(record); + let title = validateTitle(record.title); + + await this.db.executeCached( + ` + REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind, + dateAdded, title, + urlId, + validity) + VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind, + :dateAdded, NULLIF(:title, ''), + (SELECT id FROM urls + WHERE hash = hash(:url) AND + url = :url), + :validity)`, + { + guid, + parentGuid, + serverModified, + needsMerge, + kind: Ci.mozISyncedBookmarksMerger.KIND_QUERY, + dateAdded, + title, + url: url ? url.href : null, + validity, + } + ); + } + + async storeRemoteFolder(record, { needsMerge, signal }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid( + record.parentid + ); + let serverModified = determineServerModified(record); + let dateAdded = determineDateAdded(record); + let title = validateTitle(record.title); + + await this.db.executeCached( + ` + REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind, + dateAdded, title) + VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind, + :dateAdded, NULLIF(:title, ''))`, + { + guid, + parentGuid, + serverModified, + needsMerge, + kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER, + dateAdded, + title, + } + ); + + let children = record.children; + if (children && Array.isArray(children)) { + let offset = 0; + for (let chunk of lazy.PlacesUtils.chunkArray( + children, + this.db.variableLimit - 1 + )) { + if (signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while storing children for incoming folder" + ); + } + // Builds a fragment like `(?2, ?1, 0), (?3, ?1, 1), ...`, where ?1 is + // the folder's GUID, [?2, ?3] are the first and second child GUIDs + // (SQLite binding parameters index from 1), and [0, 1] are the + // positions. This lets us store the folder's children using as few + // statements as possible. + let valuesFragment = Array.from( + { length: chunk.length }, + (_, index) => `(?${index + 2}, ?1, ${offset + index})` + ).join(","); + await this.db.execute( + ` + INSERT INTO structure(guid, parentGuid, position) + VALUES ${valuesFragment}`, + [guid, ...chunk.map(lazy.PlacesSyncUtils.bookmarks.recordIdToGuid)] + ); + offset += chunk.length; + } + } + } + + async storeRemoteLivemark(record, { needsMerge }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid( + record.parentid + ); + let serverModified = determineServerModified(record); + let feedURL = validateURL(record.feedUri); + let dateAdded = determineDateAdded(record); + let title = validateTitle(record.title); + let siteURL = validateURL(record.siteUri); + + let validity = feedURL + ? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID + : Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE; + + await this.db.executeCached( + ` + REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind, + dateAdded, title, feedURL, siteURL, validity) + VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind, + :dateAdded, NULLIF(:title, ''), :feedURL, :siteURL, :validity)`, + { + guid, + parentGuid, + serverModified, + needsMerge, + kind: Ci.mozISyncedBookmarksMerger.KIND_LIVEMARK, + dateAdded, + title, + feedURL: feedURL ? feedURL.href : null, + siteURL: siteURL ? siteURL.href : null, + validity, + } + ); + } + + async storeRemoteSeparator(record, { needsMerge }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid( + record.parentid + ); + let serverModified = determineServerModified(record); + let dateAdded = determineDateAdded(record); + + await this.db.executeCached( + ` + REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind, + dateAdded) + VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind, + :dateAdded)`, + { + guid, + parentGuid, + serverModified, + needsMerge, + kind: Ci.mozISyncedBookmarksMerger.KIND_SEPARATOR, + dateAdded, + } + ); + } + + async storeRemoteTombstone(record, { needsMerge }) { + let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id); + let serverModified = determineServerModified(record); + + await this.db.executeCached( + ` + REPLACE INTO items(guid, serverModified, needsMerge, isDeleted) + VALUES(:guid, :serverModified, :needsMerge, 1)`, + { guid, serverModified, needsMerge } + ); + } + + async maybeStoreRemoteURL(url) { + await this.db.executeCached( + ` + INSERT OR IGNORE INTO urls(guid, url, hash, revHost) + VALUES(IFNULL((SELECT guid FROM urls + WHERE hash = hash(:url) AND + url = :url), + GENERATE_GUID()), :url, hash(:url), :revHost)`, + { url: url.href, revHost: lazy.PlacesUtils.getReversedHost(url) } + ); + } + + /** + * Inflates Sync records for all staged outgoing items. + * + * @param {AbortSignal} signal + * Stops fetching records when the associated `AbortController` + * is aborted. + * @return {Object} + * A `{ changeRecords, count }` tuple, where `changeRecords` is a + * changeset containing Sync record cleartexts for outgoing items and + * tombstones, keyed by their Sync record IDs, and `count` is the + * number of records. + */ + async fetchLocalChangeRecords(signal) { + let changeRecords = {}; + let childRecordIdsByLocalParentId = new Map(); + let tagsByLocalId = new Map(); + + let childGuidRows = []; + await this.db.execute( + `SELECT parentId, guid FROM structureToUpload + ORDER BY parentId, position`, + null, + (row, cancel) => { + if (signal.aborted) { + cancel(); + } else { + // `Sqlite.sys.mjs` callbacks swallow exceptions (bug 1387775), so we + // accumulate all rows in an array, and process them after. + childGuidRows.push(row); + } + } + ); + + await lazy.Async.yieldingForEach( + childGuidRows, + row => { + if (signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while fetching structure to upload" + ); + } + let localParentId = row.getResultByName("parentId"); + let childRecordId = lazy.PlacesSyncUtils.bookmarks.guidToRecordId( + row.getResultByName("guid") + ); + let childRecordIds = childRecordIdsByLocalParentId.get(localParentId); + if (childRecordIds) { + childRecordIds.push(childRecordId); + } else { + childRecordIdsByLocalParentId.set(localParentId, [childRecordId]); + } + }, + lazy.yieldState + ); + + let tagRows = []; + await this.db.execute( + `SELECT id, tag FROM tagsToUpload`, + null, + (row, cancel) => { + if (signal.aborted) { + cancel(); + } else { + tagRows.push(row); + } + } + ); + + await lazy.Async.yieldingForEach( + tagRows, + row => { + if (signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while fetching tags to upload" + ); + } + let localId = row.getResultByName("id"); + let tag = row.getResultByName("tag"); + let tags = tagsByLocalId.get(localId); + if (tags) { + tags.push(tag); + } else { + tagsByLocalId.set(localId, [tag]); + } + }, + lazy.yieldState + ); + + let itemRows = []; + await this.db.execute( + `SELECT id, syncChangeCounter, guid, isDeleted, type, isQuery, + tagFolderName, keyword, url, IFNULL(title, '') AS title, + position, parentGuid, + IFNULL(parentTitle, '') AS parentTitle, dateAdded + FROM itemsToUpload`, + null, + (row, cancel) => { + if (signal.interrupted) { + cancel(); + } else { + itemRows.push(row); + } + } + ); + + await lazy.Async.yieldingForEach( + itemRows, + row => { + if (signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while fetching items to upload" + ); + } + let syncChangeCounter = row.getResultByName("syncChangeCounter"); + + let guid = row.getResultByName("guid"); + let recordId = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(guid); + + // Tombstones don't carry additional properties. + let isDeleted = row.getResultByName("isDeleted"); + if (isDeleted) { + changeRecords[recordId] = new BookmarkChangeRecord( + syncChangeCounter, + { + id: recordId, + deleted: true, + } + ); + return; + } + + let parentGuid = row.getResultByName("parentGuid"); + let parentRecordId = + lazy.PlacesSyncUtils.bookmarks.guidToRecordId(parentGuid); + + let type = row.getResultByName("type"); + switch (type) { + case lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK: { + let isQuery = row.getResultByName("isQuery"); + if (isQuery) { + let queryCleartext = { + id: recordId, + type: "query", + // We ignore `parentid` and use the parent's `children`, but older + // Desktops and Android use `parentid` as the canonical parent. + // iOS is stricter and requires both `children` and `parentid` to + // match. + parentid: parentRecordId, + // Older Desktops use `hasDupe` (along with `parentName` for + // deduping), if hasDupe is true, then they won't attempt deduping + // (since they believe that a duplicate for this record should + // exist). We set it to true to prevent them from applying their + // deduping logic. + hasDupe: true, + parentName: row.getResultByName("parentTitle"), + // Omit `dateAdded` from the record if it's not set locally. + dateAdded: row.getResultByName("dateAdded") || undefined, + bmkUri: row.getResultByName("url"), + title: row.getResultByName("title"), + // folderName should never be an empty string or null + folderName: row.getResultByName("tagFolderName") || undefined, + }; + changeRecords[recordId] = new BookmarkChangeRecord( + syncChangeCounter, + queryCleartext + ); + return; + } + + let bookmarkCleartext = { + id: recordId, + type: "bookmark", + parentid: parentRecordId, + hasDupe: true, + parentName: row.getResultByName("parentTitle"), + dateAdded: row.getResultByName("dateAdded") || undefined, + bmkUri: row.getResultByName("url"), + title: row.getResultByName("title"), + }; + let keyword = row.getResultByName("keyword"); + if (keyword) { + bookmarkCleartext.keyword = keyword; + } + let localId = row.getResultByName("id"); + let tags = tagsByLocalId.get(localId); + if (tags) { + bookmarkCleartext.tags = tags; + } + changeRecords[recordId] = new BookmarkChangeRecord( + syncChangeCounter, + bookmarkCleartext + ); + return; + } + + case lazy.PlacesUtils.bookmarks.TYPE_FOLDER: { + let folderCleartext = { + id: recordId, + type: "folder", + parentid: parentRecordId, + hasDupe: true, + parentName: row.getResultByName("parentTitle"), + dateAdded: row.getResultByName("dateAdded") || undefined, + title: row.getResultByName("title"), + }; + let localId = row.getResultByName("id"); + let childRecordIds = childRecordIdsByLocalParentId.get(localId); + folderCleartext.children = childRecordIds || []; + changeRecords[recordId] = new BookmarkChangeRecord( + syncChangeCounter, + folderCleartext + ); + return; + } + + case lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR: { + let separatorCleartext = { + id: recordId, + type: "separator", + parentid: parentRecordId, + hasDupe: true, + parentName: row.getResultByName("parentTitle"), + dateAdded: row.getResultByName("dateAdded") || undefined, + // Older Desktops use `pos` for deduping. + pos: row.getResultByName("position"), + }; + changeRecords[recordId] = new BookmarkChangeRecord( + syncChangeCounter, + separatorCleartext + ); + return; + } + + default: + throw new TypeError("Can't create record for unknown Places item"); + } + }, + lazy.yieldState + ); + + return { changeRecords, count: itemRows.length }; + } + + /** + * Closes the mirror database connection. This is called automatically on + * shutdown, but may also be called explicitly when the mirror is no longer + * needed. + * + * @param {Boolean} [options.alsoCleanup] + * If specified, drop all temp tables, views, and triggers, + * and detach from the mirror database before closing the + * connection. Defaults to `true`. + */ + finalize({ alsoCleanup = true } = {}) { + if (!this.finalizePromise) { + this.finalizePromise = (async () => { + this.progress.step(ProgressTracker.STEPS.FINALIZE); + this.finalizeController.abort(); + this.merger.reset(); + if (alsoCleanup) { + // If the mirror is finalized explicitly, clean up temp entities and + // detach from the mirror database. We can skip this for automatic + // finalization, since the Places connection is already shutting + // down. + await cleanupMirrorDatabase(this.db); + } + await this.db.execute(`PRAGMA mirror.optimize(0x02)`); + await this.db.execute(`DETACH mirror`); + this.finalizeAt.removeBlocker(this.finalizeBound); + })(); + } + return this.finalizePromise; + } +} + +/** Key names for the key-value `meta` table. */ +SyncedBookmarksMirror.META_KEY = { + LAST_MODIFIED: "collection/lastModified", + SYNC_ID: "collection/syncId", +}; + +/** + * An error thrown when the merge was interrupted. + */ +class InterruptedError extends Error { + constructor(message) { + super(message); + this.name = "InterruptedError"; + } +} +SyncedBookmarksMirror.InterruptedError = InterruptedError; + +/** + * An error thrown when the merge failed for an unexpected reason. + */ +class MergeError extends Error { + constructor(message) { + super(message); + this.name = "MergeError"; + } +} +SyncedBookmarksMirror.MergeError = MergeError; + +/** + * An error thrown when the merge can't proceed because the local tree + * changed during the merge. + */ +class MergeConflictError extends Error { + constructor(message) { + super(message); + this.name = "MergeConflictError"; + } +} +SyncedBookmarksMirror.MergeConflictError = MergeConflictError; + +/** + * An error thrown when the mirror database is corrupt, or can't be migrated to + * the latest schema version, and must be replaced. + */ +class DatabaseCorruptError extends Error { + constructor(message) { + super(message); + this.name = "DatabaseCorruptError"; + } +} + +// Indicates if the mirror should be replaced because the database file is +// corrupt. +function isDatabaseCorrupt(error) { + if (error instanceof DatabaseCorruptError) { + return true; + } + if (error.errors) { + return error.errors.some( + error => + error instanceof Ci.mozIStorageError && + (error.result == Ci.mozIStorageError.CORRUPT || + error.result == Ci.mozIStorageError.NOTADB) + ); + } + return false; +} + +/** + * Attaches a cloned Places database connection to the mirror database, + * migrates the mirror schema to the latest version, and creates temporary + * tables, views, and triggers. + * + * @param {Sqlite.OpenedConnection} db + * The Places database connection. + * @param {String} path + * The full path to the mirror database file. + */ +async function attachAndInitMirrorDatabase(db, path) { + await db.execute(`ATTACH :path AS mirror`, { path }); + try { + await db.executeTransaction(async function () { + let currentSchemaVersion = await db.getSchemaVersion("mirror"); + if (currentSchemaVersion > 0) { + if (currentSchemaVersion < MIRROR_SCHEMA_VERSION) { + await migrateMirrorSchema(db, currentSchemaVersion); + } + } else { + await initializeMirrorDatabase(db); + } + // Downgrading from a newer profile to an older profile rolls back the + // schema version, but leaves all new columns in place. We'll run the + // migration logic again on the next upgrade. + await db.setSchemaVersion(MIRROR_SCHEMA_VERSION, "mirror"); + await initializeTempMirrorEntities(db); + }); + } catch (ex) { + await db.execute(`DETACH mirror`); + throw ex; + } +} + +/** + * Migrates the mirror database schema to the latest version. + * + * @param {Sqlite.OpenedConnection} db + * The mirror database connection. + * @param {Number} currentSchemaVersion + * The current mirror database schema version. + */ +async function migrateMirrorSchema(db, currentSchemaVersion) { + if (currentSchemaVersion < 5) { + // The mirror was pref'd off by default for schema versions 1-4. + throw new DatabaseCorruptError( + `Can't migrate from schema version ${currentSchemaVersion}; too old` + ); + } + if (currentSchemaVersion < 6) { + await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemURLs ON + items(urlId)`); + await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemKeywords ON + items(keyword) WHERE keyword NOT NULL`); + } + if (currentSchemaVersion < 7) { + await db.execute(`CREATE INDEX IF NOT EXISTS mirror.structurePositions ON + structure(parentGuid, position)`); + } + if (currentSchemaVersion < 8) { + // Not really a "schema" update, but addresses the defect from bug 1635859. + // In short, every bookmark with a corresponding entry in the mirror should + // have syncStatus = NORMAL. + await db.execute(`UPDATE moz_bookmarks AS b + SET syncStatus = ${lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL} + WHERE EXISTS (SELECT 1 FROM mirror.items + WHERE guid = b.guid)`); + } +} + +/** + * Initializes a new mirror database, creating persistent tables, indexes, and + * roots. + * + * @param {Sqlite.OpenedConnection} db + * The mirror database connection. + */ +async function initializeMirrorDatabase(db) { + // Key-value metadata table. Stores the server collection last modified time + // and sync ID. + await db.execute(`CREATE TABLE mirror.meta( + key TEXT PRIMARY KEY, + value NOT NULL + ) WITHOUT ROWID`); + + // Note: description and loadInSidebar are not used as of Firefox 63, but + // remain to avoid rebuilding the database if the user happens to downgrade. + await db.execute(`CREATE TABLE mirror.items( + id INTEGER PRIMARY KEY, + guid TEXT UNIQUE NOT NULL, + /* The "parentid" from the record. */ + parentGuid TEXT, + /* The server modified time, in milliseconds. */ + serverModified INTEGER NOT NULL DEFAULT 0, + needsMerge BOOLEAN NOT NULL DEFAULT 0, + validity INTEGER NOT NULL DEFAULT ${Ci.mozISyncedBookmarksMerger.VALIDITY_VALID}, + isDeleted BOOLEAN NOT NULL DEFAULT 0, + kind INTEGER NOT NULL DEFAULT -1, + /* The creation date, in milliseconds. */ + dateAdded INTEGER NOT NULL DEFAULT 0, + title TEXT, + urlId INTEGER REFERENCES urls(id) + ON DELETE SET NULL, + keyword TEXT, + description TEXT, + loadInSidebar BOOLEAN, + smartBookmarkName TEXT, + feedURL TEXT, + siteURL TEXT + )`); + + await db.execute(`CREATE TABLE mirror.structure( + guid TEXT, + parentGuid TEXT REFERENCES items(guid) + ON DELETE CASCADE, + position INTEGER NOT NULL, + PRIMARY KEY(parentGuid, guid) + ) WITHOUT ROWID`); + + await db.execute(`CREATE TABLE mirror.urls( + id INTEGER PRIMARY KEY, + guid TEXT NOT NULL, + url TEXT NOT NULL, + hash INTEGER NOT NULL, + revHost TEXT NOT NULL + )`); + + await db.execute(`CREATE TABLE mirror.tags( + itemId INTEGER NOT NULL REFERENCES items(id) + ON DELETE CASCADE, + tag TEXT NOT NULL + )`); + + await db.execute( + `CREATE INDEX mirror.structurePositions ON structure(parentGuid, position)` + ); + + await db.execute(`CREATE INDEX mirror.urlHashes ON urls(hash)`); + + await db.execute(`CREATE INDEX mirror.itemURLs ON items(urlId)`); + + await db.execute(`CREATE INDEX mirror.itemKeywords ON items(keyword) + WHERE keyword NOT NULL`); + + await createMirrorRoots(db); +} + +/** + * Drops all temp tables, views, and triggers used for merging, and detaches + * from the mirror database. + * + * @param {Sqlite.OpenedConnection} db + * The mirror database connection. + */ +async function cleanupMirrorDatabase(db) { + await db.executeTransaction(async function () { + await db.execute(`DROP TABLE changeGuidOps`); + await db.execute(`DROP TABLE itemsToApply`); + await db.execute(`DROP TABLE applyNewLocalStructureOps`); + await db.execute(`DROP VIEW localTags`); + await db.execute(`DROP TABLE itemsAdded`); + await db.execute(`DROP TABLE guidsChanged`); + await db.execute(`DROP TABLE itemsChanged`); + await db.execute(`DROP TABLE itemsMoved`); + await db.execute(`DROP TABLE itemsRemoved`); + await db.execute(`DROP TABLE itemsToUpload`); + await db.execute(`DROP TABLE structureToUpload`); + await db.execute(`DROP TABLE tagsToUpload`); + }); +} + +/** + * Sets up the syncable roots. All items in the mirror we apply will descend + * from these roots - however, malformed records from the server which create + * a different root *will* be created in the mirror - just not applied. + * + * + * @param {Sqlite.OpenedConnection} db + * The mirror database connection. + */ +async function createMirrorRoots(db) { + const syncableRoots = [ + { + guid: lazy.PlacesUtils.bookmarks.rootGuid, + // The Places root is its own parent, to satisfy the foreign key and + // `NOT NULL` constraints on `structure`. + parentGuid: lazy.PlacesUtils.bookmarks.rootGuid, + position: -1, + needsMerge: false, + }, + ...lazy.PlacesUtils.bookmarks.userContentRoots.map((guid, position) => { + return { + guid, + parentGuid: lazy.PlacesUtils.bookmarks.rootGuid, + position, + needsMerge: true, + }; + }), + ]; + + for (let { guid, parentGuid, position, needsMerge } of syncableRoots) { + await db.executeCached( + ` + INSERT INTO items(guid, parentGuid, kind, needsMerge) + VALUES(:guid, :parentGuid, :kind, :needsMerge)`, + { + guid, + parentGuid, + kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER, + needsMerge, + } + ); + + await db.executeCached( + ` + INSERT INTO structure(guid, parentGuid, position) + VALUES(:guid, :parentGuid, :position)`, + { guid, parentGuid, position } + ); + } +} + +/** + * Creates temporary tables, views, and triggers to apply the mirror to Places. + * + * @param {Sqlite.OpenedConnection} db + * The mirror database connection. + */ +async function initializeTempMirrorEntities(db) { + await db.execute(`CREATE TEMP TABLE changeGuidOps( + localGuid TEXT PRIMARY KEY, + mergedGuid TEXT UNIQUE NOT NULL, + syncStatus INTEGER, + level INTEGER NOT NULL, + lastModifiedMicroseconds INTEGER NOT NULL + ) WITHOUT ROWID`); + + await db.execute(` + CREATE TEMP TRIGGER changeGuids + AFTER DELETE ON changeGuidOps + BEGIN + /* Record item changed notifications for the updated GUIDs. */ + INSERT INTO guidsChanged(itemId, oldGuid, level) + SELECT b.id, OLD.localGuid, OLD.level + FROM moz_bookmarks b + WHERE b.guid = OLD.localGuid; + + UPDATE moz_bookmarks SET + guid = OLD.mergedGuid, + lastModified = OLD.lastModifiedMicroseconds, + syncStatus = IFNULL(OLD.syncStatus, syncStatus) + WHERE guid = OLD.localGuid; + END`); + + await db.execute(`CREATE TEMP TABLE itemsToApply( + mergedGuid TEXT PRIMARY KEY, + localId INTEGER UNIQUE, + remoteId INTEGER UNIQUE NOT NULL, + remoteGuid TEXT UNIQUE NOT NULL, + newLevel INTEGER NOT NULL, + newType INTEGER NOT NULL, + localDateAddedMicroseconds INTEGER, + remoteDateAddedMicroseconds INTEGER NOT NULL, + lastModifiedMicroseconds INTEGER NOT NULL, + oldTitle TEXT, + newTitle TEXT, + oldPlaceId INTEGER, + newPlaceId INTEGER, + newKeyword TEXT + )`); + + await db.execute(`CREATE INDEX existingItems ON itemsToApply(localId) + WHERE localId NOT NULL`); + + await db.execute(`CREATE INDEX oldPlaceIds ON itemsToApply(oldPlaceId) + WHERE oldPlaceId NOT NULL`); + + await db.execute(`CREATE INDEX newPlaceIds ON itemsToApply(newPlaceId) + WHERE newPlaceId NOT NULL`); + + await db.execute(`CREATE INDEX newKeywords ON itemsToApply(newKeyword) + WHERE newKeyword NOT NULL`); + + await db.execute(`CREATE TEMP TABLE applyNewLocalStructureOps( + mergedGuid TEXT PRIMARY KEY, + mergedParentGuid TEXT NOT NULL, + position INTEGER NOT NULL, + level INTEGER NOT NULL, + lastModifiedMicroseconds INTEGER NOT NULL + ) WITHOUT ROWID`); + + await db.execute(` + CREATE TEMP TRIGGER applyNewLocalStructure + AFTER DELETE ON applyNewLocalStructureOps + BEGIN + INSERT INTO itemsMoved(itemId, oldParentId, oldParentGuid, oldPosition, + level) + SELECT b.id, p.id, p.guid, b.position, OLD.level + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.guid = OLD.mergedGuid; + + UPDATE moz_bookmarks SET + parent = (SELECT id FROM moz_bookmarks + WHERE guid = OLD.mergedParentGuid), + position = OLD.position, + lastModified = OLD.lastModifiedMicroseconds + WHERE guid = OLD.mergedGuid; + END`); + + // A view of local bookmark tags. Tags, like keywords, are associated with + // URLs, so two bookmarks with the same URL should have the same tags. Unlike + // keywords, one tag may be associated with many different URLs. Tags are also + // different because they're implemented as bookmarks under the hood. Each tag + // is stored as a folder under the tags root, and tagged URLs are stored as + // untitled bookmarks under these folders. This complexity can be removed once + // bug 424160 lands. + await db.execute(` + CREATE TEMP VIEW localTags(tagEntryId, tagEntryGuid, tagFolderId, + tagFolderGuid, tagEntryPosition, tagEntryType, + tag, placeId, lastModifiedMicroseconds) AS + SELECT b.id, b.guid, p.id, p.guid, b.position, b.type, + p.title, b.fk, b.lastModified + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.type = ${lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK} AND + p.parent = (SELECT id FROM moz_bookmarks + WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')`); + + // Untags a URL by removing its tag entry. + await db.execute(` + CREATE TEMP TRIGGER untagLocalPlace + INSTEAD OF DELETE ON localTags + BEGIN + /* Record an item removed notification for the tag entry. */ + INSERT INTO itemsRemoved(itemId, parentId, position, type, placeId, guid, + parentGuid, title, isUntagging) + VALUES(OLD.tagEntryId, OLD.tagFolderId, OLD.tagEntryPosition, + OLD.tagEntryType, OLD.placeId, OLD.tagEntryGuid, + OLD.tagFolderGuid, OLD.tag, 1); + + DELETE FROM moz_bookmarks WHERE id = OLD.tagEntryId; + + /* Fix the positions of the sibling tag entries. */ + UPDATE moz_bookmarks SET + position = position - 1 + WHERE parent = OLD.tagFolderId AND + position > OLD.tagEntryPosition; + END`); + + // Tags a URL by creating a tag folder if it doesn't exist, then inserting a + // tag entry for the URL into the tag folder. `NEW.placeId` can be NULL, in + // which case we'll just create the tag folder. + await db.execute(` + CREATE TEMP TRIGGER tagLocalPlace + INSTEAD OF INSERT ON localTags + BEGIN + /* Ensure the tag folder exists. */ + INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, title, + dateAdded, lastModified) + VALUES(IFNULL((SELECT b.guid FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.title = NEW.tag AND + p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'), + GENERATE_GUID()), + (SELECT id FROM moz_bookmarks + WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'), + (SELECT COUNT(*) FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'), + ${lazy.PlacesUtils.bookmarks.TYPE_FOLDER}, NEW.tag, + NEW.lastModifiedMicroseconds, + NEW.lastModifiedMicroseconds); + + /* Record an item added notification if we created a tag folder. + "CHANGES()" returns the number of rows affected by the INSERT above: + 1 if we created the folder, or 0 if the folder already existed. */ + INSERT INTO itemsAdded(guid, isTagging) + SELECT b.guid, 1 + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE CHANGES() > 0 AND + b.title = NEW.tag AND + p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'; + + /* Add a tag entry for the URL under the tag folder. Omitting the place + ID creates a tag folder without tagging the URL. */ + INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, fk, + dateAdded, lastModified) + SELECT IFNULL((SELECT b.guid FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.fk = NEW.placeId AND + p.title = NEW.tag AND + p.parent = (SELECT id FROM moz_bookmarks + WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')), + GENERATE_GUID()), + (SELECT b.id FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}' AND + b.title = NEW.tag), + (SELECT COUNT(*) FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE p.title = NEW.tag AND + p.parent = (SELECT id FROM moz_bookmarks + WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')), + ${lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK}, NEW.placeId, + NEW.lastModifiedMicroseconds, + NEW.lastModifiedMicroseconds + WHERE NEW.placeId NOT NULL; + + /* Record an item added notification for the tag entry. */ + INSERT INTO itemsAdded(guid, isTagging) + SELECT b.guid, 1 + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE CHANGES() > 0 AND + b.fk = NEW.placeId AND + p.title = NEW.tag AND + p.parent = (SELECT id FROM moz_bookmarks + WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'); + END`); + + // Stores properties to pass to `onItem{Added, Changed, Moved, Removed}` + // bookmark observers for new, updated, moved, and deleted items. + await db.execute(`CREATE TEMP TABLE itemsAdded( + guid TEXT PRIMARY KEY, + isTagging BOOLEAN NOT NULL DEFAULT 0, + keywordChanged BOOLEAN NOT NULL DEFAULT 0, + level INTEGER NOT NULL DEFAULT -1 + ) WITHOUT ROWID`); + + await db.execute(`CREATE INDEX addedItemLevels ON itemsAdded(level)`); + + await db.execute(`CREATE TEMP TABLE guidsChanged( + itemId INTEGER PRIMARY KEY, + oldGuid TEXT NOT NULL, + level INTEGER NOT NULL DEFAULT -1 + )`); + + await db.execute(`CREATE INDEX changedGuidLevels ON guidsChanged(level)`); + + await db.execute(`CREATE TEMP TABLE itemsChanged( + itemId INTEGER PRIMARY KEY, + oldTitle TEXT, + oldPlaceId INTEGER, + keywordChanged BOOLEAN NOT NULL DEFAULT 0, + level INTEGER NOT NULL DEFAULT -1 + )`); + + await db.execute(`CREATE INDEX changedItemLevels ON itemsChanged(level)`); + + await db.execute(`CREATE TEMP TABLE itemsMoved( + itemId INTEGER PRIMARY KEY, + oldParentId INTEGER NOT NULL, + oldParentGuid TEXT NOT NULL, + oldPosition INTEGER NOT NULL, + level INTEGER NOT NULL DEFAULT -1 + )`); + + await db.execute(`CREATE INDEX movedItemLevels ON itemsMoved(level)`); + + await db.execute(`CREATE TEMP TABLE itemsRemoved( + itemId INTEGER PRIMARY KEY, + guid TEXT NOT NULL, + parentId INTEGER NOT NULL, + position INTEGER NOT NULL, + type INTEGER NOT NULL, + title TEXT NOT NULL, + placeId INTEGER, + parentGuid TEXT NOT NULL, + /* We record the original level of the removed item in the tree so that we + can notify children before parents. */ + level INTEGER NOT NULL DEFAULT -1, + isUntagging BOOLEAN NOT NULL DEFAULT 0, + keywordRemoved BOOLEAN NOT NULL DEFAULT 0 + )`); + + await db.execute( + `CREATE INDEX removedItemLevels ON itemsRemoved(level DESC)` + ); + + // Stores locally changed items staged for upload. + await db.execute(`CREATE TEMP TABLE itemsToUpload( + id INTEGER PRIMARY KEY, + guid TEXT UNIQUE NOT NULL, + syncChangeCounter INTEGER NOT NULL, + isDeleted BOOLEAN NOT NULL DEFAULT 0, + parentGuid TEXT, + parentTitle TEXT, + dateAdded INTEGER, /* In milliseconds. */ + type INTEGER, + title TEXT, + placeId INTEGER, + isQuery BOOLEAN NOT NULL DEFAULT 0, + url TEXT, + tagFolderName TEXT, + keyword TEXT, + position INTEGER + )`); + + await db.execute(`CREATE TEMP TABLE structureToUpload( + guid TEXT PRIMARY KEY, + parentId INTEGER NOT NULL REFERENCES itemsToUpload(id) + ON DELETE CASCADE, + position INTEGER NOT NULL + ) WITHOUT ROWID`); + + await db.execute( + `CREATE INDEX parentsToUpload ON structureToUpload(parentId, position)` + ); + + await db.execute(`CREATE TEMP TABLE tagsToUpload( + id INTEGER REFERENCES itemsToUpload(id) + ON DELETE CASCADE, + tag TEXT, + PRIMARY KEY(id, tag) + ) WITHOUT ROWID`); +} + +async function resetMirror(db) { + await db.execute(`DELETE FROM meta`); + await db.execute(`DELETE FROM structure`); + await db.execute(`DELETE FROM items`); + await db.execute(`DELETE FROM urls`); + + // Since we need to reset the modified times and merge flags for the syncable + // roots, we simply delete and recreate them. + await createMirrorRoots(db); +} + +// Converts a Sync record's last modified time to milliseconds. +function determineServerModified(record) { + return Math.max(record.modified * 1000, 0) || 0; +} + +// Determines a Sync record's creation date. +function determineDateAdded(record) { + let serverModified = determineServerModified(record); + return lazy.PlacesSyncUtils.bookmarks.ratchetTimestampBackwards( + record.dateAdded, + serverModified + ); +} + +function validateTitle(rawTitle) { + if (typeof rawTitle != "string" || !rawTitle) { + return null; + } + return rawTitle.slice(0, DB_TITLE_LENGTH_MAX); +} + +function validateURL(rawURL) { + if (typeof rawURL != "string" || rawURL.length > DB_URL_LENGTH_MAX) { + return null; + } + let url = null; + try { + url = new URL(rawURL); + } catch (ex) {} + return url; +} + +function validateKeyword(rawKeyword) { + if (typeof rawKeyword != "string") { + return null; + } + let keyword = rawKeyword.trim(); + // Drop empty keywords. + return keyword ? keyword.toLowerCase() : null; +} + +function validateTag(rawTag) { + if (typeof rawTag != "string") { + return null; + } + let tag = rawTag.trim(); + if (!tag || tag.length > lazy.PlacesUtils.bookmarks.MAX_TAG_LENGTH) { + // Drop empty and oversized tags. + return null; + } + return tag; +} + +/** + * Measures and logs the time taken to execute a function, using a monotonic + * clock. + * + * @param {String} name + * The name of the operation, used for logging. + * @param {Function} func + * The function to time. + * @param {Function} [recordTiming] + * An optional function with the signature `(time: Number)`, where + * `time` is the measured time. + * @return The return value of the timed function. + */ +async function withTiming(name, func, recordTiming) { + lazy.MirrorLog.debug(name); + + let startTime = Cu.now(); + let result = await func(); + let elapsedTime = Cu.now() - startTime; + + lazy.MirrorLog.debug(`${name} took ${elapsedTime.toFixed(3)}ms`); + if (typeof recordTiming == "function") { + recordTiming(elapsedTime, result); + } + + return result; +} + +/** + * Fires bookmark and keyword observer notifications for all changes made during + * the merge. + */ +class BookmarkObserverRecorder { + constructor(db, { notifyInStableOrder, signal }) { + this.db = db; + this.notifyInStableOrder = notifyInStableOrder; + this.signal = signal; + this.placesEvents = []; + this.shouldInvalidateKeywords = false; + } + + /** + * Fires observer notifications for all changed items, invalidates the + * livemark cache if necessary, and recalculates frecencies for changed + * URLs. This is called outside the merge transaction. + */ + async notifyAll() { + await this.noteAllChanges(); + if (this.shouldInvalidateKeywords) { + await lazy.PlacesUtils.keywords.invalidateCachedKeywords(); + } + this.notifyBookmarkObservers(); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted before recalculating frecencies for new URLs" + ); + } + } + + orderBy(level, parent, position) { + return `ORDER BY ${ + this.notifyInStableOrder ? `${level}, ${parent}, ${position}` : level + }`; + } + + /** + * Records Places observer notifications for removed, added, moved, and + * changed items. + */ + async noteAllChanges() { + lazy.MirrorLog.trace("Recording observer notifications for removed items"); + // `ORDER BY v.level DESC` sorts deleted children before parents, to ensure + // that we update caches in the correct order (bug 1297941). + await this.db.execute( + `SELECT v.itemId AS id, v.parentId, v.parentGuid, v.position, v.type, + (SELECT h.url FROM moz_places h WHERE h.id = v.placeId) AS url, + v.title, v.guid, v.isUntagging, v.keywordRemoved + FROM itemsRemoved v + ${this.orderBy("v.level", "v.parentId", "v.position")}`, + null, + (row, cancel) => { + if (this.signal.aborted) { + cancel(); + return; + } + let info = { + id: row.getResultByName("id"), + parentId: row.getResultByName("parentId"), + position: row.getResultByName("position"), + type: row.getResultByName("type"), + urlHref: row.getResultByName("url"), + title: row.getResultByName("title"), + guid: row.getResultByName("guid"), + parentGuid: row.getResultByName("parentGuid"), + isUntagging: row.getResultByName("isUntagging"), + }; + this.noteItemRemoved(info); + if (row.getResultByName("keywordRemoved")) { + this.shouldInvalidateKeywords = true; + } + } + ); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while recording observer notifications for removed items" + ); + } + + lazy.MirrorLog.trace("Recording observer notifications for changed GUIDs"); + await this.db.execute( + `SELECT b.id, b.lastModified, b.type, b.guid AS newGuid, + p.guid AS parentGuid, gp.guid AS grandParentGuid + FROM guidsChanged c + JOIN moz_bookmarks b ON b.id = c.itemId + JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_bookmarks gp ON gp.id = p.parent + ${this.orderBy("c.level", "b.parent", "b.position")}`, + null, + (row, cancel) => { + if (this.signal.aborted) { + cancel(); + return; + } + let info = { + id: row.getResultByName("id"), + lastModified: row.getResultByName("lastModified"), + type: row.getResultByName("type"), + newGuid: row.getResultByName("newGuid"), + parentGuid: row.getResultByName("parentGuid"), + grandParentGuid: row.getResultByName("grandParentGuid"), + }; + this.noteGuidChanged(info); + } + ); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while recording observer notifications for changed GUIDs" + ); + } + + lazy.MirrorLog.trace("Recording observer notifications for new items"); + await this.db.execute( + `SELECT b.id, p.id AS parentId, b.position, b.type, + (SELECT h.url FROM moz_places h WHERE h.id = b.fk) AS url, + IFNULL(b.title, '') AS title, b.dateAdded, b.guid, + p.guid AS parentGuid, n.isTagging, n.keywordChanged + FROM itemsAdded n + JOIN moz_bookmarks b ON b.guid = n.guid + JOIN moz_bookmarks p ON p.id = b.parent + ${this.orderBy("n.level", "b.parent", "b.position")}`, + null, + (row, cancel) => { + if (this.signal.aborted) { + cancel(); + return; + } + let info = { + id: row.getResultByName("id"), + parentId: row.getResultByName("parentId"), + position: row.getResultByName("position"), + type: row.getResultByName("type"), + urlHref: row.getResultByName("url"), + title: row.getResultByName("title"), + dateAdded: row.getResultByName("dateAdded"), + guid: row.getResultByName("guid"), + parentGuid: row.getResultByName("parentGuid"), + isTagging: row.getResultByName("isTagging"), + }; + this.noteItemAdded(info); + if (row.getResultByName("keywordChanged")) { + this.shouldInvalidateKeywords = true; + } + } + ); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while recording observer notifications for new items" + ); + } + + lazy.MirrorLog.trace("Recording observer notifications for moved items"); + await this.db.execute( + `SELECT b.id, b.guid, b.type, p.guid AS newParentGuid, c.oldParentGuid, + b.position AS newPosition, c.oldPosition, + gp.guid AS grandParentGuid, + (SELECT h.url FROM moz_places h WHERE h.id = b.fk) AS url + FROM itemsMoved c + JOIN moz_bookmarks b ON b.id = c.itemId + JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_bookmarks gp ON gp.id = p.parent + ${this.orderBy("c.level", "b.parent", "b.position")}`, + null, + (row, cancel) => { + if (this.signal.aborted) { + cancel(); + return; + } + let info = { + id: row.getResultByName("id"), + guid: row.getResultByName("guid"), + type: row.getResultByName("type"), + newParentGuid: row.getResultByName("newParentGuid"), + oldParentGuid: row.getResultByName("oldParentGuid"), + newPosition: row.getResultByName("newPosition"), + oldPosition: row.getResultByName("oldPosition"), + urlHref: row.getResultByName("url"), + grandParentGuid: row.getResultByName("grandParentGuid"), + }; + this.noteItemMoved(info); + } + ); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while recording observer notifications for moved items" + ); + } + + lazy.MirrorLog.trace("Recording observer notifications for changed items"); + await this.db.execute( + `SELECT b.id, b.guid, b.lastModified, b.type, + IFNULL(b.title, '') AS newTitle, + IFNULL(c.oldTitle, '') AS oldTitle, + (SELECT h.url FROM moz_places h + WHERE h.id = b.fk) AS newURL, + (SELECT h.url FROM moz_places h + WHERE h.id = c.oldPlaceId) AS oldURL, + p.id AS parentId, p.guid AS parentGuid, + c.keywordChanged, + gp.guid AS grandParentGuid, + (SELECT h.url FROM moz_places h WHERE h.id = b.fk) AS url + FROM itemsChanged c + JOIN moz_bookmarks b ON b.id = c.itemId + JOIN moz_bookmarks p ON p.id = b.parent + LEFT JOIN moz_bookmarks gp ON gp.id = p.parent + ${this.orderBy("c.level", "b.parent", "b.position")}`, + null, + (row, cancel) => { + if (this.signal.aborted) { + cancel(); + return; + } + let info = { + id: row.getResultByName("id"), + guid: row.getResultByName("guid"), + lastModified: row.getResultByName("lastModified"), + type: row.getResultByName("type"), + newTitle: row.getResultByName("newTitle"), + oldTitle: row.getResultByName("oldTitle"), + newURLHref: row.getResultByName("newURL"), + oldURLHref: row.getResultByName("oldURL"), + parentId: row.getResultByName("parentId"), + parentGuid: row.getResultByName("parentGuid"), + grandParentGuid: row.getResultByName("grandParentGuid"), + }; + this.noteItemChanged(info); + if (row.getResultByName("keywordChanged")) { + this.shouldInvalidateKeywords = true; + } + } + ); + if (this.signal.aborted) { + throw new SyncedBookmarksMirror.InterruptedError( + "Interrupted while recording observer notifications for changed items" + ); + } + } + + noteItemAdded(info) { + this.placesEvents.push( + new PlacesBookmarkAddition({ + id: info.id, + parentId: info.parentId, + index: info.position, + url: info.urlHref || "", + title: info.title, + // Note that both the database and the legacy `onItem{Moved, Removed, + // Changed}` notifications use microsecond timestamps, but + // `PlacesBookmarkAddition` uses milliseconds. + dateAdded: info.dateAdded / 1000, + guid: info.guid, + parentGuid: info.parentGuid, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + itemType: info.type, + isTagging: info.isTagging, + }) + ); + } + + noteGuidChanged(info) { + lazy.PlacesUtils.invalidateCachedGuidFor(info.id); + this.placesEvents.push( + new PlacesBookmarkGuid({ + id: info.id, + itemType: info.type, + url: info.urlHref, + guid: info.newGuid, + parentGuid: info.parentGuid, + lastModified: info.lastModified, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + isTagging: + info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid || + info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid, + }) + ); + } + + noteItemMoved(info) { + this.placesEvents.push( + new PlacesBookmarkMoved({ + id: info.id, + itemType: info.type, + url: info.urlHref, + guid: info.guid, + parentGuid: info.newParentGuid, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + index: info.newPosition, + oldParentGuid: info.oldParentGuid, + oldIndex: info.oldPosition, + isTagging: + info.newParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid || + info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid, + }) + ); + } + + noteItemChanged(info) { + if (info.oldTitle != info.newTitle) { + this.placesEvents.push( + new PlacesBookmarkTitle({ + id: info.id, + itemType: info.type, + url: info.urlHref, + guid: info.guid, + parentGuid: info.parentGuid, + title: info.newTitle, + lastModified: info.lastModified, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + isTagging: + info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid || + info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid, + }) + ); + } + if (info.oldURLHref != info.newURLHref) { + this.placesEvents.push( + new PlacesBookmarkUrl({ + id: info.id, + itemType: info.type, + url: info.newURLHref, + guid: info.guid, + parentGuid: info.parentGuid, + lastModified: info.lastModified, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + isTagging: + info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid || + info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid, + }) + ); + } + } + + noteItemRemoved(info) { + this.placesEvents.push( + new PlacesBookmarkRemoved({ + id: info.id, + parentId: info.parentId, + index: info.position, + url: info.urlHref || "", + title: info.title, + guid: info.guid, + parentGuid: info.parentGuid, + source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + itemType: info.type, + isTagging: info.isUntagging, + isDescendantRemoval: false, + }) + ); + } + + notifyBookmarkObservers() { + lazy.MirrorLog.trace("Notifying bookmark observers"); + + if (this.placesEvents.length) { + PlacesObservers.notifyListeners(this.placesEvents); + } + + lazy.MirrorLog.trace("Notified bookmark observers"); + } +} + +/** + * Holds Sync metadata and the cleartext for a locally changed record. The + * bookmarks engine inflates a Sync record from the cleartext, and updates the + * `synced` property for successfully uploaded items. + * + * At the end of the sync, the engine writes the uploaded cleartext back to the + * mirror, and passes the updated change record as part of the changeset to + * `PlacesSyncUtils.bookmarks.pushChanges`. + */ +class BookmarkChangeRecord { + constructor(syncChangeCounter, cleartext) { + this.tombstone = cleartext.deleted === true; + this.counter = syncChangeCounter; + this.cleartext = cleartext; + this.synced = false; + } +} + +function bagToNamedCounts(bag, names) { + let counts = []; + for (let name of names) { + let count = bag.getProperty(name); + if (count > 0) { + counts.push({ name, count }); + } + } + return counts; +} + +/** + * Returns an `AbortSignal` that aborts if either `finalizeSignal` or + * `interruptSignal` aborts. This is like `Promise.race`, but for + * cancellations. + * + * @param {AbortSignal} finalizeSignal + * @param {AbortSignal?} signal + * @return {AbortSignal} + */ +function anyAborted(finalizeSignal, interruptSignal = null) { + if (finalizeSignal.aborted || !interruptSignal) { + // If the mirror was already finalized, or we don't have an interrupt + // signal for this merge, just use the finalize signal. + return finalizeSignal; + } + if (interruptSignal.aborted) { + // If the merge was interrupted, return its already-aborted signal. + return interruptSignal; + } + // Otherwise, we return a new signal that aborts if either the mirror is + // finalized, or the merge is interrupted, whichever happens first. + let controller = new AbortController(); + function onAbort() { + finalizeSignal.removeEventListener("abort", onAbort); + interruptSignal.removeEventListener("abort", onAbort); + controller.abort(); + } + finalizeSignal.addEventListener("abort", onAbort); + interruptSignal.addEventListener("abort", onAbort); + return controller.signal; +} + +// In conclusion, this is why bookmark syncing is hard. diff --git a/toolkit/components/places/TaggingService.sys.mjs b/toolkit/components/places/TaggingService.sys.mjs new file mode 100644 index 0000000000..e27c84f844 --- /dev/null +++ b/toolkit/components/places/TaggingService.sys.mjs @@ -0,0 +1,565 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs"; + +const TOPIC_SHUTDOWN = "places-shutdown"; + +/** + * The Places Tagging Service + */ +export function TaggingService() { + this.handlePlacesEvents = this.handlePlacesEvents.bind(this); + + // Observe bookmarks changes. + PlacesUtils.observers.addListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-title-changed", + ], + this.handlePlacesEvents + ); + + // Cleanup on shutdown. + Services.obs.addObserver(this, TOPIC_SHUTDOWN); +} + +TaggingService.prototype = { + /** + * Creates a tag container under the tags-root with the given name. + * + * @param aTagName + * the name for the new tag. + * @param aSource + * a change source constant from nsINavBookmarksService::SOURCE_*. + * @returns the id of the new tag container. + */ + _createTag: function TS__createTag(aTagName, aSource) { + var newFolderId = PlacesUtils.bookmarks.createFolder( + PlacesUtils.tagsFolderId, + aTagName, + PlacesUtils.bookmarks.DEFAULT_INDEX, + /* aGuid */ null, + aSource + ); + // Add the folder to our local cache, so we can avoid doing this in the + // observer that would have to check itemType. + this._tagFolders[newFolderId] = aTagName; + + return newFolderId; + }, + + /** + * Checks whether the given uri is tagged with the given tag. + * + * @param [in] aURI + * url to check for + * @param [in] aTagName + * the tag to check for + * @returns the item id if the URI is tagged with the given tag, -1 + * otherwise. + */ + _getItemIdForTaggedURI: function TS__getItemIdForTaggedURI(aURI, aTagName) { + var tagId = this._getItemIdForTag(aTagName); + if (tagId == -1) { + return -1; + } + // Using bookmarks service API for this would be a pain. + // Until tags implementation becomes sane, go the query way. + let db = PlacesUtils.history.DBConnection; + let stmt = db.createStatement( + `SELECT id FROM moz_bookmarks + WHERE parent = :tag_id + AND fk = (SELECT id FROM moz_places WHERE url_hash = hash(:page_url) AND url = :page_url)` + ); + stmt.params.tag_id = tagId; + stmt.params.page_url = aURI.spec; + try { + if (stmt.executeStep()) { + return stmt.row.id; + } + } finally { + stmt.finalize(); + } + return -1; + }, + + /** + * Returns the folder id for a tag, or -1 if not found. + * @param [in] aTag + * string tag to search for + * @returns integer id for the bookmark folder for the tag + */ + _getItemIdForTag: function TS_getItemIdForTag(aTagName) { + for (var i in this._tagFolders) { + if (aTagName.toLowerCase() == this._tagFolders[i].toLowerCase()) { + return parseInt(i); + } + } + return -1; + }, + /** + * Makes a proper array of tag objects like { id: number, name: string }. + * + * @param aTags + * Array of tags. Entries can be tag names or concrete item id. + * @param trim [optional] + * Whether to trim passed-in named tags. Defaults to false. + * @return Array of tag objects like { id: number, name: string }. + * + * @throws Cr.NS_ERROR_INVALID_ARG if any element of the input array is not + * a valid tag. + */ + _convertInputMixedTagsArray(aTags, trim = false) { + // Handle sparse array with a .filter. + return aTags + .filter(tag => tag !== undefined) + .map(idOrName => { + let tag = {}; + if (typeof idOrName == "number" && this._tagFolders[idOrName]) { + // This is a tag folder id. + tag.id = idOrName; + // We can't know the name at this point, since a previous tag could + // want to change it. + tag.__defineGetter__("name", () => this._tagFolders[tag.id]); + } else if ( + typeof idOrName == "string" && + !!idOrName.length && + idOrName.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH + ) { + // This is a tag name. + tag.name = trim ? idOrName.trim() : idOrName; + // We can't know the id at this point, since a previous tag could + // have created it. + tag.__defineGetter__("id", () => this._getItemIdForTag(tag.name)); + } else { + throw Components.Exception( + "Invalid tag value", + Cr.NS_ERROR_INVALID_ARG + ); + } + return tag; + }); + }, + + // nsITaggingService + tagURI: function TS_tagURI(aURI, aTags, aSource) { + if (!aURI || !aTags || !Array.isArray(aTags) || !aTags.length) { + throw Components.Exception( + "Invalid value for tags", + Cr.NS_ERROR_INVALID_ARG + ); + } + + // This also does some input validation. + let tags = this._convertInputMixedTagsArray(aTags, true); + + for (let tag of tags) { + if (tag.id == -1) { + // Tag does not exist yet, create it. + this._createTag(tag.name, aSource); + } + + let itemId = this._getItemIdForTaggedURI(aURI, tag.name); + if (itemId == -1) { + // The provided URI is not yet tagged, add a tag for it. + // Note that bookmarks under tag containers must have null titles. + PlacesUtils.bookmarks.insertBookmark( + tag.id, + aURI, + PlacesUtils.bookmarks.DEFAULT_INDEX, + /* aTitle */ null, + /* aGuid */ null, + aSource + ); + } else { + // Otherwise, bump the tag's timestamp, so that we can increment the + // sync change counter for all bookmarks with the URI. + PlacesUtils.bookmarks.setItemLastModified( + itemId, + PlacesUtils.toPRTime(Date.now()), + aSource + ); + } + + // Try to preserve user's tag name casing. + // Rename the tag container so the Places view matches the most-recent + // user-typed value. + if (PlacesUtils.bookmarks.getItemTitle(tag.id) != tag.name) { + // this._tagFolders is updated by the bookmarks observer. + PlacesUtils.bookmarks.setItemTitle(tag.id, tag.name, aSource); + } + } + }, + + /** + * Removes the tag container from the tags root if the given tag is empty. + * + * @param aTagId + * the itemId of the tag element under the tags root + * @param aSource + * a change source constant from nsINavBookmarksService::SOURCE_* + */ + _removeTagIfEmpty: function TS__removeTagIfEmpty(aTagId, aSource) { + let count = 0; + let db = PlacesUtils.history.DBConnection; + let stmt = db.createStatement( + `SELECT count(*) AS count FROM moz_bookmarks + WHERE parent = :tag_id` + ); + stmt.params.tag_id = aTagId; + try { + if (stmt.executeStep()) { + count = stmt.row.count; + } + } finally { + stmt.finalize(); + } + + if (count == 0) { + PlacesUtils.bookmarks.removeItem(aTagId, aSource); + } + }, + + // nsITaggingService + untagURI: function TS_untagURI(aURI, aTags, aSource) { + if (!aURI || (aTags && (!Array.isArray(aTags) || !aTags.length))) { + throw Components.Exception( + "Invalid value for tags", + Cr.NS_ERROR_INVALID_ARG + ); + } + + if (!aTags) { + // Passing null should clear all tags for aURI, see the IDL. + // XXXmano: write a perf-sensitive version of this code path... + aTags = this.getTagsForURI(aURI); + } + + // This also does some input validation. + let tags = this._convertInputMixedTagsArray(aTags); + + let isAnyTagNotTrimmed = tags.some(tag => /^\s|\s$/.test(tag.name)); + if (isAnyTagNotTrimmed) { + throw Components.Exception( + "At least one tag passed to untagURI was not trimmed", + Cr.NS_ERROR_INVALID_ARG + ); + } + + for (let tag of tags) { + if (tag.id != -1) { + // A tag could exist. + let itemId = this._getItemIdForTaggedURI(aURI, tag.name); + if (itemId != -1) { + // There is a tagged item. + PlacesUtils.bookmarks.removeItem(itemId, aSource); + } + } + } + }, + + // nsITaggingService + getTagsForURI: function TS_getTagsForURI(aURI) { + if (!aURI) { + throw Components.Exception("Invalid uri", Cr.NS_ERROR_INVALID_ARG); + } + + let tags = []; + let db = PlacesUtils.history.DBConnection; + let stmt = db.createStatement( + `SELECT t.id AS folderId + FROM moz_bookmarks b + JOIN moz_bookmarks t on t.id = b.parent + WHERE b.fk = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url) AND + t.parent = :tags_root + ORDER BY b.lastModified DESC, b.id DESC` + ); + stmt.params.url = aURI.spec; + stmt.params.tags_root = PlacesUtils.tagsFolderId; + try { + while (stmt.executeStep()) { + try { + tags.push(this._tagFolders[stmt.row.folderId]); + } catch (ex) {} + } + } finally { + stmt.finalize(); + } + + // sort the tag list + tags.sort(function (a, b) { + return a.toLowerCase().localeCompare(b.toLowerCase()); + }); + return tags; + }, + + __tagFolders: null, + get _tagFolders() { + if (!this.__tagFolders) { + this.__tagFolders = []; + + let db = PlacesUtils.history.DBConnection; + let stmt = db.createStatement( + "SELECT id, title FROM moz_bookmarks WHERE parent = :tags_root " + ); + stmt.params.tags_root = PlacesUtils.tagsFolderId; + try { + while (stmt.executeStep()) { + this.__tagFolders[stmt.row.id] = stmt.row.title; + } + } finally { + stmt.finalize(); + } + } + + return this.__tagFolders; + }, + + // nsIObserver + observe: function TS_observe(aSubject, aTopic, aData) { + if (aTopic == TOPIC_SHUTDOWN) { + PlacesUtils.observers.removeListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-title-changed", + ], + this.handlePlacesEvents + ); + Services.obs.removeObserver(this, TOPIC_SHUTDOWN); + } + }, + + /** + * If the only bookmark items associated with aURI are contained in tag + * folders, returns the IDs of those items. This can be the case if + * the URI was bookmarked and tagged at some point, but the bookmark was + * removed, leaving only the bookmark items in tag folders. If the URI is + * either properly bookmarked or not tagged just returns and empty array. + * + * @param aURI + * A URI (string) that may or may not be bookmarked + * @returns an array of item ids + */ + _getTaggedItemIdsIfUnbookmarkedURI: + function TS__getTaggedItemIdsIfUnbookmarkedURI(url) { + var itemIds = []; + var isBookmarked = false; + + // Using bookmarks service API for this would be a pain. + // Until tags implementation becomes sane, go the query way. + let db = PlacesUtils.history.DBConnection; + let stmt = db.createStatement( + `SELECT id, parent + FROM moz_bookmarks + WHERE fk = (SELECT id FROM moz_places WHERE url_hash = hash(:page_url) AND url = :page_url)` + ); + stmt.params.page_url = url; + try { + while (stmt.executeStep() && !isBookmarked) { + if (this._tagFolders[stmt.row.parent]) { + // This is a tag entry. + itemIds.push(stmt.row.id); + } else { + // This is a real bookmark, so the bookmarked URI is not an orphan. + isBookmarked = true; + } + } + } finally { + stmt.finalize(); + } + + return isBookmarked ? [] : itemIds; + }, + + handlePlacesEvents(events) { + for (let event of events) { + switch (event.type) { + case "bookmark-added": + if ( + !event.isTagging || + event.itemType != PlacesUtils.bookmarks.TYPE_FOLDER + ) { + continue; + } + + this._tagFolders[event.id] = event.title; + break; + case "bookmark-removed": + // Item is a tag folder. + if ( + event.parentId == PlacesUtils.tagsFolderId && + this._tagFolders[event.id] + ) { + delete this._tagFolders[event.id]; + break; + } + + Services.tm.dispatchToMainThread(() => { + if (event.url && !this._tagFolders[event.parentId]) { + // Item is a bookmark that was removed from a non-tag folder. + // If the only bookmark items now associated with the bookmark's URI are + // contained in tag folders, the URI is no longer properly bookmarked, so + // untag it. + let itemIds = this._getTaggedItemIdsIfUnbookmarkedURI(event.url); + for (let i = 0; i < itemIds.length; i++) { + try { + PlacesUtils.bookmarks.removeItem(itemIds[i], event.source); + } catch (ex) {} + } + } else if (event.url && this._tagFolders[event.parentId]) { + // Item is a tag entry. If this was the last entry for this tag, remove it. + this._removeTagIfEmpty(event.parentId, event.source); + } + }); + break; + case "bookmark-moved": + if ( + this._tagFolders[event.id] && + PlacesUtils.bookmarks.tagsGuid === event.oldParentGuid && + PlacesUtils.bookmarks.tagsGuid !== event.parentGuid + ) { + delete this._tagFolders[event.id]; + } + break; + case "bookmark-title-changed": + if (this._tagFolders[event.id]) { + this._tagFolders[event.id] = event.title; + } + break; + } + } + }, + + // nsISupports + + classID: Components.ID("{bbc23860-2553-479d-8b78-94d9038334f7}"), + + QueryInterface: ChromeUtils.generateQI(["nsITaggingService", "nsIObserver"]), +}; + +/** + * Class tracking a single tag autocomplete search. + */ +class TagSearch { + constructor(searchString, autocompleteSearch, listener) { + // We need a result regardless of having matches. + this._result = Cc[ + "@mozilla.org/autocomplete/simple-result;1" + ].createInstance(Ci.nsIAutoCompleteSimpleResult); + this._result.setDefaultIndex(0); + this._result.setSearchString(searchString); + + this._autocompleteSearch = autocompleteSearch; + this._listener = listener; + } + + async start() { + if (this._canceled) { + throw new Error("Can't restart a canceled search"); + } + + let searchString = this._result.searchString; + // Only search on characters for the last tag. + let index = Math.max( + searchString.lastIndexOf(","), + searchString.lastIndexOf(";") + ); + let before = ""; + if (index != -1) { + before = searchString.slice(0, index + 1); + searchString = searchString.slice(index + 1); + // skip past whitespace + var m = searchString.match(/\s+/); + if (m) { + before += m[0]; + searchString = searchString.slice(m[0].length); + } + } + + if (searchString.length) { + let tags = await PlacesUtils.bookmarks.fetchTags(); + if (this._canceled) { + return; + } + + let lcSearchString = searchString.toLowerCase(); + let matchingTags = tags + .filter(t => t.name.toLowerCase().startsWith(lcSearchString)) + .map(t => t.name); + + for (let i = 0; i < matchingTags.length; ++i) { + let tag = matchingTags[i]; + // For each match, prepend what the user has typed so far. + this._result.appendMatch(before + tag, tag); + // In case of many tags, notify once every 10. + if (i % 10 == 0) { + this._notifyResult(true); + // yield to avoid monopolizing the main-thread + await new Promise(resolve => + Services.tm.dispatchToMainThread(resolve) + ); + if (this._canceled) { + return; + } + } + } + } + + // Search is done. + this._notifyResult(false); + } + + cancel() { + this._canceled = true; + } + + _notifyResult(searchOngoing) { + let resultCode = this._result.matchCount + ? "RESULT_SUCCESS" + : "RESULT_NOMATCH"; + if (searchOngoing) { + resultCode += "_ONGOING"; + } + this._result.setSearchResult(Ci.nsIAutoCompleteResult[resultCode]); + this._listener.onSearchResult(this._autocompleteSearch, this._result); + } +} + +// Implements nsIAutoCompleteSearch +export function TagAutoCompleteSearch() {} + +TagAutoCompleteSearch.prototype = { + /* + * Search for a given string and notify a listener of the result. + * + * @param searchString - The string to search for + * @param searchParam - An extra parameter + * @param previousResult - A previous result to use for faster searching + * @param listener - A listener to notify when the search is complete + */ + startSearch(searchString, searchParam, previousResult, listener) { + if (this._search) { + this._search.cancel(); + } + this._search = new TagSearch(searchString, this, listener); + this._search.start().catch(console.error); + }, + + /** + * Stop an asynchronous search that is in progress + */ + stopSearch() { + this._search.cancel(); + this._search = null; + }, + + classID: Components.ID("{1dcc23b0-d4cb-11dc-9ad6-479d56d89593}"), + QueryInterface: ChromeUtils.generateQI(["nsIAutoCompleteSearch"]), +}; diff --git a/toolkit/components/places/VisitInfo.cpp b/toolkit/components/places/VisitInfo.cpp new file mode 100644 index 0000000000..c957c13a6b --- /dev/null +++ b/toolkit/components/places/VisitInfo.cpp @@ -0,0 +1,57 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VisitInfo.h" +#include "nsIURI.h" + +namespace mozilla { +namespace places { + +//////////////////////////////////////////////////////////////////////////////// +//// VisitInfo + +VisitInfo::VisitInfo(int64_t aVisitId, PRTime aVisitDate, + uint32_t aTransitionType, + already_AddRefed aReferrer) + : mVisitId(aVisitId), + mVisitDate(aVisitDate), + mTransitionType(aTransitionType), + mReferrer(aReferrer) {} + +VisitInfo::~VisitInfo() = default; + +//////////////////////////////////////////////////////////////////////////////// +//// mozIVisitInfo + +NS_IMETHODIMP +VisitInfo::GetVisitId(int64_t* _visitId) { + *_visitId = mVisitId; + return NS_OK; +} + +NS_IMETHODIMP +VisitInfo::GetVisitDate(PRTime* _visitDate) { + *_visitDate = mVisitDate; + return NS_OK; +} + +NS_IMETHODIMP +VisitInfo::GetTransitionType(uint32_t* _transitionType) { + *_transitionType = mTransitionType; + return NS_OK; +} + +NS_IMETHODIMP +VisitInfo::GetReferrerURI(nsIURI** _referrer) { + NS_IF_ADDREF(*_referrer = mReferrer); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsISupports + +NS_IMPL_ISUPPORTS(VisitInfo, mozIVisitInfo) + +} // namespace places +} // namespace mozilla diff --git a/toolkit/components/places/VisitInfo.h b/toolkit/components/places/VisitInfo.h new file mode 100644 index 0000000000..7b99cd46a6 --- /dev/null +++ b/toolkit/components/places/VisitInfo.h @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_places_VisitInfo_h__ +#define mozilla_places_VisitInfo_h__ + +#include "mozIAsyncHistory.h" +#include "mozilla/Attributes.h" +#include "nsCOMPtr.h" + +class nsIURI; + +namespace mozilla { +namespace places { + +class VisitInfo final : public mozIVisitInfo { + public: + NS_DECL_ISUPPORTS + NS_DECL_MOZIVISITINFO + + VisitInfo(int64_t aVisitId, PRTime aVisitDate, uint32_t aTransitionType, + already_AddRefed aReferrer); + + private: + ~VisitInfo(); + const int64_t mVisitId; + const PRTime mVisitDate; + const uint32_t mTransitionType; + nsCOMPtr mReferrer; +}; + +} // namespace places +} // namespace mozilla + +#endif // mozilla_places_VisitInfo_h__ diff --git a/toolkit/components/places/bookmark_sync/Cargo.toml b/toolkit/components/places/bookmark_sync/Cargo.toml new file mode 100644 index 0000000000..71cd764c4b --- /dev/null +++ b/toolkit/components/places/bookmark_sync/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bookmark_sync" +version = "0.1.0" +authors = ["Lina Cambridge "] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +atomic_refcell = "0.1" +dogear = "0.5.0" +libc = "0.2" +log = "0.4" +cstr = "0.2" +moz_task = { path = "../../../../xpcom/rust/moz_task" } +nserror = { path = "../../../../xpcom/rust/nserror" } +nsstring = { path = "../../../../xpcom/rust/nsstring" } +storage = { path = "../../../../storage/rust" } +storage_variant = { path = "../../../../storage/variant" } +url = "2.0" +xpcom = { path = "../../../../xpcom/rust/xpcom" } diff --git a/toolkit/components/places/bookmark_sync/src/driver.rs b/toolkit/components/places/bookmark_sync/src/driver.rs new file mode 100644 index 0000000000..43bb1f1f79 --- /dev/null +++ b/toolkit/components/places/bookmark_sync/src/driver.rs @@ -0,0 +1,259 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{ + fmt::Write, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +use dogear::{AbortSignal, Guid, ProblemCounts, StructureCounts, TelemetryEvent}; +use log::{Level, LevelFilter, Log, Metadata, Record}; +use moz_task::{Task, TaskRunnable, ThreadPtrHandle}; +use nserror::nsresult; +use nsstring::{nsACString, nsCString, nsString}; +use storage_variant::HashPropertyBag; +use xpcom::interfaces::{mozIServicesLogSink, mozISyncedBookmarksMirrorProgressListener}; + +extern "C" { + fn NS_GeneratePlacesGUID(guid: *mut nsACString) -> nsresult; +} + +fn generate_guid() -> Result { + let mut guid = nsCString::new(); + let rv = unsafe { NS_GeneratePlacesGUID(&mut *guid) }; + if rv.succeeded() { + Ok(guid) + } else { + Err(rv) + } +} + +/// An abort controller is used to abort merges running on the storage thread +/// from the main thread. Its design is based on the DOM API of the same name. +pub struct AbortController { + aborted: AtomicBool, +} + +impl AbortController { + /// Signals the store to stop merging as soon as it can. + pub fn abort(&self) { + self.aborted.store(true, Ordering::Release) + } +} + +impl Default for AbortController { + fn default() -> AbortController { + AbortController { + aborted: AtomicBool::new(false), + } + } +} + +impl AbortSignal for AbortController { + fn aborted(&self) -> bool { + self.aborted.load(Ordering::Acquire) + } +} + +/// The merger driver, created and used on the storage thread. +pub struct Driver { + log: Logger, + progress: Option>, +} + +impl Driver { + #[inline] + pub fn new( + log: Logger, + progress: Option>, + ) -> Driver { + Driver { log, progress } + } +} + +impl dogear::Driver for Driver { + fn generate_new_guid(&self, invalid_guid: &Guid) -> dogear::Result { + generate_guid() + .map_err(|_| dogear::ErrorKind::InvalidGuid(invalid_guid.clone()).into()) + .and_then(|s| Guid::from_utf8(s.as_ref())) + } + + #[inline] + fn max_log_level(&self) -> LevelFilter { + self.log.max_level + } + + #[inline] + fn logger(&self) -> &dyn Log { + &self.log + } + + fn record_telemetry_event(&self, event: TelemetryEvent) { + if let Some(ref progress) = self.progress { + let task = RecordTelemetryEventTask { + progress: progress.clone(), + event, + }; + let _ = TaskRunnable::new( + "bookmark_sync::Driver::record_telemetry_event", + Box::new(task), + ) + .and_then(|r| TaskRunnable::dispatch(r, progress.owning_thread())); + } + } +} + +pub struct Logger { + pub max_level: LevelFilter, + logger: Option>, +} + +impl Logger { + #[inline] + pub fn new( + max_level: LevelFilter, + logger: Option>, + ) -> Logger { + Logger { max_level, logger } + } +} + +impl Log for Logger { + #[inline] + fn enabled(&self, meta: &Metadata) -> bool { + self.logger.is_some() && meta.level() <= self.max_level + } + + fn log(&self, record: &Record) { + if !self.enabled(record.metadata()) { + return; + } + if let Some(logger) = &self.logger { + let mut message = nsString::new(); + match write!(message, "{}", record.args()) { + Ok(_) => { + let task = LogTask { + logger: logger.clone(), + level: record.metadata().level(), + message, + }; + let _ = TaskRunnable::new("bookmark_sync::Logger::log", Box::new(task)) + .and_then(|r| TaskRunnable::dispatch(r, logger.owning_thread())); + } + Err(_) => {} + } + } + } + + fn flush(&self) {} +} + +/// Logs a message to the mirror logger. This task is created on the async +/// thread, and dispatched to the main thread. +struct LogTask { + logger: ThreadPtrHandle, + level: Level, + message: nsString, +} + +impl Task for LogTask { + fn run(&self) { + let logger = self.logger.get().unwrap(); + match self.level { + Level::Error => unsafe { + logger.Error(&*self.message); + }, + Level::Warn => unsafe { + logger.Warn(&*self.message); + }, + Level::Debug => unsafe { + logger.Debug(&*self.message); + }, + Level::Trace => unsafe { + logger.Trace(&*self.message); + }, + _ => {} + } + } + + fn done(&self) -> Result<(), nsresult> { + Ok(()) + } +} + +/// Calls a progress listener callback for a merge telemetry event. This task is +/// created on the async thread, and dispatched to the main thread. +struct RecordTelemetryEventTask { + progress: ThreadPtrHandle, + event: TelemetryEvent, +} + +impl Task for RecordTelemetryEventTask { + fn run(&self) { + let callback = self.progress.get().unwrap(); + let _ = match &self.event { + TelemetryEvent::FetchLocalTree(stats) => unsafe { + callback.OnFetchLocalTree( + as_millis(stats.time), + stats.items as i64, + stats.deletions as i64, + problem_counts_to_bag(&stats.problems).bag().coerce(), + ) + }, + TelemetryEvent::FetchRemoteTree(stats) => unsafe { + callback.OnFetchRemoteTree( + as_millis(stats.time), + stats.items as i64, + stats.deletions as i64, + problem_counts_to_bag(&stats.problems).bag().coerce(), + ) + }, + TelemetryEvent::Merge(time, counts) => unsafe { + callback.OnMerge( + as_millis(*time), + structure_counts_to_bag(counts).bag().coerce(), + ) + }, + TelemetryEvent::Apply(time) => unsafe { callback.OnApply(as_millis(*time)) }, + }; + } + + fn done(&self) -> std::result::Result<(), nsresult> { + Ok(()) + } +} + +fn as_millis(d: Duration) -> i64 { + d.as_secs() as i64 * 1000 + i64::from(d.subsec_millis()) +} + +fn problem_counts_to_bag(problems: &ProblemCounts) -> HashPropertyBag { + let mut bag = HashPropertyBag::new(); + bag.set("orphans", problems.orphans as i64); + bag.set("misparentedRoots", problems.misparented_roots as i64); + bag.set( + "multipleParents", + problems.multiple_parents_by_children as i64, + ); + bag.set("missingParents", problems.missing_parent_guids as i64); + bag.set("nonFolderParents", problems.non_folder_parent_guids as i64); + bag.set( + "parentChildDisagreements", + problems.parent_child_disagreements as i64, + ); + bag.set("missingChildren", problems.missing_children as i64); + bag +} + +fn structure_counts_to_bag(counts: &StructureCounts) -> HashPropertyBag { + let mut bag = HashPropertyBag::new(); + bag.set("remoteRevives", counts.remote_revives as i64); + bag.set("localDeletes", counts.local_deletes as i64); + bag.set("localRevives", counts.local_revives as i64); + bag.set("remoteDeletes", counts.remote_deletes as i64); + bag.set("dupes", counts.dupes as i64); + bag.set("items", counts.merged_nodes as i64); + bag +} diff --git a/toolkit/components/places/bookmark_sync/src/error.rs b/toolkit/components/places/bookmark_sync/src/error.rs new file mode 100644 index 0000000000..5ac7c5b62c --- /dev/null +++ b/toolkit/components/places/bookmark_sync/src/error.rs @@ -0,0 +1,106 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{error, fmt, result, string::FromUtf16Error}; + +use nserror::{ + nsresult, NS_ERROR_ABORT, NS_ERROR_FAILURE, NS_ERROR_INVALID_ARG, NS_ERROR_STORAGE_BUSY, + NS_ERROR_UNEXPECTED, +}; + +pub type Result = result::Result; + +#[derive(Debug)] +pub enum Error { + Dogear(dogear::Error), + Storage(storage::Error), + InvalidLocalRoots, + InvalidRemoteRoots, + Nsresult(nsresult), + UnknownItemType(i64), + UnknownItemKind(i64), + MalformedString(Box), + MergeConflict, + StorageBusy, + UnknownItemValidity(i64), + DidNotRun, +} + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + Error::Dogear(err) => Some(err), + Error::Storage(err) => Some(err), + _ => None, + } + } +} + +impl From for Error { + fn from(err: dogear::Error) -> Error { + Error::Dogear(err) + } +} + +impl From for Error { + fn from(err: storage::Error) -> Error { + Error::Storage(err) + } +} + +impl From for Error { + fn from(result: nsresult) -> Error { + Error::Nsresult(result) + } +} + +impl From for Error { + fn from(error: FromUtf16Error) -> Error { + Error::MalformedString(error.into()) + } +} + +impl From for nsresult { + fn from(error: Error) -> nsresult { + match error { + Error::Dogear(err) => match err.kind() { + dogear::ErrorKind::Abort => NS_ERROR_ABORT, + _ => NS_ERROR_FAILURE, + }, + Error::InvalidLocalRoots | Error::InvalidRemoteRoots | Error::DidNotRun => { + NS_ERROR_UNEXPECTED + } + Error::Storage(err) => err.into(), + Error::Nsresult(result) => result.clone(), + Error::UnknownItemType(_) + | Error::UnknownItemKind(_) + | Error::MalformedString(_) + | Error::UnknownItemValidity(_) => NS_ERROR_INVALID_ARG, + Error::MergeConflict | Error::StorageBusy => NS_ERROR_STORAGE_BUSY, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::Dogear(err) => err.fmt(f), + Error::Storage(err) => err.fmt(f), + Error::InvalidLocalRoots => f.write_str("The Places roots are invalid"), + Error::InvalidRemoteRoots => { + f.write_str("The roots in the mirror database are invalid") + } + Error::Nsresult(result) => write!(f, "Operation failed with {}", result.error_name()), + Error::UnknownItemType(typ) => write!(f, "Unknown item type {} in Places", typ), + Error::UnknownItemKind(kind) => write!(f, "Unknown item kind {} in mirror", kind), + Error::MalformedString(err) => err.fmt(f), + Error::MergeConflict => f.write_str("Local tree changed during merge"), + Error::StorageBusy => f.write_str("The database is busy"), + Error::UnknownItemValidity(validity) => { + write!(f, "Unknown item validity {} in database", validity) + } + Error::DidNotRun => write!(f, "Failed to run merge on storage thread"), + } + } +} diff --git a/toolkit/components/places/bookmark_sync/src/lib.rs b/toolkit/components/places/bookmark_sync/src/lib.rs new file mode 100644 index 0000000000..a0dadd51c2 --- /dev/null +++ b/toolkit/components/places/bookmark_sync/src/lib.rs @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![allow(non_snake_case)] + +#[macro_use] +extern crate cstr; +#[macro_use] +extern crate xpcom; + +mod driver; +mod error; +mod merger; +mod store; + +use xpcom::{interfaces::mozISyncedBookmarksMerger, RefPtr}; + +use crate::merger::SyncedBookmarksMerger; + +#[no_mangle] +pub unsafe extern "C" fn NS_NewSyncedBookmarksMerger( + result: *mut *const mozISyncedBookmarksMerger, +) { + let merger = SyncedBookmarksMerger::new(); + RefPtr::new(merger.coerce::()).forget(&mut *result); +} diff --git a/toolkit/components/places/bookmark_sync/src/merger.rs b/toolkit/components/places/bookmark_sync/src/merger.rs new file mode 100644 index 0000000000..ca0fc17a75 --- /dev/null +++ b/toolkit/components/places/bookmark_sync/src/merger.rs @@ -0,0 +1,237 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{cell::RefCell, fmt::Write, mem, sync::Arc}; + +use atomic_refcell::AtomicRefCell; +use dogear::Store; +use log::LevelFilter; +use moz_task::{Task, TaskRunnable, ThreadPtrHandle, ThreadPtrHolder}; +use nserror::{nsresult, NS_ERROR_NOT_AVAILABLE, NS_OK}; +use nsstring::nsString; +use storage::Conn; +use xpcom::{ + interfaces::{ + mozIPlacesPendingOperation, mozIServicesLogSink, mozIStorageConnection, + mozISyncedBookmarksMirrorCallback, mozISyncedBookmarksMirrorProgressListener, + }, + RefPtr, XpCom, +}; + +use crate::driver::{AbortController, Driver, Logger}; +use crate::error; +use crate::store; + +#[xpcom(implement(mozISyncedBookmarksMerger), nonatomic)] +pub struct SyncedBookmarksMerger { + db: RefCell>, + logger: RefCell>>, +} + +impl SyncedBookmarksMerger { + pub fn new() -> RefPtr { + SyncedBookmarksMerger::allocate(InitSyncedBookmarksMerger { + db: RefCell::default(), + logger: RefCell::default(), + }) + } + + xpcom_method!(get_db => GetDb() -> *const mozIStorageConnection); + fn get_db(&self) -> Result, nsresult> { + self.db + .borrow() + .as_ref() + .map(|db| RefPtr::new(db.connection())) + .ok_or(NS_OK) + } + + xpcom_method!(set_db => SetDb(connection: *const mozIStorageConnection)); + fn set_db(&self, connection: Option<&mozIStorageConnection>) -> Result<(), nsresult> { + self.db + .replace(connection.map(|connection| Conn::wrap(RefPtr::new(connection)))); + Ok(()) + } + + xpcom_method!(get_logger => GetLogger() -> *const mozIServicesLogSink); + fn get_logger(&self) -> Result, nsresult> { + match *self.logger.borrow() { + Some(ref logger) => Ok(logger.clone()), + None => Err(NS_OK), + } + } + + xpcom_method!(set_logger => SetLogger(logger: *const mozIServicesLogSink)); + fn set_logger(&self, logger: Option<&mozIServicesLogSink>) -> Result<(), nsresult> { + self.logger.replace(logger.map(RefPtr::new)); + Ok(()) + } + + xpcom_method!( + merge => Merge( + local_time_seconds: i64, + remote_time_seconds: i64, + callback: *const mozISyncedBookmarksMirrorCallback + ) -> *const mozIPlacesPendingOperation + ); + fn merge( + &self, + local_time_seconds: i64, + remote_time_seconds: i64, + callback: &mozISyncedBookmarksMirrorCallback, + ) -> Result, nsresult> { + let callback = RefPtr::new(callback); + let db = match *self.db.borrow() { + Some(ref db) => db.clone(), + None => return Err(NS_ERROR_NOT_AVAILABLE), + }; + let logger = &*self.logger.borrow(); + let async_thread = db.thread()?; + let controller = Arc::new(AbortController::default()); + let task = MergeTask::new( + &db, + Arc::clone(&controller), + logger.as_ref().cloned(), + local_time_seconds, + remote_time_seconds, + callback, + )?; + let runnable = TaskRunnable::new( + "bookmark_sync::SyncedBookmarksMerger::merge", + Box::new(task), + )?; + TaskRunnable::dispatch(runnable, &async_thread)?; + let op = MergeOp::new(controller); + Ok(RefPtr::new(op.coerce())) + } + + xpcom_method!(reset => Reset()); + fn reset(&self) -> Result<(), nsresult> { + mem::drop(self.db.borrow_mut().take()); + mem::drop(self.logger.borrow_mut().take()); + Ok(()) + } +} + +struct MergeTask { + db: Conn, + controller: Arc, + max_log_level: LevelFilter, + logger: Option>, + local_time_millis: i64, + remote_time_millis: i64, + progress: Option>, + callback: ThreadPtrHandle, + result: AtomicRefCell>, +} + +impl MergeTask { + fn new( + db: &Conn, + controller: Arc, + logger: Option>, + local_time_seconds: i64, + remote_time_seconds: i64, + callback: RefPtr, + ) -> Result { + let max_log_level = logger + .as_ref() + .and_then(|logger| { + let mut level = 0i16; + unsafe { logger.GetMaxLevel(&mut level) }.to_result().ok()?; + Some(level) + }) + .map(|level| match level { + mozIServicesLogSink::LEVEL_ERROR => LevelFilter::Error, + mozIServicesLogSink::LEVEL_WARN => LevelFilter::Warn, + mozIServicesLogSink::LEVEL_DEBUG => LevelFilter::Debug, + mozIServicesLogSink::LEVEL_TRACE => LevelFilter::Trace, + _ => LevelFilter::Off, + }) + .unwrap_or(LevelFilter::Off); + let logger = match logger { + Some(logger) => Some(ThreadPtrHolder::new(cstr!("mozIServicesLogSink"), logger)?), + None => None, + }; + let progress = callback + .query_interface::() + .and_then(|p| { + ThreadPtrHolder::new(cstr!("mozISyncedBookmarksMirrorProgressListener"), p).ok() + }); + Ok(MergeTask { + db: db.clone(), + controller, + max_log_level, + logger, + local_time_millis: local_time_seconds * 1000, + remote_time_millis: remote_time_seconds * 1000, + progress, + callback: ThreadPtrHolder::new(cstr!("mozISyncedBookmarksMirrorCallback"), callback)?, + result: AtomicRefCell::new(Err(error::Error::DidNotRun)), + }) + } + + fn merge(&self) -> error::Result { + let mut db = self.db.clone(); + if db.transaction_in_progress()? { + // If a transaction is already open, we can avoid an unnecessary + // merge, since we won't be able to apply the merged tree back to + // Places. This is common, especially if the user makes lots of + // changes at once. In that case, our merge task might run in the + // middle of a `Sqlite.sys.mjs` transaction, and fail when we try to + // open our own transaction in `Store::apply`. Since the local + // tree might be in an inconsistent state, we can't safely update + // Places. + return Err(error::Error::StorageBusy); + } + let log = Logger::new(self.max_log_level, self.logger.clone()); + let driver = Driver::new(log, self.progress.clone()); + let mut store = store::Store::new( + &mut db, + &driver, + &self.controller, + self.local_time_millis, + self.remote_time_millis, + ); + store.validate()?; + store.prepare()?; + let status = store.merge_with_driver(&driver, &*self.controller)?; + Ok(status) + } +} + +impl Task for MergeTask { + fn run(&self) { + *self.result.borrow_mut() = self.merge(); + } + + fn done(&self) -> Result<(), nsresult> { + let callback = self.callback.get().unwrap(); + match mem::replace(&mut *self.result.borrow_mut(), Err(error::Error::DidNotRun)) { + Ok(status) => unsafe { callback.HandleSuccess(status.into()) }, + Err(err) => { + let mut message = nsString::new(); + write!(message, "{}", err).unwrap(); + unsafe { callback.HandleError(err.into(), &*message) } + } + } + .to_result() + } +} + +#[xpcom(implement(mozIPlacesPendingOperation), atomic)] +pub struct MergeOp { + controller: Arc, +} + +impl MergeOp { + pub fn new(controller: Arc) -> RefPtr { + MergeOp::allocate(InitMergeOp { controller }) + } + + xpcom_method!(cancel => Cancel()); + fn cancel(&self) -> Result<(), nsresult> { + self.controller.abort(); + Ok(()) + } +} diff --git a/toolkit/components/places/bookmark_sync/src/store.rs b/toolkit/components/places/bookmark_sync/src/store.rs new file mode 100644 index 0000000000..f18beb5633 --- /dev/null +++ b/toolkit/components/places/bookmark_sync/src/store.rs @@ -0,0 +1,1328 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{collections::HashMap, convert::TryFrom, fmt}; + +use dogear::{ + debug, warn, AbortSignal, CompletionOps, Content, DeleteLocalItem, Guid, Item, Kind, + MergedRoot, Tree, UploadItem, UploadTombstone, Validity, +}; +use nsstring::nsString; +use storage::{Conn, Step}; +use url::Url; +use xpcom::interfaces::{mozISyncedBookmarksMerger, nsINavBookmarksService}; + +use crate::driver::{AbortController, Driver}; +use crate::error::{Error, Result}; + +extern "C" { + fn NS_NavBookmarksTotalSyncChanges() -> i64; +} + +fn total_sync_changes() -> i64 { + unsafe { NS_NavBookmarksTotalSyncChanges() } +} + +// Return all the non-root-roots as a 'sql set' (ie, suitable for use in an +// IN statement) +fn user_roots_as_sql_set() -> String { + format!( + "('{0}', '{1}', '{2}', '{3}', '{4}')", + dogear::MENU_GUID, + dogear::MOBILE_GUID, + dogear::TAGS_GUID, + dogear::TOOLBAR_GUID, + dogear::UNFILED_GUID + ) +} + +pub struct Store<'s> { + db: &'s mut Conn, + driver: &'s Driver, + controller: &'s AbortController, + + /// The total Sync change count before merging. We store this before + /// accessing Places, and compare the current and stored counts after + /// opening our transaction. If they match, we can safely apply the + /// tree. Otherwise, we bail and try merging again on the next sync. + total_sync_changes: i64, + + local_time_millis: i64, + remote_time_millis: i64, +} + +impl<'s> Store<'s> { + pub fn new( + db: &'s mut Conn, + driver: &'s Driver, + controller: &'s AbortController, + local_time_millis: i64, + remote_time_millis: i64, + ) -> Store<'s> { + Store { + db, + driver, + controller, + total_sync_changes: total_sync_changes(), + local_time_millis, + remote_time_millis, + } + } + + /// Ensures that all local roots are parented correctly. + /// + /// The Places root can't be in another folder, or we'll recurse infinitely + /// when we try to fetch the local tree. + /// + /// The five built-in roots should be under the Places root, or we'll build + /// and sync an invalid tree (bug 1453994, bug 1472127). + pub fn validate(&self) -> Result<()> { + self.controller.err_if_aborted()?; + let mut statement = self.db.prepare(format!( + "SELECT NOT EXISTS( + SELECT 1 FROM moz_bookmarks + WHERE id = (SELECT parent FROM moz_bookmarks + WHERE guid = '{root}') + ) AND NOT EXISTS( + SELECT 1 FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.guid IN {user_roots} AND + p.guid <> '{root}' + )", + root = dogear::ROOT_GUID, + user_roots = user_roots_as_sql_set(), + ))?; + let has_valid_roots = match statement.step()? { + Some(row) => row.get_by_index::(0)? == 1, + None => false, + }; + if has_valid_roots { + Ok(()) + } else { + Err(Error::InvalidLocalRoots) + } + } + + /// Prepares the mirror database for a merge. + pub fn prepare(&self) -> Result<()> { + // Sync associates keywords with bookmarks, and doesn't sync POST data; + // Places associates keywords with (URL, POST data) pairs, and multiple + // bookmarks may have the same URL. When a keyword changes, clients + // should reupload all bookmarks with the affected URL (see + // `PlacesSyncUtils.bookmarks.addSyncChangesForBookmarksWithURL` and + // bug 1328737). Just in case, we flag any remote bookmarks that have + // different keywords for the same URL, or the same keyword for + // different URLs, for reupload. + self.controller.err_if_aborted()?; + self.db.exec(format!( + "UPDATE items SET + validity = {} + WHERE validity = {} AND ( + urlId IN ( + /* Same URL, different keywords. `COUNT` ignores NULLs, so + we need to count them separately. This handles cases where + a keyword was removed from one, but not all bookmarks with + the same URL. */ + SELECT urlId FROM items + GROUP BY urlId + HAVING COUNT(DISTINCT keyword) + + COUNT(DISTINCT CASE WHEN keyword IS NULL + THEN 1 END) > 1 + ) OR keyword IN ( + /* Different URLs, same keyword. Bookmarks with keywords but + without URLs are already invalid, so we don't need to handle + NULLs here. */ + SELECT keyword FROM items + WHERE keyword NOT NULL + GROUP BY keyword + HAVING COUNT(DISTINCT urlId) > 1 + ) + )", + mozISyncedBookmarksMerger::VALIDITY_REUPLOAD, + mozISyncedBookmarksMerger::VALIDITY_VALID, + ))?; + Ok(()) + } + + /// Creates a local tree item from a row in the `localItems` CTE. + fn local_row_to_item(&self, step: &Step) -> Result<(Item, Option)> { + let raw_guid: nsString = step.get_by_name("guid")?; + let guid = Guid::from_utf16(&*raw_guid)?; + + let raw_url_href: Option = step.get_by_name("url")?; + let (url, validity) = match raw_url_href { + // Local items might have (syntactically) invalid URLs, as in bug + // 1615931. If we try to sync these items, other clients will flag + // them as invalid (see `SyncedBookmarksMirror#storeRemote{Bookmark, + // Query}`), delete them when merging, and upload tombstones for + // them. We can avoid this extra round trip by flagging the local + // item as invalid. If there's a corresponding remote item with a + // valid URL, we'll replace the local item with it; if there isn't, + // we'll delete the local item. + Some(raw_url_href) => match Url::parse(&String::from_utf16(&*raw_url_href)?) { + Ok(url) => (Some(url), Validity::Valid), + Err(err) => { + warn!( + self.driver, + "Failed to parse URL for local item {}: {}", guid, err + ); + (None, Validity::Replace) + } + }, + None => (None, Validity::Valid), + }; + + let typ: i64 = step.get_by_name("type")?; + let kind = match u16::try_from(typ) { + Ok(nsINavBookmarksService::TYPE_BOOKMARK) => match url.as_ref() { + Some(u) if u.scheme() == "place" => Kind::Query, + _ => Kind::Bookmark, + }, + Ok(nsINavBookmarksService::TYPE_FOLDER) => { + let is_livemark: i64 = step.get_by_name("isLivemark")?; + if is_livemark == 1 { + Kind::Livemark + } else { + Kind::Folder + } + } + Ok(nsINavBookmarksService::TYPE_SEPARATOR) => Kind::Separator, + _ => return Err(Error::UnknownItemType(typ)), + }; + + let mut item = Item::new(guid, kind); + + let local_modified: i64 = step.get_by_name_or_default("localModified"); + item.age = (self.local_time_millis - local_modified).max(0); + + let sync_change_counter: i64 = step.get_by_name("syncChangeCounter")?; + item.needs_merge = sync_change_counter > 0; + + item.validity = validity; + + let content = if item.validity == Validity::Replace || item.guid == dogear::ROOT_GUID { + None + } else { + let sync_status: i64 = step.get_by_name("syncStatus")?; + match u16::try_from(sync_status) { + Ok(nsINavBookmarksService::SYNC_STATUS_NORMAL) => None, + _ => match kind { + Kind::Bookmark | Kind::Query => { + let raw_title: nsString = step.get_by_name("title")?; + let title = String::from_utf16(&*raw_title)?; + url.map(|url| Content::Bookmark { + title, + url_href: url.into_string(), + }) + } + Kind::Folder | Kind::Livemark => { + let raw_title: nsString = step.get_by_name("title")?; + let title = String::from_utf16(&*raw_title)?; + Some(Content::Folder { title }) + } + Kind::Separator => Some(Content::Separator), + }, + } + }; + + Ok((item, content)) + } + + /// Creates a remote tree item from a row in `mirror.items`. + fn remote_row_to_item(&self, step: &Step) -> Result<(Item, Option)> { + let raw_guid: nsString = step.get_by_name("guid")?; + let guid = Guid::from_utf16(&*raw_guid)?; + + let raw_kind: i64 = step.get_by_name("kind")?; + let kind = Kind::from_column(raw_kind)?; + + let mut item = Item::new(guid, kind); + + let remote_modified: i64 = step.get_by_name("serverModified")?; + item.age = (self.remote_time_millis - remote_modified).max(0); + + let needs_merge: i32 = step.get_by_name("needsMerge")?; + item.needs_merge = needs_merge == 1; + + let raw_validity: i64 = step.get_by_name("validity")?; + item.validity = Validity::from_column(raw_validity)?; + + let content = if item.validity == Validity::Replace + || item.guid == dogear::ROOT_GUID + || !item.needs_merge + { + None + } else { + match kind { + Kind::Bookmark | Kind::Query => { + let raw_title: nsString = step.get_by_name("title")?; + let title = String::from_utf16(&*raw_title)?; + let raw_url_href: Option = step.get_by_name("url")?; + match raw_url_href { + Some(raw_url_href) => { + // Unlike for local items, we don't parse URLs for + // remote items, since `storeRemote{Bookmark, + // Query}` already parses and canonicalizes them + // before inserting them into the mirror database. + let url_href = String::from_utf16(&*raw_url_href)?; + Some(Content::Bookmark { title, url_href }) + } + None => None, + } + } + Kind::Folder | Kind::Livemark => { + let raw_title: nsString = step.get_by_name("title")?; + let title = String::from_utf16(&*raw_title)?; + Some(Content::Folder { title }) + } + Kind::Separator => Some(Content::Separator), + } + }; + + Ok((item, content)) + } +} + +impl<'s> dogear::Store for Store<'s> { + type Ok = ApplyStatus; + type Error = Error; + + /// Builds a fully rooted, consistent tree from the items and tombstones in + /// Places. + fn fetch_local_tree(&self) -> Result { + let mut root_statement = self.db.prepare(format!( + "SELECT guid, type, syncChangeCounter, syncStatus, + lastModified / 1000 AS localModified, + NULL AS url, 0 AS isLivemark + FROM moz_bookmarks + WHERE guid = '{}'", + dogear::ROOT_GUID + ))?; + let mut builder = match root_statement.step()? { + Some(step) => { + let (item, _) = self.local_row_to_item(&step)?; + Tree::with_root(item) + } + None => return Err(Error::InvalidLocalRoots), + }; + + // Add items and contents to the builder, keeping track of their + // structure in a separate map. We can't call `p.by_structure(...)` + // after adding the item, because this query might return rows for + // children before their parents. This approach also lets us scan + // `moz_bookmarks` once, using the index on `(b.parent, b.position)` + // to avoid a temp B-tree for the `ORDER BY`. + let mut child_guids_by_parent_guid: HashMap> = HashMap::new(); + let mut items_statement = self.db.prepare(format!( + "SELECT b.guid, p.guid AS parentGuid, b.type, b.syncChangeCounter, + b.syncStatus, b.lastModified / 1000 AS localModified, + IFNULL(b.title, '') AS title, + (SELECT h.url FROM moz_places h WHERE h.id = b.fk) AS url, + 0 AS isLivemark + FROM moz_bookmarks b + JOIN moz_bookmarks p ON p.id = b.parent + WHERE b.guid <> '{}' + ORDER BY b.parent, b.position", + dogear::ROOT_GUID, + ))?; + while let Some(step) = items_statement.step()? { + self.controller.err_if_aborted()?; + let (item, content) = self.local_row_to_item(&step)?; + + let raw_parent_guid: nsString = step.get_by_name("parentGuid")?; + let parent_guid = Guid::from_utf16(&*raw_parent_guid)?; + child_guids_by_parent_guid + .entry(parent_guid) + .or_default() + .push(item.guid.clone()); + + let mut p = builder.item(item)?; + if let Some(content) = content { + p.content(content); + } + } + + // At this point, we've added entries for all items to the tree, so + // we can add their structure info. + for (parent_guid, child_guids) in &child_guids_by_parent_guid { + for child_guid in child_guids { + self.controller.err_if_aborted()?; + builder.parent_for(child_guid).by_structure(parent_guid)?; + } + } + + let mut deletions_statement = self.db.prepare("SELECT guid FROM moz_bookmarks_deleted")?; + while let Some(step) = deletions_statement.step()? { + self.controller.err_if_aborted()?; + let raw_guid: nsString = step.get_by_name("guid")?; + let guid = Guid::from_utf16(&*raw_guid)?; + builder.deletion(guid); + } + + let tree = Tree::try_from(builder)?; + Ok(tree) + } + + /// Builds a fully rooted, consistent tree from the items and tombstones in the + /// mirror. + fn fetch_remote_tree(&self) -> Result { + let mut root_statement = self.db.prepare(format!( + "SELECT guid, serverModified, kind, needsMerge, validity + FROM items + WHERE guid = '{}'", + dogear::ROOT_GUID, + ))?; + let mut builder = match root_statement.step()? { + Some(step) => { + let (item, _) = self.remote_row_to_item(&step)?; + Tree::with_root(item) + } + None => return Err(Error::InvalidRemoteRoots), + }; + builder.reparent_orphans_to(&dogear::UNFILED_GUID); + + let mut items_statement = self.db.prepare(format!( + "SELECT v.guid, v.parentGuid, v.serverModified, v.kind, + IFNULL(v.title, '') AS title, v.needsMerge, v.validity, + v.isDeleted, + (SELECT u.url FROM urls u + WHERE u.id = v.urlId) AS url + FROM items v + WHERE v.guid <> '{}' + ORDER BY v.guid", + dogear::ROOT_GUID, + ))?; + while let Some(step) = items_statement.step()? { + self.controller.err_if_aborted()?; + + let is_deleted: i64 = step.get_by_name("isDeleted")?; + if is_deleted == 1 { + let needs_merge: i32 = step.get_by_name("needsMerge")?; + if needs_merge == 0 { + // Ignore already-merged tombstones. These aren't persisted + // locally, so merging them is a no-op. + continue; + } + let raw_guid: nsString = step.get_by_name("guid")?; + let guid = Guid::from_utf16(&*raw_guid)?; + builder.deletion(guid); + } else { + let (item, content) = self.remote_row_to_item(&step)?; + let mut p = builder.item(item)?; + if let Some(content) = content { + p.content(content); + } + let raw_parent_guid: Option = step.get_by_name("parentGuid")?; + if let Some(raw_parent_guid) = raw_parent_guid { + p.by_parent_guid(Guid::from_utf16(&*raw_parent_guid)?)?; + } + } + } + + let mut structure_statement = self.db.prepare(format!( + "SELECT guid, parentGuid FROM structure + WHERE guid <> '{}' + ORDER BY parentGuid, position", + dogear::ROOT_GUID, + ))?; + while let Some(step) = structure_statement.step()? { + self.controller.err_if_aborted()?; + let raw_guid: nsString = step.get_by_name("guid")?; + let guid = Guid::from_utf16(&*raw_guid)?; + + let raw_parent_guid: nsString = step.get_by_name("parentGuid")?; + let parent_guid = Guid::from_utf16(&*raw_parent_guid)?; + + builder.parent_for(&guid).by_children(&parent_guid)?; + } + + let tree = Tree::try_from(builder)?; + Ok(tree) + } + + fn apply<'t>(&mut self, root: MergedRoot<'t>) -> Result { + let ops = root.completion_ops_with_signal(self.controller)?; + + if ops.is_empty() { + // If we don't have any items to apply, upload, or delete, + // no need to open a transaction at all. + return Ok(ApplyStatus::Skipped); + } + + // Apply the merged tree and stage outgoing items. This transaction + // blocks writes from the main connection until it's committed, so we + // try to do as little work as possible within it. + if self.db.transaction_in_progress()? { + return Err(Error::StorageBusy); + } + let tx = self.db.transaction()?; + if self.total_sync_changes != total_sync_changes() { + return Err(Error::MergeConflict); + } + + debug!(self.driver, "Updating local items in Places"); + update_local_items_in_places( + &tx, + &self.driver, + &self.controller, + self.local_time_millis, + &ops, + )?; + + debug!(self.driver, "Staging items to upload"); + stage_items_to_upload( + &tx, + &self.driver, + &self.controller, + &ops.upload_items, + &ops.upload_tombstones, + )?; + + cleanup(&tx)?; + tx.commit()?; + + Ok(ApplyStatus::Merged) + } +} + +/// Builds a temporary table with the merge states of all nodes in the merged +/// tree and updates Places to match the merged tree. +/// +/// Conceptually, we examine the merge state of each item, and either leave the +/// item unchanged, upload the local side, apply the remote side, or apply and +/// then reupload the remote side with a new structure. +/// +/// Note that we update Places and flag items *before* upload, while iOS +/// updates the mirror *after* a successful upload. This simplifies our +/// implementation, though we lose idempotent merges. If upload is interrupted, +/// the next sync won't distinguish between new merge states from the previous +/// sync, and local changes. +fn update_local_items_in_places<'t>( + db: &Conn, + driver: &Driver, + controller: &AbortController, + local_time_millis: i64, + ops: &CompletionOps<'t>, +) -> Result<()> { + debug!( + driver, + "Cleaning up observer notifications left from last sync" + ); + controller.err_if_aborted()?; + db.exec( + "DELETE FROM itemsAdded; + DELETE FROM guidsChanged; + DELETE FROM itemsChanged; + DELETE FROM itemsRemoved; + DELETE FROM itemsMoved;", + )?; + + // Places uses microsecond timestamps for dates added and last modified + // times, rounded to the nearest millisecond. Using `now` for the local + // time lets us set modified times deterministically for tests. + let now = local_time_millis * 1000; + + // Insert URLs for new remote items into the `moz_places` table. We need to + // do this before inserting new remote items, since we need Place IDs for + // both old and new URLs. + debug!(driver, "Inserting Places for new items"); + for chunk in ops.apply_remote_items.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "INSERT OR IGNORE INTO moz_places(url, url_hash, rev_host, hidden, + frecency, guid) + SELECT u.url, u.hash, u.revHost, + (CASE WHEN u.url BETWEEN 'place:' AND 'place:' || X'FFFF' THEN 1 ELSE 0 END), + (CASE v.kind WHEN {} THEN 0 ELSE -1 END), + IFNULL((SELECT h.guid FROM moz_places h + WHERE h.url_hash = u.hash AND + h.url = u.url), u.guid) + FROM items v + JOIN urls u ON u.id = v.urlId + WHERE v.guid IN ({})", + mozISyncedBookmarksMerger::KIND_QUERY, + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + let remote_guid = nsString::from(&*op.remote_node().guid); + statement.bind_by_index(index as u32, remote_guid)?; + } + statement.execute()?; + } + + // Trigger frecency updates for all new origins. + debug!(driver, "Updating origins for new URLs"); + controller.err_if_aborted()?; + db.exec("DELETE FROM moz_updateoriginsinsert_temp")?; + + // Build a table of new and updated items. + debug!(driver, "Staging apply remote item ops"); + for chunk in ops.apply_remote_items.chunks(db.variable_limit()? / 3) { + // CTEs in `WITH` clauses aren't indexed, so this query needs a full + // table scan on `ops`. But that's okay; a separate temp table for ops + // would also need a full scan. Note that we need both the local _and_ + // remote GUIDs here, because we haven't changed the local GUIDs yet. + let mut statement = db.prepare(format!( + "WITH + ops(mergedGuid, localGuid, remoteGuid, level) AS (VALUES {}) + INSERT INTO itemsToApply(mergedGuid, localId, remoteId, + remoteGuid, newLevel, + newType, + localDateAddedMicroseconds, + remoteDateAddedMicroseconds, + lastModifiedMicroseconds, + oldTitle, newTitle, oldPlaceId, + newPlaceId, + newKeyword) + SELECT n.mergedGuid, b.id, v.id, + v.guid, n.level, + (CASE WHEN v.kind IN ({}, {}) THEN {} + WHEN v.kind IN ({}, {}) THEN {} + ELSE {} + END), + b.dateAdded, + v.dateAdded * 1000, + MAX(v.dateAdded * 1000, {}), + b.title, v.title, b.fk, + (SELECT h.id FROM moz_places h + JOIN urls u ON u.hash = h.url_hash + WHERE u.id = v.urlId AND + u.url = h.url), + v.keyword + FROM ops n + JOIN items v ON v.guid = n.remoteGuid + LEFT JOIN moz_bookmarks b ON b.guid = n.localGuid", + repeat_display(chunk.len(), ",", |index, f| { + let op = &chunk[index]; + write!(f, "(?, ?, ?, {})", op.level) + }), + mozISyncedBookmarksMerger::KIND_BOOKMARK, + mozISyncedBookmarksMerger::KIND_QUERY, + nsINavBookmarksService::TYPE_BOOKMARK, + mozISyncedBookmarksMerger::KIND_FOLDER, + mozISyncedBookmarksMerger::KIND_LIVEMARK, + nsINavBookmarksService::TYPE_FOLDER, + nsINavBookmarksService::TYPE_SEPARATOR, + now, + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + + let offset = (index * 3) as u32; + + // In most cases, the merged and remote GUIDs are the same for new + // items. For updates, all three are typically the same. We could + // try to avoid binding duplicates, but that complicates chunking, + // and we don't expect many items to change after the first sync. + let merged_guid = nsString::from(&*op.merged_node.guid); + statement.bind_by_index(offset, merged_guid)?; + + let local_guid = op + .merged_node + .merge_state + .local_node() + .map(|node| nsString::from(&*node.guid)); + statement.bind_by_index(offset + 1, local_guid)?; + + let remote_guid = nsString::from(&*op.remote_node().guid); + statement.bind_by_index(offset + 2, remote_guid)?; + } + statement.execute()?; + } + + debug!(driver, "Staging change GUID ops"); + for chunk in ops.change_guids.chunks(db.variable_limit()? / 2) { + let mut statement = db.prepare(format!( + "INSERT INTO changeGuidOps(localGuid, mergedGuid, syncStatus, level, + lastModifiedMicroseconds) + VALUES {}", + repeat_display(chunk.len(), ",", |index, f| { + let op = &chunk[index]; + // If only the local GUID changed, the item was deduped, so we + // can mark it as syncing. Otherwise, we changed an invalid + // GUID locally or remotely, so we leave its original sync + // status in place until we've uploaded it. + let sync_status = if op.merged_node.remote_guid_changed() { + None + } else { + Some(nsINavBookmarksService::SYNC_STATUS_NORMAL) + }; + write!( + f, + "(?, ?, {}, {}, {})", + NullableFragment(sync_status), + op.level, + now + ) + }) + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + + let offset = (index * 2) as u32; + + let local_guid = nsString::from(&*op.local_node().guid); + statement.bind_by_index(offset, local_guid)?; + + let merged_guid = nsString::from(&*op.merged_node.guid); + statement.bind_by_index(offset + 1, merged_guid)?; + } + statement.execute()?; + } + + debug!(driver, "Staging apply new local structure ops"); + for chunk in ops + .apply_new_local_structure + .chunks(db.variable_limit()? / 2) + { + let mut statement = db.prepare(format!( + "INSERT INTO applyNewLocalStructureOps(mergedGuid, mergedParentGuid, + position, level, + lastModifiedMicroseconds) + VALUES {}", + repeat_display(chunk.len(), ",", |index, f| { + let op = &chunk[index]; + write!(f, "(?, ?, {}, {}, {})", op.position, op.level, now) + }) + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + + let offset = (index * 2) as u32; + + let merged_guid = nsString::from(&*op.merged_node.guid); + statement.bind_by_index(offset, merged_guid)?; + + let merged_parent_guid = nsString::from(&*op.merged_parent_node.guid); + statement.bind_by_index(offset + 1, merged_parent_guid)?; + } + statement.execute()?; + } + + debug!(driver, "Removing tombstones for revived items"); + for chunk in ops.delete_local_tombstones.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "DELETE FROM moz_bookmarks_deleted + WHERE guid IN ({})", + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(&*op.guid().as_str()))?; + } + statement.execute()?; + } + + debug!( + driver, + "Inserting new tombstones for non-syncable and invalid items" + ); + for chunk in ops.insert_local_tombstones.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "INSERT INTO moz_bookmarks_deleted(guid, dateRemoved) + VALUES {}", + repeat_display(chunk.len(), ",", |_, f| write!(f, "(?, {})", now)), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index( + index as u32, + nsString::from(&*op.remote_node().guid.as_str()), + )?; + } + statement.execute()?; + } + + debug!(driver, "Removing local items"); + for chunk in ops.delete_local_items.chunks(db.variable_limit()?) { + remove_local_items(&db, driver, controller, chunk)?; + } + + // Fires the `changeGuids` trigger. + debug!(driver, "Changing GUIDs"); + controller.err_if_aborted()?; + db.exec("DELETE FROM changeGuidOps")?; + + debug!(driver, "Applying remote items"); + apply_remote_items(db, driver, controller)?; + + // Trigger frecency updates for all affected origins. + debug!(driver, "Updating origins for changed URLs"); + controller.err_if_aborted()?; + db.exec("DELETE FROM moz_updateoriginsupdate_temp")?; + + // Fires the `applyNewLocalStructure` trigger. + debug!(driver, "Applying new local structure"); + controller.err_if_aborted()?; + db.exec("DELETE FROM applyNewLocalStructureOps")?; + + debug!( + driver, + "Resetting change counters for items that shouldn't be uploaded" + ); + for chunk in ops.set_local_merged.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "UPDATE moz_bookmarks SET + syncChangeCounter = 0 + WHERE guid IN ({})", + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(&*op.merged_node.guid))?; + } + statement.execute()?; + } + + debug!( + driver, + "Bumping change counters for items that should be uploaded" + ); + for chunk in ops.set_local_unmerged.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "UPDATE moz_bookmarks SET + syncChangeCounter = 1 + WHERE guid IN ({})", + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(&*op.merged_node.guid))?; + } + statement.execute()?; + } + + debug!(driver, "Flagging applied remote items as merged"); + for chunk in ops.set_remote_merged.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "UPDATE items SET + needsMerge = 0 + WHERE guid IN ({})", + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(op.guid().as_str()))?; + } + statement.execute()?; + } + + Ok(()) +} + +/// Upserts all new and updated items from the `itemsToApply` table into Places. +fn apply_remote_items(db: &Conn, driver: &Driver, controller: &AbortController) -> Result<()> { + debug!(driver, "Recording item added notifications for new items"); + controller.err_if_aborted()?; + db.exec( + "INSERT INTO itemsAdded(guid, keywordChanged, level) + SELECT n.mergedGuid, n.newKeyword NOT NULL OR + EXISTS(SELECT 1 FROM moz_keywords k + WHERE k.place_id = n.newPlaceId OR + k.keyword = n.newKeyword), + n.newLevel + FROM itemsToApply n + WHERE n.localId IS NULL", + )?; + + debug!( + driver, + "Recording item changed notifications for existing items" + ); + controller.err_if_aborted()?; + db.exec( + "INSERT INTO itemsChanged(itemId, oldTitle, oldPlaceId, keywordChanged, + level) + SELECT n.localId, n.oldTitle, n.oldPlaceId, + n.newKeyword NOT NULL OR EXISTS( + SELECT 1 FROM moz_keywords k + WHERE k.place_id IN (n.oldPlaceId, n.newPlaceId) OR + k.keyword = n.newKeyword + ), + n.newLevel + FROM itemsToApply n + WHERE n.localId NOT NULL", + )?; + + // Remove all keywords from old and new URLs, and remove new keywords from + // all existing URLs. The `NOT NULL` conditions are important; they ensure + // that SQLite uses our partial indexes, instead of a table scan. + debug!(driver, "Removing old keywords"); + controller.err_if_aborted()?; + db.exec( + "DELETE FROM moz_keywords + WHERE place_id IN (SELECT oldPlaceId FROM itemsToApply + WHERE oldPlaceId NOT NULL) OR + place_id IN (SELECT newPlaceId FROM itemsToApply + WHERE newPlaceId NOT NULL) OR + keyword IN (SELECT newKeyword FROM itemsToApply + WHERE newKeyword NOT NULL) + ", + )?; + + debug!(driver, "Removing old tags"); + controller.err_if_aborted()?; + db.exec( + "DELETE FROM localTags + WHERE placeId IN (SELECT oldPlaceId FROM itemsToApply + WHERE oldPlaceId NOT NULL) OR + placeId IN (SELECT newPlaceId FROM itemsToApply + WHERE newPlaceId NOT NULL)", + )?; + + // Insert and update items, using -1 for new items' parent IDs and + // positions. We'll update these later, when we apply the new local + // structure. This is a full table scan on `itemsToApply`. The no-op + // `WHERE` clause is necessary to avoid a parsing ambiguity. + debug!(driver, "Upserting new items"); + controller.err_if_aborted()?; + db.exec(format!( + "INSERT INTO moz_bookmarks(id, guid, parent, position, type, fk, title, + dateAdded, + lastModified, + syncStatus, syncChangeCounter) + SELECT localId, mergedGuid, -1, -1, newType, newPlaceId, newTitle, + /* Pick the older of the local and remote date added. We'll + weakly reupload any items with an older local date. */ + MIN(IFNULL(localDateAddedMicroseconds, + remoteDateAddedMicroseconds), + remoteDateAddedMicroseconds), + /* The last modified date should always be newer than the date + added, so we pick the newer of the two here. */ + MAX(lastModifiedMicroseconds, remoteDateAddedMicroseconds), + {syncStatusNormal}, 0 + FROM itemsToApply + WHERE 1 + ON CONFLICT(id) DO UPDATE SET + title = excluded.title, + dateAdded = excluded.dateAdded, + lastModified = excluded.lastModified, + syncStatus = {syncStatusNormal}, + /* It's important that we update the URL *after* removing old keywords + and *before* inserting new ones, so that the above DELETEs select + the correct affected items. */ + fk = excluded.fk", + syncStatusNormal = nsINavBookmarksService::SYNC_STATUS_NORMAL + ))?; + // The roots are never in `itemsToApply` but still need to (well, at least + // *should*) have a syncStatus of NORMAL after a reconcilliation. The + // ROOT_GUID doesn't matter in practice, but we include it to be consistent. + db.exec(format!( + "UPDATE moz_bookmarks SET + syncStatus={syncStatusNormal} + WHERE guid IN {user_roots} OR + guid = '{root}' + ", + syncStatusNormal = nsINavBookmarksService::SYNC_STATUS_NORMAL, + root = dogear::ROOT_GUID, + user_roots = user_roots_as_sql_set(), + ))?; + + // Flag frecencies for recalculation. This is a multi-index OR that uses the + // `oldPlacesIds` and `newPlaceIds` partial indexes, since `<>` is only true + // if both terms are not NULL. Without those constraints, the subqueries + // would scan `itemsToApply` twice. The `oldPlaceId <> newPlaceId` and + // `newPlaceId <> oldPlaceId` checks exclude items where the URL didn't + // change; we don't need to recalculate their frecencies. + debug!(driver, "Flagging frecencies for recalculation"); + controller.err_if_aborted()?; + db.exec( + "UPDATE moz_places SET + recalc_frecency = 1, recalc_alt_frecency = 1 + WHERE frecency <> 0 AND ( + id IN ( + SELECT oldPlaceId FROM itemsToApply + WHERE oldPlaceId <> newPlaceId + ) OR id IN ( + SELECT newPlaceId FROM itemsToApply + WHERE newPlaceId <> oldPlaceId + ) + )", + )?; + + debug!(driver, "Inserting new keywords for new URLs"); + controller.err_if_aborted()?; + db.exec( + "INSERT OR IGNORE INTO moz_keywords(keyword, place_id, post_data) + SELECT newKeyword, newPlaceId, '' + FROM itemsToApply + WHERE newKeyword NOT NULL", + )?; + + debug!(driver, "Inserting new tags for new URLs"); + controller.err_if_aborted()?; + db.exec( + "INSERT INTO localTags(tag, placeId, lastModifiedMicroseconds) + SELECT t.tag, n.newPlaceId, n.lastModifiedMicroseconds + FROM itemsToApply n + JOIN tags t ON t.itemId = n.remoteId", + )?; + + Ok(()) +} + +/// Removes deleted local items from Places. +fn remove_local_items( + db: &Conn, + driver: &Driver, + controller: &AbortController, + ops: &[DeleteLocalItem], +) -> Result<()> { + debug!(driver, "Recording observer notifications for removed items"); + let mut observer_statement = db.prepare(format!( + "WITH + ops(guid, level) AS (VALUES {}) + INSERT INTO itemsRemoved(itemId, parentId, position, type, title, + placeId, guid, parentGuid, level, keywordRemoved) + SELECT b.id, b.parent, b.position, b.type, IFNULL(b.title, \"\"), b.fk, + b.guid, p.guid, n.level, EXISTS(SELECT 1 FROM moz_keywords k WHERE k.place_id = b.fk) + FROM ops n + JOIN moz_bookmarks b ON b.guid = n.guid + JOIN moz_bookmarks p ON p.id = b.parent", + repeat_display(ops.len(), ",", |index, f| { + let op = &ops[index]; + write!(f, "(?, {})", op.local_node().level()) + }), + ))?; + for (index, op) in ops.iter().enumerate() { + controller.err_if_aborted()?; + observer_statement.bind_by_index( + index as u32, + nsString::from(&*op.local_node().guid.as_str()), + )?; + } + observer_statement.execute()?; + + debug!(driver, "Recalculating frecencies for removed bookmark URLs"); + let mut frecency_statement = db.prepare(format!( + "UPDATE moz_places SET + recalc_frecency = 1, recalc_alt_frecency = 1 + WHERE id IN (SELECT b.fk FROM moz_bookmarks b + WHERE b.guid IN ({})) AND + frecency <> 0", + repeat_sql_vars(ops.len()) + ))?; + for (index, op) in ops.iter().enumerate() { + controller.err_if_aborted()?; + frecency_statement.bind_by_index( + index as u32, + nsString::from(&*op.local_node().guid.as_str()), + )?; + } + frecency_statement.execute()?; + + // This can be removed in bug 1460577. + debug!(driver, "Removing annos for deleted items"); + let mut annos_statement = db.prepare(format!( + "DELETE FROM moz_items_annos + WHERE item_id = (SELECT b.id FROM moz_bookmarks b + WHERE b.guid IN ({}))", + repeat_sql_vars(ops.len()), + ))?; + for (index, op) in ops.iter().enumerate() { + controller.err_if_aborted()?; + annos_statement.bind_by_index( + index as u32, + nsString::from(&*op.local_node().guid.as_str()), + )?; + } + annos_statement.execute()?; + + debug!( + driver, + "Removing keywords associated with deleted bookmarks" + ); + let mut keywords_statement = db.prepare(format!( + "DELETE FROM moz_keywords + WHERE place_id IN (SELECT b.fk FROM moz_bookmarks b + WHERE b.guid IN ({}))", + repeat_sql_vars(ops.len()), + ))?; + for (index, op) in ops.iter().enumerate() { + controller.err_if_aborted()?; + keywords_statement.bind_by_index( + index as u32, + nsString::from(&*op.local_node().guid.as_str()), + )?; + } + keywords_statement.execute()?; + + debug!(driver, "Removing deleted items from Places"); + let mut delete_statement = db.prepare(format!( + "DELETE FROM moz_bookmarks + WHERE guid IN ({})", + repeat_sql_vars(ops.len()), + ))?; + for (index, op) in ops.iter().enumerate() { + controller.err_if_aborted()?; + delete_statement.bind_by_index( + index as u32, + nsString::from(&*op.local_node().guid.as_str()), + )?; + } + delete_statement.execute()?; + + Ok(()) +} + +/// Stores a snapshot of all locally changed items in a temporary table for +/// upload. This is called from within the merge transaction, to ensure that +/// changes made during the sync don't cause us to upload inconsistent records. +/// +/// For an example of why we use a temporary table instead of reading directly +/// from Places, consider a user adding a bookmark, then changing its parent +/// folder. We first add the bookmark to the default folder, bump the change +/// counter of the new bookmark and the default folder, then trigger a sync. +/// Depending on how quickly the user picks the new parent, we might upload +/// a record for the default folder, commit the move, then upload the bookmark. +/// We'll still upload the new parent on the next sync, but, in the meantime, +/// we've introduced a parent-child disagreement. This can also happen if the +/// user moves many items between folders. +/// +/// Conceptually, `itemsToUpload` is a transient "view" of locally changed +/// items. The change counter in Places is the persistent record of items that +/// we need to upload, so, if upload is interrupted or fails, we'll stage the +/// items again on the next sync. +fn stage_items_to_upload( + db: &Conn, + driver: &Driver, + controller: &AbortController, + upload_items: &[UploadItem], + upload_tombstones: &[UploadTombstone], +) -> Result<()> { + debug!(driver, "Cleaning up staged items left from last sync"); + controller.err_if_aborted()?; + db.exec("DELETE FROM itemsToUpload")?; + + // Stage remotely changed items with older local creation dates. These are + // tracked "weakly": if the upload is interrupted or fails, we won't + // reupload the record on the next sync. + debug!(driver, "Staging items with older local dates added"); + controller.err_if_aborted()?; + db.exec(format!( + "INSERT OR IGNORE INTO itemsToUpload(id, guid, syncChangeCounter, + parentGuid, parentTitle, dateAdded, + type, title, placeId, isQuery, url, + keyword, position, tagFolderName) + {} + JOIN itemsToApply n ON n.mergedGuid = b.guid + WHERE n.localDateAddedMicroseconds < n.remoteDateAddedMicroseconds", + UploadItemsFragment("b"), + ))?; + + debug!(driver, "Staging remaining locally changed items for upload"); + for chunk in upload_items.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "INSERT OR IGNORE INTO itemsToUpload(id, guid, syncChangeCounter, + parentGuid, parentTitle, + dateAdded, type, title, + placeId, isQuery, url, keyword, + position, tagFolderName) + {} + WHERE b.guid IN ({})", + UploadItemsFragment("b"), + repeat_sql_vars(chunk.len()), + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(&*op.merged_node.guid))?; + } + statement.execute()?; + } + + // Record the child GUIDs of locally changed folders, which we use to + // populate the `children` array in the record. + debug!(driver, "Staging structure to upload"); + controller.err_if_aborted()?; + db.exec( + " + INSERT INTO structureToUpload(guid, parentId, position) + SELECT b.guid, b.parent, b.position + FROM moz_bookmarks b + JOIN itemsToUpload o ON o.id = b.parent", + )?; + + // Stage tags for outgoing bookmarks. + debug!(driver, "Staging tags to upload"); + controller.err_if_aborted()?; + db.exec( + " + INSERT OR IGNORE INTO tagsToUpload(id, tag) + SELECT o.id, t.tag + FROM localTags t + JOIN itemsToUpload o ON o.placeId = t.placeId", + )?; + + // Finally, stage tombstones for deleted items. Ignore conflicts if we have + // tombstones for undeleted items; Places Maintenance should clean these up. + debug!(driver, "Staging tombstones to upload"); + for chunk in upload_tombstones.chunks(db.variable_limit()?) { + let mut statement = db.prepare(format!( + "INSERT OR IGNORE INTO itemsToUpload(guid, syncChangeCounter, isDeleted) + VALUES {}", + repeat_display(chunk.len(), ",", |_, f| write!(f, "(?, 1, 1)")) + ))?; + for (index, op) in chunk.iter().enumerate() { + controller.err_if_aborted()?; + statement.bind_by_index(index as u32, nsString::from(op.guid().as_str()))?; + } + statement.execute()?; + } + + Ok(()) +} + +fn cleanup(db: &Conn) -> Result<()> { + db.exec("DELETE FROM itemsToApply")?; + Ok(()) +} + +/// Formats a list of binding parameters for inclusion in a SQL list. +#[inline] +fn repeat_sql_vars(count: usize) -> impl fmt::Display { + repeat_display(count, ",", |_, f| write!(f, "?")) +} + +/// Construct a `RepeatDisplay` that will repeatedly call `fmt_one` with a +/// formatter `count` times, separated by `sep`. This is copied from the +/// `sql_support` crate in `application-services`. +#[inline] +fn repeat_display<'a, F>(count: usize, sep: &'a str, fmt_one: F) -> RepeatDisplay<'a, F> +where + F: Fn(usize, &mut fmt::Formatter) -> fmt::Result, +{ + RepeatDisplay { + count, + sep, + fmt_one, + } +} + +/// Helper type for printing repeated strings more efficiently. +#[derive(Debug, Clone)] +struct RepeatDisplay<'a, F> { + count: usize, + sep: &'a str, + fmt_one: F, +} + +impl<'a, F> fmt::Display for RepeatDisplay<'a, F> +where + F: Fn(usize, &mut fmt::Formatter) -> fmt::Result, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.count { + if i != 0 { + f.write_str(self.sep)?; + } + (self.fmt_one)(i, f)?; + } + Ok(()) + } +} + +/// Converts between a type `T` and its SQL representation. +trait Column { + fn from_column(raw: T) -> Result + where + Self: Sized; + fn into_column(self) -> T; +} + +impl Column for Kind { + fn from_column(raw: i64) -> Result { + Ok(match i16::try_from(raw) { + Ok(mozISyncedBookmarksMerger::KIND_BOOKMARK) => Kind::Bookmark, + Ok(mozISyncedBookmarksMerger::KIND_QUERY) => Kind::Query, + Ok(mozISyncedBookmarksMerger::KIND_FOLDER) => Kind::Folder, + Ok(mozISyncedBookmarksMerger::KIND_LIVEMARK) => Kind::Livemark, + Ok(mozISyncedBookmarksMerger::KIND_SEPARATOR) => Kind::Separator, + _ => return Err(Error::UnknownItemKind(raw)), + }) + } + + fn into_column(self) -> i64 { + match self { + Kind::Bookmark => mozISyncedBookmarksMerger::KIND_BOOKMARK as i64, + Kind::Query => mozISyncedBookmarksMerger::KIND_QUERY as i64, + Kind::Folder => mozISyncedBookmarksMerger::KIND_FOLDER as i64, + Kind::Livemark => mozISyncedBookmarksMerger::KIND_LIVEMARK as i64, + Kind::Separator => mozISyncedBookmarksMerger::KIND_SEPARATOR as i64, + } + } +} + +impl Column for Validity { + fn from_column(raw: i64) -> Result { + Ok(match i16::try_from(raw) { + Ok(mozISyncedBookmarksMerger::VALIDITY_VALID) => Validity::Valid, + Ok(mozISyncedBookmarksMerger::VALIDITY_REUPLOAD) => Validity::Reupload, + Ok(mozISyncedBookmarksMerger::VALIDITY_REPLACE) => Validity::Replace, + _ => return Err(Error::UnknownItemValidity(raw).into()), + }) + } + + fn into_column(self) -> i64 { + match self { + Validity::Valid => mozISyncedBookmarksMerger::VALIDITY_VALID as i64, + Validity::Reupload => mozISyncedBookmarksMerger::VALIDITY_REUPLOAD as i64, + Validity::Replace => mozISyncedBookmarksMerger::VALIDITY_REPLACE as i64, + } + } +} + +/// Formats an optional value so that it can be included in a SQL statement. +struct NullableFragment(Option); + +impl fmt::Display for NullableFragment +where + T: fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.0 { + Some(v) => v.fmt(f), + None => write!(f, "NULL"), + } + } +} + +/// Formats a `SELECT` statement for staging local items in the `itemsToUpload` +/// table. +struct UploadItemsFragment(&'static str); + +impl fmt::Display for UploadItemsFragment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "SELECT {0}.id, {0}.guid, {}.syncChangeCounter, + p.guid AS parentGuid, p.title AS parentTitle, + {0}.dateAdded / 1000 AS dateAdded, {0}.type, {0}.title, + h.id AS placeId, + IFNULL(substr(h.url, 1, 6) = 'place:', 0) AS isQuery, + h.url, + (SELECT keyword FROM moz_keywords WHERE place_id = h.id), + {0}.position, + (SELECT get_query_param(substr(url, 7), 'tag') + WHERE substr(h.url, 1, 6) = 'place:') AS tagFolderName + FROM moz_bookmarks {0} + JOIN moz_bookmarks p ON p.id = {0}.parent + LEFT JOIN moz_places h ON h.id = {0}.fk", + self.0 + ) + } +} + +pub enum ApplyStatus { + Merged, + Skipped, +} + +impl From for bool { + fn from(status: ApplyStatus) -> bool { + match status { + ApplyStatus::Merged => true, + ApplyStatus::Skipped => false, + } + } +} diff --git a/toolkit/components/places/components.conf b/toolkit/components/places/components.conf new file mode 100644 index 0000000000..ee5a4d0bbd --- /dev/null +++ b/toolkit/components/places/components.conf @@ -0,0 +1,128 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'name': 'History', + 'cid': '{0937a705-91a6-417a-8292-b22eb10da86c}', + 'contract_ids': ['@mozilla.org/browser/history;1'], + 'singleton': True, + 'type': 'mozilla::places::History', + 'headers': ['mozilla/places/History.h'], + 'constructor': 'mozilla::places::History::GetSingleton', + }, + { + 'cid': '{e8b8bdb7-c96c-4d82-9c6f-2b3c585ec7ea}', + 'contract_ids': ['@mozilla.org/network/protocol;1?name=moz-anno'], + 'type': 'nsAnnoProtocolHandler', + 'headers': ['/toolkit/components/places/nsAnnoProtocolHandler.h'], + 'protocol_config': { + 'scheme': 'moz-anno', + 'flags': [ + 'URI_NORELATIVE', + 'URI_NOAUTH', + 'URI_DANGEROUS_TO_LOAD', + 'URI_IS_LOCAL_RESOURCE', + ], + }, + }, + { + 'cid': '{984e3259-9266-49cf-b605-60b022a00756}', + 'contract_ids': ['@mozilla.org/browser/favicon-service;1'], + 'singleton': True, + 'type': 'nsFaviconService', + 'headers': ['/toolkit/components/places/nsFaviconService.h'], + 'constructor': 'nsFaviconService::GetSingleton', + }, + { + 'cid': '{9de95a0c-39a4-4d64-9a53-17940dd7cabb}', + 'contract_ids': ['@mozilla.org/browser/nav-bookmarks-service;1'], + 'singleton': True, + 'type': 'nsNavBookmarks', + 'headers': ['/toolkit/components/places/nsNavBookmarks.h'], + 'constructor': 'nsNavBookmarks::GetSingleton', + }, + { + 'cid': '{88cecbb7-6c63-4b3b-8cd4-84f3b8228c69}', + 'contract_ids': ['@mozilla.org/browser/nav-history-service;1'], + 'singleton': True, + 'type': 'nsNavHistory', + 'headers': ['/toolkit/components/places/nsNavHistory.h'], + 'constructor': 'nsNavHistory::GetSingleton', + 'categories': {'vacuum-participant': 'Places'}, + }, + + { + 'cid': '{bbc23860-2553-479d-8b78-94d9038334f7}', + 'contract_ids': ['@mozilla.org/browser/tagging-service;1'], + 'esModule': 'resource://gre/modules/TaggingService.sys.mjs', + 'constructor': 'TaggingService', + }, + { + 'cid': '{1dcc23b0-d4cb-11dc-9ad6-479d56d89593}', + 'contract_ids': ['@mozilla.org/autocomplete/search;1?name=places-tag-autocomplete'], + 'esModule': 'resource://gre/modules/TaggingService.sys.mjs', + 'constructor': 'TagAutoCompleteSearch', + }, + + { + 'cid': '{705a423f-2f69-42f3-b9fe-1517e0dee56f}', + 'contract_ids': ['@mozilla.org/places/expiration;1'], + 'esModule': 'resource://gre/modules/PlacesExpiration.sys.mjs', + 'constructor': 'nsPlacesExpiration', + 'categories': {'places-init-complete': 'nsPlacesExpiration'}, + }, + + { + 'cid': '{d38926e0-29c1-11eb-8588-0800200c9a66}', + 'contract_ids': ['@mozilla.org/places/databaseUtilsIdleMaintenance;1'], + 'esModule': 'resource://gre/modules/PlacesDBUtils.sys.mjs', + 'constructor': 'PlacesDBUtilsIdleMaintenance', + 'categories': {'idle-daily': 'PlacesDBUtilsIdleMaintenance'}, + }, + + { + 'cid': '{60a1f7c6-4ff9-4a42-84d3-5a185faa6f32}', + 'contract_ids': ['@mozilla.org/network/protocol;1?name=page-icon'], + 'singleton': True, + 'type': 'mozilla::places::PageIconProtocolHandler', + 'headers': ['mozilla/places/PageIconProtocolHandler.h'], + 'constructor': 'mozilla::places::PageIconProtocolHandler::GetSingleton', + 'protocol_config': { + 'scheme': 'page-icon', + 'flags': [ + 'URI_STD', + 'URI_IS_UI_RESOURCE', + 'URI_IS_LOCAL_RESOURCE', + 'URI_NORELATIVE', + 'URI_NOAUTH', + ], + }, + }, + + { + 'cid': '{7d47b41d-7cc5-4882-b293-d8f3f3b48b46}', + 'contract_ids': ['@mozilla.org/browser/synced-bookmarks-merger;1'], + 'type': 'mozISyncedBookmarksMerger', + 'headers': ['mozilla/places/SyncedBookmarksMirror.h'], + 'constructor': 'mozilla::places::NewSyncedBookmarksMerger', + }, + + { + 'cid': '{bd0a4d3b-ff26-4d4d-9a62-a513e1c1bf92}', + 'contract_ids': ['@mozilla.org/places/previews-helper;1'], + 'esModule': 'resource://gre/modules/PlacesPreviews.sys.mjs', + 'constructor': 'PlacesPreviewsHelperService', + }, + + { + 'cid': '{1141fd31-4c1a-48eb-8f1a-2f05fad94085}', + 'contract_ids': ['@mozilla.org/places/frecency-recalculator;1'], + 'esModule': 'resource://gre/modules/PlacesFrecencyRecalculator.sys.mjs', + 'constructor': 'PlacesFrecencyRecalculator', + 'categories': {'places-init-complete': 'PlacesFrecencyRecalculator'}, + }, +] diff --git a/toolkit/components/places/moz.build b/toolkit/components/places/moz.build new file mode 100644 index 0000000000..80a0845cd4 --- /dev/null +++ b/toolkit/components/places/moz.build @@ -0,0 +1,88 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if CONFIG["MOZ_PLACES"]: + TEST_DIRS += ["tests"] + +XPIDL_SOURCES += [ + "nsINavHistoryService.idl", +] + +XPIDL_MODULE = "places" + +if CONFIG["MOZ_PLACES"]: + XPIDL_SOURCES += [ + "mozIAsyncHistory.idl", + "mozIPlacesAutoComplete.idl", + "mozIPlacesPendingOperation.idl", + "mozISyncedBookmarksMirror.idl", + "nsIFaviconService.idl", + "nsINavBookmarksService.idl", + "nsIPlacesPreviewsHelperService.idl", + "nsITaggingService.idl", + ] + + EXPORTS.mozilla.places = [ + "Database.h", + "History.h", + "INativePlacesEventCallback.h", + "NotifyRankingChanged.h", + "PageIconProtocolHandler.h", + "Shutdown.h", + "SyncedBookmarksMirror.h", + ] + + UNIFIED_SOURCES += [ + "Database.cpp", + "FaviconHelpers.cpp", + "Helpers.cpp", + "History.cpp", + "nsAnnoProtocolHandler.cpp", + "nsFaviconService.cpp", + "nsNavBookmarks.cpp", + "nsNavHistory.cpp", + "nsNavHistoryQuery.cpp", + "nsNavHistoryResult.cpp", + "PageIconProtocolHandler.cpp", + "PlaceInfo.cpp", + "Shutdown.cpp", + "SQLFunctions.cpp", + "VisitInfo.cpp", + ] + + LOCAL_INCLUDES += [ + "../build", + ] + + EXTRA_JS_MODULES += [ + "BookmarkHTMLUtils.sys.mjs", + "BookmarkJSONUtils.sys.mjs", + "Bookmarks.sys.mjs", + "ExtensionSearchHandler.sys.mjs", + "History.sys.mjs", + "PlacesBackups.sys.mjs", + "PlacesDBUtils.sys.mjs", + "PlacesExpiration.sys.mjs", + "PlacesFrecencyRecalculator.sys.mjs", + "PlacesPreviews.sys.mjs", + "PlacesQuery.sys.mjs", + "PlacesSyncUtils.sys.mjs", + "PlacesTransactions.sys.mjs", + "PlacesUtils.sys.mjs", + "SyncedBookmarksMirror.sys.mjs", + "TaggingService.sys.mjs", + ] + + XPCOM_MANIFESTS += [ + "components.conf", + ] + + FINAL_LIBRARY = "xul" + +include("/ipc/chromium/chromium-config.mozbuild") + +with Files("**"): + BUG_COMPONENT = ("Toolkit", "Places") diff --git a/toolkit/components/places/mozIAsyncHistory.idl b/toolkit/components/places/mozIAsyncHistory.idl new file mode 100644 index 0000000000..90c32b1090 --- /dev/null +++ b/toolkit/components/places/mozIAsyncHistory.idl @@ -0,0 +1,186 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; +interface nsIVariant; + +[scriptable, uuid(41e4ccc9-f0c8-4cd7-9753-7a38514b8488)] +interface mozIVisitInfo : nsISupports +{ + /** + * The machine-local (internal) id of the visit. + */ + readonly attribute long long visitId; + + /** + * The time the visit occurred. + */ + readonly attribute PRTime visitDate; + + /** + * The transition type used to get to this visit. One of the TRANSITION_TYPE + * constants on nsINavHistory. + * + * @see nsINavHistory.idl + */ + readonly attribute unsigned long transitionType; + + /** + * The referring URI of this visit. This may be null. + */ + readonly attribute nsIURI referrerURI; +}; + +[scriptable, uuid(ad83e137-c92a-4b7b-b67e-0a318811f91e)] +interface mozIPlaceInfo : nsISupports +{ + /** + * The machine-local (internal) id of the place. + */ + readonly attribute long long placeId; + + /** + * The globally unique id of the place. + */ + readonly attribute ACString guid; + + /** + * The URI of the place. + */ + readonly attribute nsIURI uri; + + /** + * The title associated with the place. + */ + readonly attribute AString title; + + /** + * The frecency of the place. + */ + readonly attribute long long frecency; + + /** + * An array of mozIVisitInfo objects for the place. + */ + [implicit_jscontext] + readonly attribute jsval visits; +}; + +/** + * Shared Callback interface for mozIAsyncHistory methods. The semantics + * for each method are detailed in mozIAsyncHistory. + */ +[scriptable, uuid(1f266877-2859-418b-a11b-ec3ae4f4f93d)] +interface mozIVisitInfoCallback : nsISupports +{ + /** + * Called when the given place could not be processed. + * + * @param aResultCode + * nsresult indicating the failure reason. + * @param aPlaceInfo + * The information that was given to the caller for the place. + */ + void handleError(in nsresult aResultCode, + in mozIPlaceInfo aPlaceInfo); + + /** + * Called for each place processed successfully. + * + * @param aPlaceInfo + * The current info stored for the place. + */ + void handleResult(in mozIPlaceInfo aPlaceInfo); + + /** + * Called when all records were processed. + * @param aUpdatedItems + * How many items were successfully updated. + */ + void handleCompletion(in unsigned long aUpdatedItems); + + /** + * These two attributes govern whether we attempt to call + * handleResult and handleError, respectively, if/once + * results/errors occur. + */ + readonly attribute bool ignoreResults; + readonly attribute bool ignoreErrors; +}; + +[scriptable, function, uuid(994092bf-936f-449b-8dd6-0941e024360d)] +interface mozIVisitedStatusCallback : nsISupports +{ + /** + * Notifies whether a certain URI has been visited. + * + * @param aURI + * URI being notified about. + * @param aVisitedStatus + * The visited status of aURI. + */ + void isVisited(in nsIURI aURI, + in boolean aVisitedStatus); +}; + +/** + * This interface contains APIs for cpp consumers. + * Javascript consumers should look at History.jsm instead, + * that is exposed through PlacesUtils.history. + * + * If you're evaluating adding a new history API, it should + * usually go to History.jsm, unless it needs to do long and + * expensive work in a batch, then it could be worth doing + * that in History.cpp. + */ + +[scriptable, uuid(1643EFD2-A329-4733-A39D-17069C8D3B2D)] +interface mozIAsyncHistory : nsISupports +{ + /** + * Adds a set of visits for one or more mozIPlaceInfo objects, and updates + * each mozIPlaceInfo's title or guid. + * + * aCallback.handleResult is called for each visit added. + * + * @param aPlaceInfo + * The mozIPlaceInfo object[s] containing the information to store or + * update. This can be a single object, or an array of objects. + * @param [optional] aCallback + * A mozIVisitInfoCallback object which consists of callbacks to be + * notified for successful and/or failed changes. + * + * @throws NS_ERROR_INVALID_ARG + * - Passing in NULL for aPlaceInfo. + * - Not providing at least one valid guid, or uri for all + * mozIPlaceInfo object[s]. + * - Not providing an array or nothing for the visits property of + * mozIPlaceInfo. + * - Not providing a visitDate and transitionType for each + * mozIVisitInfo. + * - Providing an invalid transitionType for a mozIVisitInfo. + */ + [implicit_jscontext] + void updatePlaces(in jsval aPlaceInfo, + [optional] in mozIVisitInfoCallback aCallback); + + /** + * Checks if a given URI has been visited. + * + * @param aURI + * The URI to check for. + * @param aCallback + * A mozIVisitStatusCallback object which receives the visited status. + */ + void isURIVisited(in nsIURI aURI, + in mozIVisitedStatusCallback aCallback); + + /** + * Helper to clear any internal state caches, like the recent URIs list. + * This may be useful in testing code. + */ + void clearCache(); +}; diff --git a/toolkit/components/places/mozIPlacesAutoComplete.idl b/toolkit/components/places/mozIPlacesAutoComplete.idl new file mode 100644 index 0000000000..1b05c55c8f --- /dev/null +++ b/toolkit/components/places/mozIPlacesAutoComplete.idl @@ -0,0 +1,113 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 sts=2 + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; + +/** + * This interface provides some constants used by the Places AutoComplete + * search provider as well as methods to track opened pages for AutoComplete + * purposes. + */ +[scriptable, uuid(61b6348a-09e1-4810-8057-f8cb3cec6ef8)] +interface mozIPlacesAutoComplete : nsISupports +{ + ////////////////////////////////////////////////////////////////////////////// + //// Matching Constants + + // A few of these are not used in Firefox, but are still referenced in + // comm-central. + + /** + * Match anywhere in each searchable term. + */ + const long MATCH_ANYWHERE = 0; + + /** + * Match first on word boundaries, and if we do not get enough results, then + * match anywhere in each searchable term. + */ + const long MATCH_BOUNDARY_ANYWHERE = 1; + + /** + * Match on word boundaries in each searchable term. + */ + const long MATCH_BOUNDARY = 2; + + /** + * Match only the beginning of each search term. + */ + const long MATCH_BEGINNING = 3; + + /** + * Match anywhere in each searchable term without doing any transformation + * or stripping on the underlying data. + */ + const long MATCH_ANYWHERE_UNMODIFIED = 4; + + /** + * Match only the beginning of each search term using a case sensitive + * comparator. + */ + const long MATCH_BEGINNING_CASE_SENSITIVE = 5; + + ////////////////////////////////////////////////////////////////////////////// + //// Search Behavior Constants + + /** + * Search through history. + */ + const long BEHAVIOR_HISTORY = 1 << 0; + + /** + * Search though bookmarks. + */ + const long BEHAVIOR_BOOKMARK = 1 << 1; + + /** + * Search through tags. + */ + const long BEHAVIOR_TAG = 1 << 2; + + /** + * Search the title of pages. + */ + const long BEHAVIOR_TITLE = 1 << 3; + + /** + * Search the URL of pages. + */ + const long BEHAVIOR_URL = 1 << 4; + + /** + * Search for typed pages. + * No more supported by Firefox, it is still being used by comm-central clients. + */ + const long BEHAVIOR_TYPED = 1 << 5; + + /** + * Search javascript: URLs. + */ + const long BEHAVIOR_JAVASCRIPT = 1 << 6; + + /** + * Search for pages that have been marked as being opened, such as a tab + * in a tabbrowser. + */ + const long BEHAVIOR_OPENPAGE = 1 << 7; + + /** + * Use intersection between history, typed, bookmark, tag and openpage + * instead of union, when the restrict bit is set. + */ + const long BEHAVIOR_RESTRICT = 1 << 8; + + /** + * Include search suggestions from the currently selected search provider. + */ + const long BEHAVIOR_SEARCH = 1 << 9; +}; diff --git a/toolkit/components/places/mozIPlacesPendingOperation.idl b/toolkit/components/places/mozIPlacesPendingOperation.idl new file mode 100644 index 0000000000..678a908708 --- /dev/null +++ b/toolkit/components/places/mozIPlacesPendingOperation.idl @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +[scriptable, uuid(ebd31374-3808-40e4-9e73-303bf70467c3)] +interface mozIPlacesPendingOperation : nsISupports { + /** + * Cancels a pending operation, if possible. This will only fail if you try + * to cancel more than once. + */ + void cancel(); +}; diff --git a/toolkit/components/places/mozISyncedBookmarksMirror.idl b/toolkit/components/places/mozISyncedBookmarksMirror.idl new file mode 100644 index 0000000000..0ceaeff5ad --- /dev/null +++ b/toolkit/components/places/mozISyncedBookmarksMirror.idl @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozIServicesLogSink.idl" +#include "nsISupports.idl" + +interface mozIPlacesPendingOperation; +interface mozIStorageConnection; +interface nsIPropertyBag; + +// A progress listener used to record telemetry. +[scriptable, uuid(6239ffe3-6ffd-49ac-8b1d-958407395bf9)] +interface mozISyncedBookmarksMirrorProgressListener : nsISupports { + // Called after the merger fetches the local tree from Places, with the time + // taken to build the tree, number of items in the tree, and a map of + // structure problem counts for validation telemetry. + void onFetchLocalTree(in long long took, in long long itemCount, + in long long deletedCount, in nsIPropertyBag problems); + + // Called after the merger builds the remote tree from the mirror database. + void onFetchRemoteTree(in long long took, in long long itemCount, + in long long deletedCount, in nsIPropertyBag problems); + + // Called after the merger builds the merged tree, including structure change + // counts for event telemetry. + void onMerge(in long long took, in nsIPropertyBag counts); + + // Called after the merger finishes applying the merged tree to Places and + // staging outgoing items. + void onApply(in long long took); +}; + +// A callback called when the merge finishes. +[scriptable, uuid(d23fdfea-92c8-409d-a516-08ae395d578f)] +interface mozISyncedBookmarksMirrorCallback : nsISupports { + void handleSuccess(in bool result); + void handleError(in nsresult code, in AString message); +}; + +[scriptable, uuid(37485984-a6ab-46e3-9b0c-e8b613413ef3)] +interface mozISyncedBookmarksMirrorLogger : nsISupports { + const short LEVEL_OFF = 0; + const short LEVEL_ERROR = 1; + const short LEVEL_WARN = 2; + const short LEVEL_DEBUG = 3; + const short LEVEL_TRACE = 4; + + attribute short maxLevel; + + void error(in AString message); + void warn(in AString message); + void debug(in AString message); + void trace(in AString message); +}; + +[scriptable, builtinclass, uuid(f0a6217d-8344-4e68-9995-bbf5554be86e)] +interface mozISyncedBookmarksMerger : nsISupports { + // Synced item kinds. These are stored in the mirror database. + const short KIND_BOOKMARK = 1; + const short KIND_QUERY = 2; + const short KIND_FOLDER = 3; + const short KIND_LIVEMARK = 4; + const short KIND_SEPARATOR = 5; + + // Synced item validity states. These are also stored in the mirror + // database. `REUPLOAD` means a remote item can be fixed up and applied, + // and should be reuploaded. `REPLACE` means a remote item isn't valid + // at all, and should either be replaced with a valid local copy, or deleted + // if a valid local copy doesn't exist. + const short VALIDITY_VALID = 1; + const short VALIDITY_REUPLOAD = 2; + const short VALIDITY_REPLACE = 3; + + // The mirror database connection to use for merging. The merge runs on the + // connection's async thread, to avoid blocking the main thread. The + // connection must be open, and the database schema, temp tables, and + // triggers must be set up before calling `merge`. + attribute mozIStorageConnection db; + + // Optional; used for logging. + attribute mozIServicesLogSink logger; + + // Merges the local and remote bookmark trees, applies the merged tree to + // Places, and stages locally changed and reconciled items for upload. When + // the merge finishes, either `callback.handleSuccess` or + // `callback.handleError` are called. If `callback` also implements + // `mozISyncedBookmarksMergerProgressListener`, the merger reports progress + // after each step. Returns an object with a `cancel` method that can be used + // to interrupt the merge. + mozIPlacesPendingOperation merge( + in long long localTimeSeconds, + in long long remoteTimeSeconds, + in mozISyncedBookmarksMirrorCallback callback + ); + + // Resets the database connection and logger. This does _not_ automatically + // close the database connection. + void reset(); +}; diff --git a/toolkit/components/places/nsAnnoProtocolHandler.cpp b/toolkit/components/places/nsAnnoProtocolHandler.cpp new file mode 100644 index 0000000000..c3ad2e6410 --- /dev/null +++ b/toolkit/components/places/nsAnnoProtocolHandler.cpp @@ -0,0 +1,359 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Implementation of moz-anno: URLs for accessing favicons. The urls are sent + * to the favicon service. If the favicon service doesn't have the + * data, a stream containing the default favicon will be returned. + * + * The reference to annotations ("moz-anno") is a leftover from previous + * iterations of this component. As of now the moz-anno protocol is independent + * of annotations. + */ + +#include "nsAnnoProtocolHandler.h" +#include "nsFaviconService.h" +#include "nsICancelable.h" +#include "nsIChannel.h" +#include "nsIInputStream.h" +#include "nsISupportsUtils.h" +#include "nsIURI.h" +#include "nsNetUtil.h" +#include "nsInputStreamPump.h" +#include "nsContentUtils.h" +#include "nsServiceManagerUtils.h" +#include "nsStringStream.h" +#include "SimpleChannel.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/storage.h" +#include "mozIStorageResultSet.h" +#include "mozIStorageRow.h" +#include "Helpers.h" +#include "FaviconHelpers.h" + +using namespace mozilla; +using namespace mozilla::places; + +//////////////////////////////////////////////////////////////////////////////// +//// Global Functions + +/** + * Creates a channel to obtain the default favicon. + */ +static nsresult GetDefaultIcon(nsIChannel* aOriginalChannel, + nsIChannel** aChannel) { + nsCOMPtr defaultIconURI; + nsresult rv = NS_NewURI(getter_AddRefs(defaultIconURI), + nsLiteralCString(FAVICON_DEFAULT_URL)); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr loadInfo = aOriginalChannel->LoadInfo(); + rv = NS_NewChannelInternal(aChannel, defaultIconURI, loadInfo); + NS_ENSURE_SUCCESS(rv, rv); + Unused << (*aChannel)->SetContentType( + nsLiteralCString(FAVICON_DEFAULT_MIMETYPE)); + Unused << aOriginalChannel->SetContentType( + nsLiteralCString(FAVICON_DEFAULT_MIMETYPE)); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// faviconAsyncLoader + +namespace { + +/** + * An instance of this class is passed to the favicon service as the callback + * for getting favicon data from the database. We'll get this data back in + * HandleResult, and on HandleCompletion, we'll close our output stream which + * will close the original channel for the favicon request. + * + * However, if an error occurs at any point and we don't have mData, we will + * just fallback to the default favicon. If anything happens at that point, the + * world must be against us, so we can do nothing. + */ +class faviconAsyncLoader : public AsyncStatementCallback, public nsICancelable { + NS_DECL_NSICANCELABLE + NS_DECL_ISUPPORTS_INHERITED + + public: + faviconAsyncLoader(nsIChannel* aChannel, nsIStreamListener* aListener, + uint16_t aPreferredSize) + : mChannel(aChannel), + mListener(aListener), + mPreferredSize(aPreferredSize) { + MOZ_ASSERT(aChannel, "Not providing a channel will result in crashes!"); + MOZ_ASSERT(aListener, + "Not providing a stream listener will result in crashes!"); + MOZ_ASSERT(aChannel, "Not providing a channel!"); + } + + ////////////////////////////////////////////////////////////////////////////// + //// mozIStorageStatementCallback + + NS_IMETHOD HandleResult(mozIStorageResultSet* aResultSet) override { + nsCOMPtr row; + while (NS_SUCCEEDED(aResultSet->GetNextRow(getter_AddRefs(row))) && row) { + int32_t width; + nsresult rv = row->GetInt32(1, &width); + NS_ENSURE_SUCCESS(rv, rv); + + // Check if we already found an image >= than the preferred size, + // otherwise keep examining the next results. + if (width < mPreferredSize && !mData.IsEmpty()) { + return NS_OK; + } + + // Eventually override the default mimeType for svg. + if (width == UINT16_MAX) { + rv = mChannel->SetContentType(nsLiteralCString(SVG_MIME_TYPE)); + } else { + rv = mChannel->SetContentType(nsLiteralCString(PNG_MIME_TYPE)); + } + NS_ENSURE_SUCCESS(rv, rv); + + // Obtain the binary blob that contains our favicon data. + uint8_t* data; + uint32_t dataLen; + rv = row->GetBlob(0, &dataLen, &data); + NS_ENSURE_SUCCESS(rv, rv); + mData.Adopt(TO_CHARBUFFER(data), dataLen); + } + + return NS_OK; + } + + static void CancelRequest(nsIStreamListener* aListener, nsIChannel* aChannel, + nsresult aResult) { + MOZ_ASSERT(aListener); + MOZ_ASSERT(aChannel); + + aListener->OnStartRequest(aChannel); + aListener->OnStopRequest(aChannel, aResult); + aChannel->CancelWithReason(NS_BINDING_ABORTED, + "faviconAsyncLoader::CancelRequest"_ns); + } + + NS_IMETHOD HandleCompletion(uint16_t aReason) override { + MOZ_DIAGNOSTIC_ASSERT(mListener); + MOZ_ASSERT(mChannel); + NS_ENSURE_TRUE(mListener, NS_ERROR_UNEXPECTED); + NS_ENSURE_TRUE(mChannel, NS_ERROR_UNEXPECTED); + + // Ensure we'll break possible cycles with the listener. + auto cleanup = MakeScopeExit([&]() { + mListener = nullptr; + mChannel = nullptr; + }); + + if (mCanceled) { + // The channel that has created this faviconAsyncLoader has been canceled. + CancelRequest(mListener, mChannel, mStatus); + return NS_OK; + } + + nsresult rv; + + nsCOMPtr loadInfo = mChannel->LoadInfo(); + nsCOMPtr target = + nsContentUtils::GetEventTargetByLoadInfo(loadInfo, TaskCategory::Other); + if (!mData.IsEmpty()) { + nsCOMPtr stream; + rv = NS_NewCStringInputStream(getter_AddRefs(stream), mData); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + if (NS_SUCCEEDED(rv)) { + RefPtr pump; + rv = nsInputStreamPump::Create(getter_AddRefs(pump), stream, 0, 0, true, + target); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + if (NS_SUCCEEDED(rv)) { + rv = pump->AsyncRead(mListener); + if (NS_FAILED(rv)) { + CancelRequest(mListener, mChannel, rv); + return rv; + } + + mPump = pump; + return NS_OK; + } + } + } + + // Fallback to the default favicon. + // we should pass the loadInfo of the original channel along + // to the new channel. Note that mChannel can not be null, + // constructor checks that. + rv = GetDefaultIcon(mChannel, getter_AddRefs(mDefaultIconChannel)); + if (NS_FAILED(rv)) { + CancelRequest(mListener, mChannel, rv); + return rv; + } + + rv = mDefaultIconChannel->AsyncOpen(mListener); + if (NS_FAILED(rv)) { + mDefaultIconChannel = nullptr; + CancelRequest(mListener, mChannel, rv); + return rv; + } + + return NS_OK; + } + + protected: + virtual ~faviconAsyncLoader() = default; + + private: + nsCOMPtr mChannel; + nsCOMPtr mDefaultIconChannel; + nsCOMPtr mListener; + nsCOMPtr mPump; + nsCString mData; + uint16_t mPreferredSize; + bool mCanceled{false}; + nsresult mStatus{NS_OK}; +}; + +NS_IMPL_ISUPPORTS_INHERITED(faviconAsyncLoader, AsyncStatementCallback, + nsICancelable) + +NS_IMETHODIMP +faviconAsyncLoader::Cancel(nsresult aStatus) { + if (mCanceled) { + return NS_OK; + } + + mCanceled = true; + mStatus = aStatus; + + if (mPump) { + mPump->Cancel(aStatus); + mPump = nullptr; + } + + if (mDefaultIconChannel) { + mDefaultIconChannel->Cancel(aStatus); + mDefaultIconChannel = nullptr; + } + + return NS_OK; +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// +//// nsAnnoProtocolHandler + +NS_IMPL_ISUPPORTS(nsAnnoProtocolHandler, nsIProtocolHandler) + +// nsAnnoProtocolHandler::GetScheme + +NS_IMETHODIMP +nsAnnoProtocolHandler::GetScheme(nsACString& aScheme) { + aScheme.AssignLiteral("moz-anno"); + return NS_OK; +} + +// nsAnnoProtocolHandler::NewChannel +// + +NS_IMETHODIMP +nsAnnoProtocolHandler::NewChannel(nsIURI* aURI, nsILoadInfo* aLoadInfo, + nsIChannel** _retval) { + NS_ENSURE_ARG_POINTER(aURI); + + // annotation info + nsCOMPtr annoURI; + nsAutoCString annoName; + nsresult rv = ParseAnnoURI(aURI, getter_AddRefs(annoURI), annoName); + NS_ENSURE_SUCCESS(rv, rv); + + // Only favicon annotation are supported. + if (!annoName.EqualsLiteral(FAVICON_ANNOTATION_NAME)) + return NS_ERROR_INVALID_ARG; + + return NewFaviconChannel(aURI, annoURI, aLoadInfo, _retval); +} + +// nsAnnoProtocolHandler::AllowPort +// +// Don't override any bans on bad ports. + +NS_IMETHODIMP +nsAnnoProtocolHandler::AllowPort(int32_t port, const char* scheme, + bool* _retval) { + *_retval = false; + return NS_OK; +} + +// nsAnnoProtocolHandler::ParseAnnoURI +// +// Splits an annotation URL into its URI and name parts + +nsresult nsAnnoProtocolHandler::ParseAnnoURI(nsIURI* aURI, nsIURI** aResultURI, + nsCString& aName) { + nsresult rv; + nsAutoCString path; + rv = aURI->GetPathQueryRef(path); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t firstColon = path.FindChar(':'); + if (firstColon <= 0) return NS_ERROR_MALFORMED_URI; + + rv = NS_NewURI(aResultURI, Substring(path, firstColon + 1)); + NS_ENSURE_SUCCESS(rv, rv); + + aName = Substring(path, 0, firstColon); + return NS_OK; +} + +nsresult nsAnnoProtocolHandler::NewFaviconChannel(nsIURI* aURI, + nsIURI* aAnnotationURI, + nsILoadInfo* aLoadInfo, + nsIChannel** _channel) { + // Create our channel. We'll call SetContentType with the right type when + // we know what it actually is. + nsCOMPtr channel = NS_NewSimpleChannel( + aURI, aLoadInfo, aAnnotationURI, + [](nsIStreamListener* listener, nsIChannel* channel, + nsIURI* annotationURI) -> RequestOrReason { + auto fallback = [&]() -> RequestOrReason { + nsCOMPtr chan; + nsresult rv = GetDefaultIcon(channel, getter_AddRefs(chan)); + NS_ENSURE_SUCCESS(rv, Err(rv)); + + rv = chan->AsyncOpen(listener); + NS_ENSURE_SUCCESS(rv, Err(rv)); + + nsCOMPtr request(chan); + return RequestOrCancelable(WrapNotNull(request)); + }; + + // Now we go ahead and get our data asynchronously for the favicon. + // Ignore the ref part of the URI before querying the database because + // we may have added a size fragment for rendering purposes. + nsFaviconService* faviconService = + nsFaviconService::GetFaviconService(); + nsAutoCString faviconSpec; + nsresult rv = annotationURI->GetSpecIgnoringRef(faviconSpec); + // Any failures fallback to the default icon channel. + if (NS_FAILED(rv) || !faviconService) return fallback(); + + uint16_t preferredSize = UINT16_MAX; + MOZ_ALWAYS_SUCCEEDS(faviconService->PreferredSizeFromURI( + annotationURI, &preferredSize)); + nsCOMPtr callback = + new faviconAsyncLoader(channel, listener, preferredSize); + if (!callback) return fallback(); + + rv = faviconService->GetFaviconDataAsync(faviconSpec, callback); + if (NS_FAILED(rv)) return fallback(); + + nsCOMPtr cancelable = do_QueryInterface(callback); + return RequestOrCancelable(WrapNotNull(cancelable)); + }); + NS_ENSURE_TRUE(channel, NS_ERROR_OUT_OF_MEMORY); + + channel.forget(_channel); + return NS_OK; +} diff --git a/toolkit/components/places/nsAnnoProtocolHandler.h b/toolkit/components/places/nsAnnoProtocolHandler.h new file mode 100644 index 0000000000..8f526f3398 --- /dev/null +++ b/toolkit/components/places/nsAnnoProtocolHandler.h @@ -0,0 +1,56 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsAnnoProtocolHandler_h___ +#define nsAnnoProtocolHandler_h___ + +#include "nsCOMPtr.h" +#include "nsIProtocolHandler.h" +#include "nsIURI.h" +#include "nsString.h" +#include "nsWeakReference.h" +#include "mozilla/Attributes.h" + +// {e8b8bdb7-c96c-4d82-9c6f-2b3c585ec7ea} +#define NS_ANNOPROTOCOLHANDLER_CID \ + { \ + 0xe8b8bdb7, 0xc96c, 0x4d82, { \ + 0x9c, 0x6f, 0x2b, 0x3c, 0x58, 0x5e, 0xc7, 0xea \ + } \ + } + +class nsAnnoProtocolHandler final : public nsIProtocolHandler, + public nsSupportsWeakReference { + public: + nsAnnoProtocolHandler() = default; + + NS_DECL_ISUPPORTS + NS_DECL_NSIPROTOCOLHANDLER + + private: + ~nsAnnoProtocolHandler() = default; + + protected: + nsresult ParseAnnoURI(nsIURI* aURI, nsIURI** aResultURI, nsCString& aName); + + /** + * Obtains a new channel to be used to get a favicon from the database. This + * method is asynchronous. + * + * @param aURI + * The URI the channel will be created for. This is the URI that is + * set as the original URI on the channel. + * @param aAnnotationURI + * The URI that holds the data needed to get the favicon from the + * database. + * @param aLoadInfo + * The loadinfo that requested the resource load. + * @returns (via _channel) the channel that will obtain the favicon data. + */ + nsresult NewFaviconChannel(nsIURI* aURI, nsIURI* aAnnotationURI, + nsILoadInfo* aLoadInfo, nsIChannel** _channel); +}; + +#endif /* nsAnnoProtocolHandler_h___ */ diff --git a/toolkit/components/places/nsFaviconService.cpp b/toolkit/components/places/nsFaviconService.cpp new file mode 100644 index 0000000000..cf708f136a --- /dev/null +++ b/toolkit/components/places/nsFaviconService.cpp @@ -0,0 +1,837 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This is the favicon service, which stores favicons for web pages with your + * history as you browse. It is also used to save the favicons for bookmarks. + * + * DANGER: The history query system makes assumptions about the favicon storage + * so that icons can be quickly generated for history/bookmark result sets. If + * you change the database layout at all, you will have to update both services. + */ + +#include "nsFaviconService.h" + +#include "nsNavHistory.h" +#include "nsPlacesMacros.h" +#include "Helpers.h" + +#include "nsNetUtil.h" +#include "nsReadableUtils.h" +#include "nsStreamUtils.h" +#include "plbase64.h" +#include "nsIClassInfoImpl.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/LoadInfo.h" +#include "mozilla/NullPrincipal.h" +#include "mozilla/Preferences.h" +#include "nsILoadInfo.h" +#include "nsIContentPolicy.h" +#include "nsIProtocolHandler.h" +#include "nsIScriptError.h" +#include "nsContentUtils.h" +#include "imgICache.h" + +#define UNASSOCIATED_FAVICONS_LENGTH 32 + +// When replaceFaviconData is called, we store the icons in an in-memory cache +// instead of in storage. Icons in the cache are expired according to this +// interval. +#define UNASSOCIATED_ICON_EXPIRY_INTERVAL 60000 + +using namespace mozilla; +using namespace mozilla::places; + +const uint16_t gFaviconSizes[7] = {192, 144, 96, 64, 48, 32, 16}; + +/** + * Used to notify a topic to system observers on async execute completion. + * Will throw on error. + */ +class ExpireFaviconsStatementCallbackNotifier : public AsyncStatementCallback { + public: + ExpireFaviconsStatementCallbackNotifier(); + NS_IMETHOD HandleCompletion(uint16_t aReason) override; +}; + +namespace { + +/** + * Extracts and filters native sizes from the given container, based on the + * list of sizes we are supposed to retain. + * All calculation is done considering square sizes and the largest side. + * In case of multiple frames of the same size, only the first one is retained. + */ +nsresult GetFramesInfoForContainer(imgIContainer* aContainer, + nsTArray& aFramesInfo) { + // Don't extract frames from animated images. + bool animated; + nsresult rv = aContainer->GetAnimated(&animated); + if (NS_FAILED(rv) || !animated) { + nsTArray nativeSizes; + rv = aContainer->GetNativeSizes(nativeSizes); + if (NS_SUCCEEDED(rv) && nativeSizes.Length() > 1) { + for (uint32_t i = 0; i < nativeSizes.Length(); ++i) { + nsIntSize nativeSize = nativeSizes[i]; + // Only retain square frames. + if (nativeSize.width != nativeSize.height) { + continue; + } + // Check if it's one of the sizes we care about. + auto end = std::end(gFaviconSizes); + const uint16_t* matchingSize = + std::find(std::begin(gFaviconSizes), end, nativeSize.width); + if (matchingSize != end) { + // We must avoid duped sizes, an image could contain multiple frames + // of the same size, but we can only store one. We could use an + // hashtable, but considered the average low number of frames, we'll + // just do a linear search. + bool dupe = false; + for (const auto& frameInfo : aFramesInfo) { + if (frameInfo.width == *matchingSize) { + dupe = true; + break; + } + } + if (!dupe) { + aFramesInfo.AppendElement(FrameData(i, *matchingSize)); + } + } + } + } + } + + if (aFramesInfo.Length() == 0) { + // Always have at least the default size. + int32_t width; + rv = aContainer->GetWidth(&width); + NS_ENSURE_SUCCESS(rv, rv); + int32_t height; + rv = aContainer->GetHeight(&height); + NS_ENSURE_SUCCESS(rv, rv); + // For non-square images, pick the largest side. + aFramesInfo.AppendElement(FrameData(0, std::max(width, height))); + } + return NS_OK; +} + +} // namespace + +PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsFaviconService, gFaviconService) + +NS_IMPL_CLASSINFO(nsFaviconService, nullptr, 0, NS_FAVICONSERVICE_CID) +NS_IMPL_ISUPPORTS_CI(nsFaviconService, nsIFaviconService, nsITimerCallback, + nsINamed) + +nsFaviconService::nsFaviconService() + : mUnassociatedIcons(UNASSOCIATED_FAVICONS_LENGTH), + mDefaultIconURIPreferredSize(UINT16_MAX) { + NS_ASSERTION(!gFaviconService, + "Attempting to create two instances of the service!"); + gFaviconService = this; +} + +nsFaviconService::~nsFaviconService() { + NS_ASSERTION(gFaviconService == this, + "Deleting a non-singleton instance of the service"); + if (gFaviconService == this) gFaviconService = nullptr; +} + +Atomic nsFaviconService::sLastInsertedIconId(0); + +void // static +nsFaviconService::StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId) { + MOZ_ASSERT(aTable.EqualsLiteral("moz_icons")); + sLastInsertedIconId = aLastInsertedId; +} + +nsresult nsFaviconService::Init() { + mDB = Database::GetDatabase(); + NS_ENSURE_STATE(mDB); + + mExpireUnassociatedIconsTimer = NS_NewTimer(); + NS_ENSURE_STATE(mExpireUnassociatedIconsTimer); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::ExpireAllFavicons() { + NS_ENSURE_STATE(mDB); + + nsCOMPtr removePagesStmt = + mDB->GetAsyncStatement("DELETE FROM moz_pages_w_icons"); + NS_ENSURE_STATE(removePagesStmt); + nsCOMPtr removeIconsStmt = + mDB->GetAsyncStatement("DELETE FROM moz_icons"); + NS_ENSURE_STATE(removeIconsStmt); + nsCOMPtr unlinkIconsStmt = + mDB->GetAsyncStatement("DELETE FROM moz_icons_to_pages"); + NS_ENSURE_STATE(unlinkIconsStmt); + + nsTArray> stmts = { + ToRefPtr(std::move(removePagesStmt)), + ToRefPtr(std::move(removeIconsStmt)), + ToRefPtr(std::move(unlinkIconsStmt))}; + nsCOMPtr conn = mDB->MainConn(); + if (!conn) { + return NS_ERROR_UNEXPECTED; + } + nsCOMPtr ps; + RefPtr callback = + new ExpireFaviconsStatementCallbackNotifier(); + return conn->ExecuteAsync(stmts, callback, getter_AddRefs(ps)); +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsITimerCallback + +NS_IMETHODIMP +nsFaviconService::Notify(nsITimer* timer) { + if (timer != mExpireUnassociatedIconsTimer.get()) { + return NS_ERROR_INVALID_ARG; + } + + PRTime now = PR_Now(); + for (auto iter = mUnassociatedIcons.Iter(); !iter.Done(); iter.Next()) { + UnassociatedIconHashKey* iconKey = iter.Get(); + if (now - iconKey->created >= UNASSOCIATED_ICON_EXPIRY_INTERVAL) { + iter.Remove(); + } + } + + // Re-init the expiry timer if the cache isn't empty. + if (mUnassociatedIcons.Count() > 0) { + mExpireUnassociatedIconsTimer->InitWithCallback( + this, UNASSOCIATED_ICON_EXPIRY_INTERVAL, nsITimer::TYPE_ONE_SHOT); + } + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////// +//// nsINamed + +NS_IMETHODIMP +nsFaviconService::GetName(nsACString& aName) { + aName.AssignLiteral("nsFaviconService"); + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsIFaviconService + +NS_IMETHODIMP +nsFaviconService::GetDefaultFavicon(nsIURI** _retval) { + NS_ENSURE_ARG_POINTER(_retval); + + // not found, use default + if (!mDefaultIcon) { + nsresult rv = NS_NewURI(getter_AddRefs(mDefaultIcon), + nsLiteralCString(FAVICON_DEFAULT_URL)); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsCOMPtr uri = mDefaultIcon; + uri.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::GetDefaultFaviconMimeType(nsACString& _retval) { + _retval = nsLiteralCString(FAVICON_DEFAULT_MIMETYPE); + return NS_OK; +} + +void nsFaviconService::ClearImageCache(nsIURI* aImageURI) { + MOZ_ASSERT(aImageURI, "Must pass a non-null URI"); + nsCOMPtr imgCache; + nsresult rv = + GetImgTools()->GetImgCacheForDocument(nullptr, getter_AddRefs(imgCache)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + if (NS_SUCCEEDED(rv)) { + Unused << imgCache->RemoveEntry(aImageURI, nullptr); + } +} + +NS_IMETHODIMP +nsFaviconService::SetAndFetchFaviconForPage( + nsIURI* aPageURI, nsIURI* aFaviconURI, bool aForceReload, + uint32_t aFaviconLoadType, nsIFaviconDataCallback* aCallback, + nsIPrincipal* aLoadingPrincipal, uint64_t aRequestContextID, + mozIPlacesPendingOperation** _canceler) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aPageURI); + NS_ENSURE_ARG(aFaviconURI); + NS_ENSURE_ARG_POINTER(_canceler); + + nsCOMPtr loadingPrincipal = aLoadingPrincipal; + MOZ_ASSERT(loadingPrincipal, + "please provide aLoadingPrincipal for this favicon"); + if (!loadingPrincipal) { + // Let's default to the nullPrincipal if no loadingPrincipal is provided. + AutoTArray params = { + u"nsFaviconService::setAndFetchFaviconForPage()"_ns, + u"nsFaviconService::setAndFetchFaviconForPage(..., " + "[optional aLoadingPrincipal])"_ns}; + nsContentUtils::ReportToConsole( + nsIScriptError::warningFlag, "Security by Default"_ns, + nullptr, // aDocument + nsContentUtils::eNECKO_PROPERTIES, "APIDeprecationWarning", params); + loadingPrincipal = NullPrincipal::CreateWithoutOriginAttributes(); + } + NS_ENSURE_TRUE(loadingPrincipal, NS_ERROR_FAILURE); + + bool loadPrivate = + aFaviconLoadType == nsIFaviconService::FAVICON_LOAD_PRIVATE; + + // Build page data. + PageData page; + nsresult rv = aPageURI->GetSpec(page.spec); + NS_ENSURE_SUCCESS(rv, rv); + // URIs can arguably lack a host. + Unused << aPageURI->GetHost(page.host); + if (StringBeginsWith(page.host, "www."_ns)) { + page.host.Cut(0, 4); + } + bool canAddToHistory; + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(navHistory, NS_ERROR_OUT_OF_MEMORY); + rv = navHistory->CanAddURI(aPageURI, &canAddToHistory); + NS_ENSURE_SUCCESS(rv, rv); + page.canAddToHistory = !!canAddToHistory && !loadPrivate; + + // Build icon data. + IconData icon; + // If we have an in-memory icon payload, it overwrites the actual request. + UnassociatedIconHashKey* iconKey = mUnassociatedIcons.GetEntry(aFaviconURI); + if (iconKey) { + icon = iconKey->iconData; + mUnassociatedIcons.RemoveEntry(iconKey); + } else { + icon.fetchMode = aForceReload ? FETCH_ALWAYS : FETCH_IF_MISSING; + rv = aFaviconURI->GetSpec(icon.spec); + NS_ENSURE_SUCCESS(rv, rv); + // URIs can arguably lack a host. + Unused << aFaviconURI->GetHost(icon.host); + if (StringBeginsWith(icon.host, "www."_ns)) { + icon.host.Cut(0, 4); + } + } + + // A root icon is when the icon and page have the same host and the path + // is just /favicon.ico. These icons are considered valid for the whole + // origin and expired with the origin through a trigger. + nsAutoCString path; + if (NS_SUCCEEDED(aFaviconURI->GetPathQueryRef(path)) && + !icon.host.IsEmpty() && icon.host.Equals(page.host) && + path.EqualsLiteral("/favicon.ico")) { + icon.rootIcon = 1; + } + + // If the page url points to an image, the icon's url will be the same. + // TODO (Bug 403651): store a resample of the image. For now avoid that + // for database size and UX concerns. + // Don't store favicons for error pages either. + if (icon.spec.Equals(page.spec) || + icon.spec.EqualsLiteral(FAVICON_CERTERRORPAGE_URL) || + icon.spec.EqualsLiteral(FAVICON_ERRORPAGE_URL)) { + return NS_OK; + } + + RefPtr event = new AsyncFetchAndSetIconForPage( + icon, page, loadPrivate, aCallback, aLoadingPrincipal, aRequestContextID); + + // Get the target thread and start the work. + // DB will be updated and observers notified when data has finished loading. + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + DB->DispatchToAsyncThread(event); + + // Return this event to the caller to allow aborting an eventual fetch. + event.forget(_canceler); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::ReplaceFaviconData(nsIURI* aFaviconURI, + const nsTArray& aData, + const nsACString& aMimeType, + PRTime aExpiration) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aFaviconURI); + NS_ENSURE_ARG(aData.Length() > 0); + NS_ENSURE_ARG(aMimeType.Length() > 0); + NS_ENSURE_ARG(imgLoader::SupportImageWithMimeType( + aMimeType, AcceptedMimeTypes::IMAGES_AND_DOCUMENTS)); + + PRTime now = PR_Now(); + if (aExpiration < now + MIN_FAVICON_EXPIRATION) { + // Invalid input, just use the default. + aExpiration = now + MAX_FAVICON_EXPIRATION; + } + + UnassociatedIconHashKey* iconKey = mUnassociatedIcons.PutEntry(aFaviconURI); + if (!iconKey) { + return NS_ERROR_OUT_OF_MEMORY; + } + + iconKey->created = PR_Now(); + + // If the cache contains unassociated icons, an expiry timer should already + // exist, otherwise there may be a timer left hanging around, so make sure we + // fire a new one. + int32_t unassociatedCount = mUnassociatedIcons.Count(); + if (unassociatedCount == 1) { + mExpireUnassociatedIconsTimer->Cancel(); + mExpireUnassociatedIconsTimer->InitWithCallback( + this, UNASSOCIATED_ICON_EXPIRY_INTERVAL, nsITimer::TYPE_ONE_SHOT); + } + + IconData* iconData = &(iconKey->iconData); + iconData->expiration = aExpiration; + iconData->status = ICON_STATUS_CACHED; + iconData->fetchMode = FETCH_NEVER; + nsresult rv = aFaviconURI->GetSpec(iconData->spec); + NS_ENSURE_SUCCESS(rv, rv); + // URIs can arguably lack a host. + Unused << aFaviconURI->GetHost(iconData->host); + if (StringBeginsWith(iconData->host, "www."_ns)) { + iconData->host.Cut(0, 4); + } + + // Note we can't set rootIcon here, because don't know the page it will be + // associated with. We'll do that later in SetAndFetchFaviconForPage if the + // icon doesn't exist; otherwise, if AsyncReplaceFaviconData updates an + // existing icon, it will take care of not overwriting an existing + // root = 1 value. + + IconPayload payload; + payload.mimeType = aMimeType; + payload.data.Assign(TO_CHARBUFFER(aData.Elements()), aData.Length()); + if (payload.mimeType.EqualsLiteral(SVG_MIME_TYPE)) { + payload.width = UINT16_MAX; + } + // There may already be a previous payload, so ensure to only have one. + iconData->payloads.Clear(); + iconData->payloads.AppendElement(payload); + + rv = OptimizeIconSizes(*iconData); + NS_ENSURE_SUCCESS(rv, rv); + + // If there's not valid payload, don't store the icon into to the database. + if ((*iconData).payloads.Length() == 0) { + // We cannot optimize this favicon size and we are over the maximum size + // allowed, so we will not save data to the db to avoid bloating it. + mUnassociatedIcons.RemoveEntry(aFaviconURI); + return NS_ERROR_FAILURE; + } + + // If the database contains an icon at the given url, we will update the + // database immediately so that the associated pages are kept in sync. + // Otherwise, do nothing and let the icon be picked up from the memory hash. + RefPtr event = + new AsyncReplaceFaviconData(*iconData); + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + DB->DispatchToAsyncThread(event); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::ReplaceFaviconDataFromDataURL( + nsIURI* aFaviconURI, const nsAString& aDataURL, PRTime aExpiration, + nsIPrincipal* aLoadingPrincipal) { + NS_ENSURE_ARG(aFaviconURI); + NS_ENSURE_TRUE(aDataURL.Length() > 0, NS_ERROR_INVALID_ARG); + PRTime now = PR_Now(); + if (aExpiration < now + MIN_FAVICON_EXPIRATION) { + // Invalid input, just use the default. + aExpiration = now + MAX_FAVICON_EXPIRATION; + } + + nsCOMPtr dataURI; + nsresult rv = NS_NewURI(getter_AddRefs(dataURI), aDataURL); + NS_ENSURE_SUCCESS(rv, rv); + + // Use the data: protocol handler to convert the data. + nsCOMPtr ioService = do_GetIOService(&rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr protocolHandler; + rv = ioService->GetProtocolHandler("data", getter_AddRefs(protocolHandler)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr loadingPrincipal = aLoadingPrincipal; + MOZ_ASSERT(loadingPrincipal, + "please provide aLoadingPrincipal for this favicon"); + if (!loadingPrincipal) { + // Let's default to the nullPrincipal if no loadingPrincipal is provided. + AutoTArray params = { + u"nsFaviconService::ReplaceFaviconDataFromDataURL()"_ns, + u"nsFaviconService::ReplaceFaviconDataFromDataURL(...," + " [optional aLoadingPrincipal])"_ns}; + nsContentUtils::ReportToConsole( + nsIScriptError::warningFlag, "Security by Default"_ns, + nullptr, // aDocument + nsContentUtils::eNECKO_PROPERTIES, "APIDeprecationWarning", params); + + loadingPrincipal = NullPrincipal::CreateWithoutOriginAttributes(); + } + NS_ENSURE_TRUE(loadingPrincipal, NS_ERROR_FAILURE); + + nsCOMPtr loadInfo = new mozilla::net::LoadInfo( + loadingPrincipal, + nullptr, // aTriggeringPrincipal + nullptr, // aLoadingNode + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT | + nsILoadInfo::SEC_ALLOW_CHROME | nsILoadInfo::SEC_DISALLOW_SCRIPT, + nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON); + + nsCOMPtr channel; + rv = protocolHandler->NewChannel(dataURI, loadInfo, getter_AddRefs(channel)); + NS_ENSURE_SUCCESS(rv, rv); + + // Blocking stream is OK for data URIs. + nsCOMPtr stream; + rv = channel->Open(getter_AddRefs(stream)); + NS_ENSURE_SUCCESS(rv, rv); + + uint64_t available64; + rv = stream->Available(&available64); + NS_ENSURE_SUCCESS(rv, rv); + if (available64 == 0 || available64 > UINT32_MAX / sizeof(uint8_t)) + return NS_ERROR_FILE_TOO_BIG; + uint32_t available = (uint32_t)available64; + + // Read all the decoded data. + nsTArray buffer; + buffer.SetLength(available); + uint32_t numRead; + rv = stream->Read(TO_CHARBUFFER(buffer.Elements()), available, &numRead); + if (NS_FAILED(rv) || numRead != available) { + return rv; + } + + nsAutoCString mimeType; + rv = channel->GetContentType(mimeType); + if (NS_FAILED(rv)) { + return rv; + } + + // ReplaceFaviconData can now do the dirty work. + rv = ReplaceFaviconData(aFaviconURI, buffer, mimeType, aExpiration); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::GetFaviconURLForPage(nsIURI* aPageURI, + nsIFaviconDataCallback* aCallback, + uint16_t aPreferredWidth) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aPageURI); + NS_ENSURE_ARG(aCallback); + // Use the default value, may be UINT16_MAX if a default is not set. + if (aPreferredWidth == 0) { + aPreferredWidth = mDefaultIconURIPreferredSize; + } + + nsAutoCString pageSpec; + nsresult rv = aPageURI->GetSpec(pageSpec); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString pageHost; + // It's expected that some domains may not have a host. + Unused << aPageURI->GetHost(pageHost); + + RefPtr event = new AsyncGetFaviconURLForPage( + pageSpec, pageHost, aPreferredWidth, aCallback); + + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + DB->DispatchToAsyncThread(event); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::GetFaviconDataForPage(nsIURI* aPageURI, + nsIFaviconDataCallback* aCallback, + uint16_t aPreferredWidth) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aPageURI); + NS_ENSURE_ARG(aCallback); + // Use the default value, may be UINT16_MAX if a default is not set. + if (aPreferredWidth == 0) { + aPreferredWidth = mDefaultIconURIPreferredSize; + } + + nsAutoCString pageSpec; + nsresult rv = aPageURI->GetSpec(pageSpec); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString pageHost; + // It's expected that some domains may not have a host. + Unused << aPageURI->GetHost(pageHost); + + RefPtr event = new AsyncGetFaviconDataForPage( + pageSpec, pageHost, aPreferredWidth, aCallback); + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + DB->DispatchToAsyncThread(event); + + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::CopyFavicons(nsIURI* aFromPageURI, nsIURI* aToPageURI, + uint32_t aFaviconLoadType, + nsIFaviconDataCallback* aCallback) { + MOZ_ASSERT(NS_IsMainThread()); + NS_ENSURE_ARG(aFromPageURI); + NS_ENSURE_ARG(aToPageURI); + NS_ENSURE_TRUE( + aFaviconLoadType >= nsIFaviconService::FAVICON_LOAD_PRIVATE && + aFaviconLoadType <= nsIFaviconService::FAVICON_LOAD_NON_PRIVATE, + NS_ERROR_INVALID_ARG); + + PageData fromPage; + nsresult rv = aFromPageURI->GetSpec(fromPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + PageData toPage; + rv = aToPageURI->GetSpec(toPage.spec); + NS_ENSURE_SUCCESS(rv, rv); + + bool canAddToHistory; + nsNavHistory* navHistory = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(navHistory, NS_ERROR_OUT_OF_MEMORY); + rv = navHistory->CanAddURI(aToPageURI, &canAddToHistory); + NS_ENSURE_SUCCESS(rv, rv); + toPage.canAddToHistory = + !!canAddToHistory && + aFaviconLoadType != nsIFaviconService::FAVICON_LOAD_PRIVATE; + + RefPtr event = + new AsyncCopyFavicons(fromPage, toPage, aCallback); + + // Get the target thread and start the work. + // DB will be updated and observers notified when done. + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + DB->DispatchToAsyncThread(event); + + return NS_OK; +} + +nsresult nsFaviconService::GetFaviconLinkForIcon(nsIURI* aFaviconURI, + nsIURI** aOutputURI) { + NS_ENSURE_ARG(aFaviconURI); + NS_ENSURE_ARG_POINTER(aOutputURI); + + nsAutoCString spec; + if (aFaviconURI) { + nsresult rv = aFaviconURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + } + return GetFaviconLinkForIconString(spec, aOutputURI); +} + +// nsFaviconService::GetFaviconLinkForIconString +// +// This computes a favicon URL with string input and using the cached +// default one to minimize parsing. + +nsresult nsFaviconService::GetFaviconLinkForIconString(const nsCString& aSpec, + nsIURI** aOutput) { + if (aSpec.IsEmpty()) { + return GetDefaultFavicon(aOutput); + } + + if (StringBeginsWith(aSpec, "chrome:"_ns)) { + // pass through for chrome URLs, since they can be referenced without + // this service + return NS_NewURI(aOutput, aSpec); + } + + nsAutoCString annoUri; + annoUri.AssignLiteral("moz-anno:" FAVICON_ANNOTATION_NAME ":"); + annoUri += aSpec; + return NS_NewURI(aOutput, annoUri); +} + +/** + * Checks the icon and evaluates if it needs to be optimized. + * + * @param aIcon + * The icon to be evaluated. + */ +nsresult nsFaviconService::OptimizeIconSizes(IconData& aIcon) { + // TODO (bug 1346139): move optimization to the async thread. + MOZ_ASSERT(NS_IsMainThread()); + // There should only be a single payload at this point, it may have to be + // split though, if it's an ico file. + MOZ_ASSERT(aIcon.payloads.Length() == 1); + + // Even if the page provides a large image for the favicon (eg, a highres + // image or a multiresolution .ico file), don't try to store more data than + // needed. + IconPayload payload = aIcon.payloads[0]; + if (payload.mimeType.EqualsLiteral(SVG_MIME_TYPE)) { + // Nothing to optimize, but check the payload size. + if (payload.data.Length() >= nsIFaviconService::MAX_FAVICON_BUFFER_SIZE) { + aIcon.payloads.Clear(); + } + return NS_OK; + } + + // Make space for the optimized payloads. + aIcon.payloads.Clear(); + + // decode image + nsCOMPtr container; + nsresult rv = GetImgTools()->DecodeImageFromBuffer( + payload.data.get(), payload.data.Length(), payload.mimeType, + getter_AddRefs(container)); + NS_ENSURE_SUCCESS(rv, rv); + + // For ICO files, we must evaluate each of the frames we care about. + nsTArray framesInfo; + rv = GetFramesInfoForContainer(container, framesInfo); + NS_ENSURE_SUCCESS(rv, rv); + + for (const auto& frameInfo : framesInfo) { + IconPayload newPayload; + newPayload.mimeType = nsLiteralCString(PNG_MIME_TYPE); + newPayload.width = frameInfo.width; + for (uint16_t size : gFaviconSizes) { + // The icon could be smaller than 16, that is our minimum. + // Icons smaller than 16px are kept as-is. + if (frameInfo.width >= 16) { + if (size > frameInfo.width) { + continue; + } + newPayload.width = size; + } + + // If the original payload is png, the size is the same and not animated, + // rescale the image only if it's larger than the maximum allowed. + bool animated; + if (newPayload.mimeType.Equals(payload.mimeType) && + newPayload.width == frameInfo.width && + payload.data.Length() < nsIFaviconService::MAX_FAVICON_BUFFER_SIZE && + (NS_FAILED(container->GetAnimated(&animated)) || !animated)) { + newPayload.data = payload.data; + break; + } + + // Otherwise, scale and recompress. Rescaling will also take care of + // extracting a static image from an animated one. + // Since EncodeScaledImage uses SYNC_DECODE, it will pick the best + // frame. + nsCOMPtr iconStream; + rv = GetImgTools()->EncodeScaledImage(container, newPayload.mimeType, + newPayload.width, newPayload.width, + u""_ns, getter_AddRefs(iconStream)); + NS_ENSURE_SUCCESS(rv, rv); + // Read the stream into the new buffer. + rv = NS_ConsumeStream(iconStream, UINT32_MAX, newPayload.data); + NS_ENSURE_SUCCESS(rv, rv); + + // If the icon size is good, we are done, otherwise try the next size. + if (newPayload.data.Length() < + nsIFaviconService::MAX_FAVICON_BUFFER_SIZE) { + break; + } + } + + MOZ_ASSERT(newPayload.data.Length() < + nsIFaviconService::MAX_FAVICON_BUFFER_SIZE); + if (newPayload.data.Length() < nsIFaviconService::MAX_FAVICON_BUFFER_SIZE) { + aIcon.payloads.AppendElement(newPayload); + } + } + + return NS_OK; +} + +nsresult nsFaviconService::GetFaviconDataAsync( + const nsCString& aFaviconURI, mozIStorageStatementCallback* aCallback) { + MOZ_ASSERT(aCallback, "Doesn't make sense to call this without a callback"); + + nsCOMPtr stmt = mDB->GetAsyncStatement( + "/*Do not warn (bug no: not worth adding an index */ " + "SELECT data, width FROM moz_icons " + "WHERE fixed_icon_url_hash = hash(fixup_url(:url)) AND icon_url = :url " + "ORDER BY width DESC"); + NS_ENSURE_STATE(stmt); + + nsresult rv = URIBinder::Bind(stmt, "url"_ns, aFaviconURI); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr pendingStatement; + return stmt->ExecuteAsync(aCallback, getter_AddRefs(pendingStatement)); +} + +NS_IMETHODIMP +nsFaviconService::SetDefaultIconURIPreferredSize(uint16_t aDefaultSize) { + mDefaultIconURIPreferredSize = aDefaultSize > 0 ? aDefaultSize : UINT16_MAX; + return NS_OK; +} + +NS_IMETHODIMP +nsFaviconService::PreferredSizeFromURI(nsIURI* aURI, uint16_t* _size) { + NS_ENSURE_ARG(aURI); + *_size = mDefaultIconURIPreferredSize; + nsAutoCString ref; + // Check for a ref first. + if (NS_FAILED(aURI->GetRef(ref)) || ref.Length() == 0) return NS_OK; + + // Look for a "size=" fragment. + int32_t start = ref.RFind("size="); + if (start >= 0 && ref.Length() > static_cast(start) + 5) { + nsDependentCSubstring size; + // This is safe regardless, since Rebind checks start is not over Length(). + size.Rebind(ref, start + 5); + // Check if the string contains any non-digit. + auto begin = size.BeginReading(), end = size.EndReading(); + for (auto ch = begin; ch < end; ++ch) { + if (*ch < '0' || *ch > '9') { + // Not a digit. + return NS_OK; + } + } + // Convert the string to an integer value. + nsresult rv; + uint16_t val = PromiseFlatCString(size).ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + *_size = val; + } + } + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// ExpireFaviconsStatementCallbackNotifier + +ExpireFaviconsStatementCallbackNotifier:: + ExpireFaviconsStatementCallbackNotifier() = default; + +NS_IMETHODIMP +ExpireFaviconsStatementCallbackNotifier::HandleCompletion(uint16_t aReason) { + // We should dispatch only if expiration has been successful. + if (aReason != mozIStorageStatementCallback::REASON_FINISHED) return NS_OK; + + nsCOMPtr observerService = + mozilla::services::GetObserverService(); + if (observerService) { + (void)observerService->NotifyObservers( + nullptr, NS_PLACES_FAVICONS_EXPIRED_TOPIC_ID, nullptr); + } + + return NS_OK; +} diff --git a/toolkit/components/places/nsFaviconService.h b/toolkit/components/places/nsFaviconService.h new file mode 100644 index 0000000000..792bf37faf --- /dev/null +++ b/toolkit/components/places/nsFaviconService.h @@ -0,0 +1,147 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsFaviconService_h_ +#define nsFaviconService_h_ + +#include + +#include "Database.h" +#include "FaviconHelpers.h" +#include "imgITools.h" +#include "mozilla/Attributes.h" +#include "mozilla/storage.h" +#include "nsCOMPtr.h" +#include "nsComponentManagerUtils.h" +#include "nsIFaviconService.h" +#include "nsINamed.h" +#include "nsITimer.h" +#include "nsServiceManagerUtils.h" +#include "nsString.h" +#include "nsTHashtable.h" +#include "nsToolkitCompsCID.h" +#include "nsURIHashKey.h" + +// The target dimension in pixels for favicons we store, in reverse order. +// When adding/removing sizes from here, make sure to update the vector size. +extern const uint16_t gFaviconSizes[7]; + +// forward class definitions +class mozIStorageStatementCallback; + +class UnassociatedIconHashKey : public nsURIHashKey { + public: + explicit UnassociatedIconHashKey(const nsIURI* aURI) : nsURIHashKey(aURI) {} + UnassociatedIconHashKey(UnassociatedIconHashKey&& aOther) + : nsURIHashKey(std::move(aOther)), + iconData(std::move(aOther.iconData)), + created(std::move(aOther.created)) { + MOZ_ASSERT_UNREACHABLE("Do not call me!"); + } + mozilla::places::IconData iconData; + PRTime created; +}; + +class nsFaviconService final : public nsIFaviconService, + public nsITimerCallback, + public nsINamed { + public: + nsFaviconService(); + + /** + * Obtains the service's object. + */ + static already_AddRefed GetSingleton(); + + /** + * Initializes the service's object. This should only be called once. + */ + nsresult Init(); + + /** + * Returns a cached pointer to the favicon service for consumers in the + * places directory. + */ + static nsFaviconService* GetFaviconService() { + if (!gFaviconService) { + nsCOMPtr serv = + do_GetService(NS_FAVICONSERVICE_CONTRACTID); + NS_ENSURE_TRUE(serv, nullptr); + NS_ASSERTION(gFaviconService, "Should have static instance pointer now"); + } + return gFaviconService; + } + + // addition to API for strings to prevent excessive parsing of URIs + nsresult GetFaviconLinkForIconString(const nsCString& aIcon, + nsIURI** aOutput); + + nsresult OptimizeIconSizes(mozilla::places::IconData& aIcon); + + /** + * Obtains the favicon data asynchronously. + * + * @param aFaviconSpec + * The spec of the URI representing the favicon we are looking for. + * @param aCallback + * The callback where results or errors will be dispatch to. In the + * returned result, the favicon binary data will be at index 0, and the + * mime type will be at index 1. + */ + nsresult GetFaviconDataAsync(const nsCString& aFaviconSpec, + mozIStorageStatementCallback* aCallback); + + /** + * Clears the image cache for the given image spec. + * + * @param aImageURI + * The URI of the image to clear cache for. + */ + void ClearImageCache(nsIURI* aImageURI); + + static mozilla::Atomic sLastInsertedIconId; + static void StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId); + + NS_DECL_ISUPPORTS + NS_DECL_NSIFAVICONSERVICE + NS_DECL_NSITIMERCALLBACK + NS_DECL_NSINAMED + + private: + imgITools* GetImgTools() { + if (!mImgTools) { + mImgTools = do_CreateInstance("@mozilla.org/image/tools;1"); + } + return mImgTools; + } + + ~nsFaviconService(); + + RefPtr mDB; + + nsCOMPtr mExpireUnassociatedIconsTimer; + nsCOMPtr mImgTools; + + static nsFaviconService* gFaviconService; + + /** + * A cached URI for the default icon. We return this a lot, and don't want to + * re-parse and normalize our unchanging string many times. Important: do + * not return this directly; use Clone() since callers may change the object + * they get back. May be null, in which case it needs initialization. + */ + nsCOMPtr mDefaultIcon; + + // This class needs access to the icons cache. + friend class mozilla::places::AsyncReplaceFaviconData; + nsTHashtable mUnassociatedIcons; + + uint16_t mDefaultIconURIPreferredSize; +}; + +#define FAVICON_ANNOTATION_NAME "favicon" + +#endif // nsFaviconService_h_ diff --git a/toolkit/components/places/nsIFaviconService.idl b/toolkit/components/places/nsIFaviconService.idl new file mode 100644 index 0000000000..b7f9046757 --- /dev/null +++ b/toolkit/components/places/nsIFaviconService.idl @@ -0,0 +1,339 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; +interface nsIPrincipal; +interface mozIPlacesPendingOperation; +interface nsIFaviconDataCallback; + +[scriptable, uuid(e81e0b0c-b9f1-4c2e-8f3c-b809933cf73c)] +interface nsIFaviconService : nsISupports +{ + // The favicon is being loaded from a private browsing window + const unsigned long FAVICON_LOAD_PRIVATE = 1; + // The favicon is being loaded from a non-private browsing window + const unsigned long FAVICON_LOAD_NON_PRIVATE = 2; + + /** + * The limit in bytes of the size of favicons in memory and passed via the + * favicon protocol. + */ + const unsigned long MAX_FAVICON_BUFFER_SIZE = 65536; + + /** + * For a given icon URI, this will return a URI that will result in the image. + * In most cases, this is an annotation URI. For chrome URIs, this will do + * nothing but returning the input URI. + * + * No validity checking is done. If you pass an icon URI that we've never + * seen, you'll get back a URI that references an invalid icon. The moz-anno + * protocol handler's special case for "favicon" annotations will resolve + * invalid icons to the default icon, although without caching. + * For invalid chrome URIs, you'll get a broken image. + * + * @param aFaviconURI + * The URI of an icon in the favicon service. + * @return A URI that will give you the icon image. This is NOT the URI of + * the icon as set on the page, but a URI that will give you the + * data out of the favicon service. For a normal page with a + * favicon we've stored, this will be an annotation URI which will + * then cause the corresponding favicon data to be loaded async from + * this service. For pages where we don't have a favicon, this will + * be a chrome URI of the default icon. For chrome URIs, the + * output will be the same as the input. + */ + nsIURI getFaviconLinkForIcon(in nsIURI aFaviconURI); + + /** + * Expire all known favicons from the database. + * + * @note This is an async method. + * On successful completion a "places-favicons-expired" notification is + * dispatched through observer's service. + */ + void expireAllFavicons(); + + /** + * Sets the default size returned by preferredSizeFromURI when the uri doesn't + * specify a size ref. If this is not invoked first, or 0 is passed to it, + * preferredSizeFromURI() will return UINT16_MAX, that matches the biggest + * icon available. + */ + void setDefaultIconURIPreferredSize(in unsigned short aDefaultSize); + + /** + * Tries to extract the preferred size from an icon uri ref fragment. + * + * @param aURI + * The URI to parse. + * @return The preferred size, or a default size set through + * setDefaultIconURIPreferredSize, or UINT16_MAX if neither are set. + */ + unsigned short preferredSizeFromURI(in nsIURI aURI); + + /** + * The default favicon URI + */ + readonly attribute nsIURI defaultFavicon; + + /** + * The default favicon mimeType + */ + readonly attribute AUTF8String defaultFaviconMimeType; + + /** + * Declares that a given page uses a favicon with the given URI and + * attempts to fetch and save the icon data by loading the favicon URI + * through an async network request. + * + * If the icon data already exists, we won't try to reload the icon unless + * aForceReload is true. Similarly, if the icon is in the failed favicon + * cache we won't do anything unless aForceReload is true, in which case + * we'll try to reload the favicon. + * + * This function will only save favicons for pages that are already stored in + * the database, like visited pages or bookmarks. For any other URIs, it + * will succeed but do nothing. This function will also ignore the error + * page favicon URI (see FAVICON_ERRORPAGE_URL below). + * + * Icons that fail to load will automatically be added to the failed favicon + * cache, and this function will not save favicons for non-bookmarked URIs + * when history is disabled. + * + * @note This function is identical to + * nsIFaviconService::setAndLoadFaviconForPage. + * + * @param aPageURI + * URI of the page whose favicon is being set. + * @param aFaviconURI + * URI of the favicon to associate with the page. + * @param aForceReload + * If aForceReload is false, we try to reload the favicon only if we + * don't have it or it has expired from the cache. Setting + * aForceReload to true causes us to reload the favicon even if we + * have a usable copy. + * @param aFaviconLoadType + * Set to FAVICON_LOAD_PRIVATE if the favicon is loaded from a private + * browsing window. Set to FAVICON_LOAD_NON_PRIVATE otherwise. + * @param [optional] aCallback + * Once we're done setting and/or fetching the favicon, we invoke this + * callback. + * @param [optional] aLoadingPrincipal + * Principal of the page whose favicon is being set. If this argument + * is omitted, the loadingPrincipal defaults to the nullPrincipal. + * @param [optional] aRequestContextID + * used to inform Necko of how to link the + * favicon request with other requests in the same tab. + * + * @see nsIFaviconDataCallback in nsIFaviconService.idl. + */ + mozIPlacesPendingOperation setAndFetchFaviconForPage( + in nsIURI aPageURI, + in nsIURI aFaviconURI, + in boolean aForceReload, + in unsigned long aFaviconLoadType, + [optional] in nsIFaviconDataCallback aCallback, + [optional] in nsIPrincipal aLoadingPrincipal, + [optional] in unsigned long long aRequestContextID); + + /** + * Sets the data for a given favicon URI either by replacing existing data in + * the database or taking the place of otherwise fetched icon data when + * calling setAndFetchFaviconForPage later. + * + * Favicon data for favicon URIs that are not associated with a page URI via + * setAndFetchFaviconForPage will be stored in memory, but may be expired at + * any time, so you should make an effort to associate favicon URIs with page + * URIs as soon as possible. + * + * It's better to not use this function for chrome: icon URIs since you can + * reference the chrome image yourself. getFaviconLinkForIcon/Page will ignore + * any associated data if the favicon URI is "chrome:" and just return the + * same chrome URI. + * + * This function does NOT send out notifications that the data has changed. + * Pages using this favicons that are visible in history or bookmarks views + * will keep the old icon until they have been refreshed by other means. + * + * This function tries to optimize the favicon size, if it is bigger + * than a defined limit we will try to convert it to a 16x16 png image. + * If the conversion fails and favicon is still bigger than our max accepted + * size it won't be saved. + * + * @param aFaviconURI + * URI of the favicon whose data is being set. + * @param aData + * Binary contents of the favicon to save + * @param aMimeType + * MIME type of the data to store. This is important so that we know + * what to report when the favicon is used. You should always set this + * param unless you are clearing an icon. + * @param [optional] aExpiration + * Time in microseconds since the epoch when this favicon expires. + * Until this time, we won't try to load it again. + * @throws NS_ERROR_FAILURE + * Thrown if the favicon is overbloated and won't be saved to the db. + */ + void replaceFaviconData(in nsIURI aFaviconURI, + in Array aData, + in AUTF8String aMimeType, + [optional] in PRTime aExpiration); + + /** + * Same as replaceFaviconData but the data is provided by a string + * containing a data URL. + * + * @see replaceFaviconData + * + * @param aFaviconURI + * URI of the favicon whose data is being set. + * @param aDataURL + * string containing a data URL that represents the contents of + * the favicon to save + * @param [optional] aExpiration + * Time in microseconds since the epoch when this favicon expires. + * Until this time, we won't try to load it again. + * @param [optional] aLoadingPrincipal + * Principal of the page whose favicon is being set. If this argument + * is omitted, the loadingPrincipal defaults to the nullPrincipal. + * @throws NS_ERROR_FAILURE + * Thrown if the favicon is overbloated and won't be saved to the db. + */ + void replaceFaviconDataFromDataURL(in nsIURI aFaviconURI, + in AString aDataURL, + [optional] in PRTime aExpiration, + [optional] in nsIPrincipal aLoadingPrincipal); + + /** + * Retrieves the favicon URI associated to the given page, if any. + * + * @param aPageURI + * URI of the page whose favicon URI we're looking up. + * @param aCallback + * This callback is always invoked to notify the result of the lookup. + * The aURI parameter will be the favicon URI, or null when no favicon + * is associated with the page or an error occurred while fetching it. + * aDataLen will be always 0, aData will be an empty array, and + * aMimeType will be an empty string, regardless of whether a favicon + * was found. + * @param [optional] aPreferredWidth + * The preferred icon width, skip or pass 0 for the default value, + * set through setDefaultIconURIPreferredSize. + * + * @note If a favicon specific to this page cannot be found, this will try to + * fallback to the /favicon.ico for the root domain. + * + * @see nsIFaviconDataCallback in nsIFaviconService.idl. + */ + void getFaviconURLForPage(in nsIURI aPageURI, + in nsIFaviconDataCallback aCallback, + [optional] in unsigned short aPreferredWidth); + + /** + * Retrieves the favicon URI and data associated to the given page, if any. + * If the page icon is not available, it will try to return the root domain + * icon data, when it's known. + * + * @param aPageURI + * URI of the page whose favicon URI and data we're looking up. + * @param aCallback + * This callback is always invoked to notify the result of the lookup. The aURI + * parameter will be the favicon URI, or null when no favicon is + * associated with the page or an error occurred while fetching it. If + * aURI is not null, the other parameters may contain the favicon data. + * However, if no favicon data is currently associated with the favicon + * URI, aDataLen will be 0, aData will be an empty array, and aMimeType + * will be an empty string. + * @param [optional] aPreferredWidth + * The preferred icon width, skip or pass 0 for the default value, + * set through setDefaultIconURIPreferredSize. + * @note If a favicon specific to this page cannot be found, this will try to + * fallback to the /favicon.ico for the root domain. + * + * @see nsIFaviconDataCallback in nsIFaviconService.idl. + */ + void getFaviconDataForPage(in nsIURI aPageURI, + in nsIFaviconDataCallback aCallback, + [optional] in unsigned short aPreferredWidth); + + /** + * Copies cached favicons from a page to another one. + * + * @param aFromPageURI + * URI of the originating page. + * @param aToPageURI + * URI of the destination page. + * @param aFaviconLoadType + * Set to FAVICON_LOAD_PRIVATE if the copy is started from a private + * browsing window. Set to FAVICON_LOAD_NON_PRIVATE otherwise. + * @param [optional] aCallback + * Once we're done copying the favicon, we invoke this callback. + * If a copy has been done, the callback will report one of the + * favicons uri as aFaviconURI, otherwise all the params will be null. + */ + void copyFavicons(in nsIURI aFromPageURI, + in nsIURI aToPageURI, + in unsigned long aFaviconLoadType, + [optional] in nsIFaviconDataCallback aCallback); +}; + +[scriptable, function, uuid(c85e5c82-b70f-4621-9528-beb2aa47fb44)] +interface nsIFaviconDataCallback : nsISupports +{ + /** + * Called when the required favicon's information is available. + * + * It's up to the invoking method to state if the callback is always invoked, + * or called on success only. Check the method documentation to ensure that. + * + * The caller will receive the most information we can gather on the icon, + * but it's not guaranteed that all of them will be set. For some method + * we could not know the favicon's data (it could just be too expensive to + * get it, or the method does not require we actually have any data). + * It's up to the caller to check aDataLen > 0 before using any data-related + * information like mime-type or data itself. + * + * @param aFaviconURI + * Receives the "favicon URI" (not the "favicon link URI") associated + * to the requested page. This can be null if there is no associated + * favicon URI, or the callback is notifying a failure. + * @param aDataLen + * Size of the icon data in bytes. Notice that a value of 0 does not + * necessarily mean that we don't have an icon. + * @param aData + * Icon data, or an empty array if aDataLen is 0. + * @param aMimeType + * Mime type of the icon, or an empty string if aDataLen is 0. + * @param aWidth + * Width of the icon. 0 if the width is unknown or if the icon is + * vectorial. + * + * @note If you want to open a network channel to access the favicon, it's + * recommended that you call the getFaviconLinkForIcon method to convert + * the "favicon URI" into a "favicon link URI". + */ + void onComplete(in nsIURI aFaviconURI, + in unsigned long aDataLen, + [const,array,size_is(aDataLen)] in octet aData, + in AUTF8String aMimeType, + in unsigned short aWidth); +}; + +%{C++ + +/** + * Notification sent when all favicons are expired. + */ +#define NS_PLACES_FAVICONS_EXPIRED_TOPIC_ID "places-favicons-expired" + +#define FAVICON_DEFAULT_URL "chrome://global/skin/icons/defaultFavicon.svg" +#define FAVICON_DEFAULT_MIMETYPE "image/svg+xml" + +#define FAVICON_ERRORPAGE_URL "chrome://global/skin/icons/info.svg" +#define FAVICON_CERTERRORPAGE_URL "chrome://global/skin/icons/warning.svg" + +%} diff --git a/toolkit/components/places/nsINavBookmarksService.idl b/toolkit/components/places/nsINavBookmarksService.idl new file mode 100644 index 0000000000..1891d7c6d0 --- /dev/null +++ b/toolkit/components/places/nsINavBookmarksService.idl @@ -0,0 +1,211 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIFile; +interface nsIURI; +interface nsITransaction; + +/** + * The BookmarksService interface provides methods for managing bookmarked + * history items. Bookmarks consist of a set of user-customizable + * folders. A URI in history can be contained in one or more such folders. + */ + +[scriptable, uuid(24533891-afa6-4663-b72d-3143d03f1b04)] +interface nsINavBookmarksService : nsISupports +{ + /** + * The item ID of the top-level folder that contain the tag "folders". + */ + readonly attribute long long tagsFolder; + + /** + * The total number of Sync changes (inserts, updates, deletes, merges, and + * uploads) recorded since Places startup for all bookmarks. + * + * Note that this is *not* the number of bookmark syncs. It's a monotonically + * increasing counter incremented for every change that affects a bookmark's + * `syncChangeCounter`. + * + * The counter can be used to avoid keeping an exclusive transaction open for + * time-consuming work. One way to do that is to store the current value of + * the counter, do the work, start a transaction, check the current value + * again, and compare it to the stored value to determine if the database + * changed during the work. + * + * The bookmarks mirror does this to check for changes between building and + * applying a merged tree. This avoids blocking the main Places connection + * during the merge, and ensures that the new tree still applies cleanly. + */ + readonly attribute long long totalSyncChanges; + + /** + * This value should be used for APIs that allow passing in an index + * where an index is not known, or not required to be specified. + * e.g.: When appending an item to a folder. + */ + const short DEFAULT_INDEX = -1; + + const unsigned short TYPE_BOOKMARK = 1; + const unsigned short TYPE_FOLDER = 2; + const unsigned short TYPE_SEPARATOR = 3; + // Dynamic containers are deprecated and unsupported. + // This const exists just to avoid reusing the value. + const unsigned short TYPE_DYNAMIC_CONTAINER = 4; + + // Change source constants. These are used to distinguish changes made by + // Sync and bookmarks import from other Places consumers, though they can + // be extended to support other callers. Sources are passed as optional + // parameters to methods used by Sync, and forwarded to observers. + const unsigned short SOURCE_DEFAULT = 0; + const unsigned short SOURCE_SYNC = 1; + const unsigned short SOURCE_IMPORT = 2; + const unsigned short SOURCE_SYNC_REPARENT_REMOVED_FOLDER_CHILDREN = 4; + const unsigned short SOURCE_RESTORE = 5; + const unsigned short SOURCE_RESTORE_ON_STARTUP = 6; + + /** + * Sync status flags, stored in Places for each item. These affect conflict + * resolution, when an item is changed both locally and remotely; deduping, + * when a local item matches a remote item with similar contents and different + * GUIDs; and whether we write a tombstone when an item is deleted locally. + * + * Status | Description | Conflict | Can | Needs + * | | resolution | dedupe? | tombstone? + * ----------------------------------------------------------------------- + * UNKNOWN | Automatically restored | Prefer | No | No + * | on startup to recover | deletion | | + * | from database corruption, | | | + * | or sync ID change on | | | + * | server. | | | + * ----------------------------------------------------------------------- + * NEW | Item not uploaded to | Prefer | Yes | No + * | server yet, or Sync | newer | | + * | disconnected. | | | + * ----------------------------------------------------------------------- + * NORMAL | Item uploaded to server. | Prefer | No | Yes + * | | newer | | + */ + const unsigned short SYNC_STATUS_UNKNOWN = 0; + const unsigned short SYNC_STATUS_NEW = 1; + const unsigned short SYNC_STATUS_NORMAL = 2; + + /** + * Inserts a child bookmark into the given folder. + * + * @param aParentId + * The id of the parent folder + * @param aURI + * The URI to insert + * @param aIndex + * The index to insert at, or DEFAULT_INDEX to append + * @param aTitle + * The title for the new bookmark + * @param [optional] aGuid + * The GUID to be set for the new item. If not set, a new GUID is + * generated. Unless you've a very sound reason, such as an undo + * manager implementation, do not pass this argument. + * @param [optional] aSource + * The change source. This is forwarded to all bookmark observers, + * allowing them to distinguish between insertions from different + * callers. Defaults to SOURCE_DEFAULT if omitted. + * @return The ID of the newly-created bookmark. + * + * @note aTitle will be truncated to TITLE_LENGTH_MAX and + * aURI will be truncated to URI_LENGTH_MAX. + * @throws if aGuid is malformed. + */ + [can_run_script] + long long insertBookmark(in long long aParentId, in nsIURI aURI, + in long aIndex, in AUTF8String aTitle, + [optional] in ACString aGuid, + [optional] in unsigned short aSource); + + /** + * Removes a child item. Used to delete a bookmark or separator. + * @param aItemId + * The child item to remove + * @param [optional] aSource + * The change source, forwarded to all bookmark observers. Defaults + * to SOURCE_DEFAULT. + */ + [can_run_script] + void removeItem(in long long aItemId, [optional] in unsigned short aSource); + + /** + * Creates a new child folder and inserts it under the given parent. + * @param aParentFolder + * The id of the parent folder + * @param aName + * The name of the new folder + * @param aIndex + * The index to insert at, or DEFAULT_INDEX to append + * @param [optional] aGuid + * The GUID to be set for the new item. If not set, a new GUID is + * generated. Unless you've a very sound reason, such as an undo + * manager implementation, do not pass this argument. + * @param [optional] aSource + * The change source, forwarded to all bookmark observers. Defaults + * to SOURCE_DEFAULT. + * @return The ID of the newly-inserted folder. + * @throws if aGuid is malformed. + */ + [can_run_script] + long long createFolder(in long long aParentFolder, in AUTF8String name, + in long index, + [optional] in ACString aGuid, + [optional] in unsigned short aSource); + + /** + * Set the title for an item. + * @param aItemId + * The id of the item whose title should be updated. + * @param aTitle + * The new title for the bookmark. + * @param [optional] aSource + * The change source, forwarded to all bookmark observers. Defaults + * to SOURCE_DEFAULT. + * + * @note aTitle will be truncated to TITLE_LENGTH_MAX. + */ + [can_run_script] + void setItemTitle(in long long aItemId, in AUTF8String aTitle, + [optional] in unsigned short aSource); + + /** + * Get the title for an item. + * + * If no item title is available it will return a void string (null in JS). + * + * @param aItemId + * The id of the item whose title should be retrieved + * @return The title of the item. + */ + AUTF8String getItemTitle(in long long aItemId); + + /** + * Set the last modified time for an item. + * + * @param aItemId + * the id of the item whose last modified time should be updated. + * @param aLastModified + * the new last modified value in microseconds. Note that it is + * rounded down to milliseconds precision. + * @param [optional] aSource + * The change source, forwarded to all bookmark observers. Defaults + * to SOURCE_DEFAULT. + * + * @note This is the only method that will send an itemChanged notification + * for the property. lastModified will still be updated in + * any other method that changes an item property, but we will send + * the corresponding itemChanged notification instead. + */ + [can_run_script] + void setItemLastModified(in long long aItemId, + in PRTime aLastModified, + [optional] in unsigned short aSource); +}; diff --git a/toolkit/components/places/nsINavHistoryService.idl b/toolkit/components/places/nsINavHistoryService.idl new file mode 100644 index 0000000000..2b70360f48 --- /dev/null +++ b/toolkit/components/places/nsINavHistoryService.idl @@ -0,0 +1,1197 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Using Places services after quit-application is not reliable, so make + * sure to do any shutdown work on quit-application, or history + * synchronization could fail, losing latest changes. + */ + +#include "nsISupports.idl" + +interface nsIArray; +interface nsIFile; +interface nsIObserver; +interface nsIVariant; +interface nsIURI; + +interface mozIStorageConnection; +interface mozIStorageStatementCallback; +interface mozIStoragePendingStatement; +interface nsIAsyncShutdownClient; + +interface nsINavHistoryContainerResultNode; +interface nsINavHistoryQueryResultNode; +interface nsINavHistoryQuery; +interface nsINavHistoryQueryOptions; +interface nsINavHistoryResult; + +[scriptable, uuid(91d104bb-17ef-404b-9f9a-d9ed8de6824c)] +interface nsINavHistoryResultNode : nsISupports +{ + /** + * Indentifies the parent result node in the result set. This is null for + * top level nodes. + */ + readonly attribute nsINavHistoryContainerResultNode parent; + + /** + * The history-result to which this node belongs. + */ + readonly attribute nsINavHistoryResult parentResult; + + /** + * URI of the resource in question. For visits and URLs, this is the URL of + * the page. For folders and queries, this is the place: URI of the + * corresponding folder or query. This may be empty for other types of + * objects like host containers. + */ + readonly attribute AUTF8String uri; + + /** + * Identifies the type of this node. This node can then be QI-ed to the + * corresponding specialized result node interface. + */ + const unsigned long RESULT_TYPE_URI = 0; // nsINavHistoryResultNode + + // Visit nodes are deprecated and unsupported. + // This line exists just to avoid reusing the value: + // const unsigned long RESULT_TYPE_VISIT = 1; + + // Full visit nodes are deprecated and unsupported. + // This line exists just to avoid reusing the value: + // const unsigned long RESULT_TYPE_FULL_VISIT = 2; + + // Dynamic containers are deprecated and unsupported. + // This const exists just to avoid reusing the value: + // const unsigned long RESULT_TYPE_DYNAMIC_CONTAINER = 4; // nsINavHistoryContainerResultNode + + const unsigned long RESULT_TYPE_QUERY = 5; // nsINavHistoryQueryResultNode + const unsigned long RESULT_TYPE_FOLDER = 6; // nsINavHistoryQueryResultNode + const unsigned long RESULT_TYPE_SEPARATOR = 7; // nsINavHistoryResultNode + const unsigned long RESULT_TYPE_FOLDER_SHORTCUT = 9; // nsINavHistoryQueryResultNode + readonly attribute unsigned long type; + + /** + * Title of the web page, or of the node's query (day, host, folder, etc) + */ + readonly attribute AUTF8String title; + + /** + * Total number of times the URI has ever been accessed. For hosts, this + * is the total of the children under it, NOT the total times the host has + * been accessed (this would require an additional query, so is not given + * by default when most of the time it is never needed). + */ + readonly attribute unsigned long accessCount; + + /** + * This is the time the user accessed the page. + * + * If this is a visit, it is the exact time that the page visit occurred. + * + * If this is a URI, it is the most recent time that the URI was visited. + * Even if you ask for all URIs for a given date range long ago, this might + * contain today's date if the URI was visited today. + * + * For hosts, or other node types with children, this is the most recent + * access time for any of the children. + * + * For days queries this is the respective endTime - a maximum possible + * visit time to fit in the day range. + */ + readonly attribute PRTime time; + + /** + * This URI can be used as an image source URI and will give you the favicon + * for the page. It is *not* the URI of the favicon, but rather something + * that will resolve to the actual image. + * + * In most cases, this is an annotation URI that will query the favicon + * service. If the entry has no favicon, this is the chrome URI of the + * default favicon. If the favicon originally lived in chrome, this will + * be the original chrome URI of the icon. + */ + readonly attribute AUTF8String icon; + + /** + * This is the number of levels between this node and the top of the + * hierarchy. The members of result.children have indentLevel = 0, their + * children have indentLevel = 1, etc. The indent level of the root node is + * set to -1. + */ + readonly attribute long indentLevel; + + /** + * When this item is in a bookmark folder (parent is of type folder), this is + * the index into that folder of this node. These indices start at 0 and + * increase in the order that they appear in the bookmark folder. For items + * that are not in a bookmark folder, this value is -1. + */ + readonly attribute long bookmarkIndex; + + /** + * If the node is an item (bookmark, folder or a separator) this value is the + * row ID of that bookmark in the database. For other nodes, this value is + * set to -1. + */ + readonly attribute long long itemId; + + /** + * If the node is an item (bookmark, folder or a separator) this value is the + * time that the item was created. For other nodes, this value is 0. + */ + readonly attribute PRTime dateAdded; + + /** + * If the node is an item (bookmark, folder or a separator) this value is the + * time that the item was last modified. For other nodes, this value is 0. + * + * @note When an item is added lastModified is set to the same value as + * dateAdded. + */ + readonly attribute PRTime lastModified; + + /** + * For uri nodes, this is a sorted list of the tags, delimited with commans, + * for the uri represented by this node. Otherwise this is an empty string. + */ + readonly attribute AString tags; + + /** + * The unique ID associated with the page. It my return an empty string + * if the result node is a non-URI node. + */ + readonly attribute ACString pageGuid; + + /** + * The unique ID associated with the bookmark. It returns an empty string + * if the result node is not associated with a bookmark, a folder or a + * separator. + */ + readonly attribute ACString bookmarkGuid; + + /** + * The unique ID associated with the history visit. For node types other than + * history visit nodes, this value is -1. + */ + readonly attribute long long visitId; + + /** + * The unique ID associated with visit node which was the referrer of this + * history visit. For node types other than history visit nodes, or visits + * without any known referrer, this value is -1. + */ + readonly attribute long long fromVisitId; + + /** + * The transition type associated with this visit. For node types other than + * history visit nodes, this value is 0. + */ + readonly attribute unsigned long visitType; +}; + + +/** + * Base class for container results. This includes all types of groupings. + * Bookmark folders and places queries will be QueryResultNodes which extends + * these items. + */ +[scriptable, uuid(3E9CC95F-0D93-45F1-894F-908EEB9866D7)] +interface nsINavHistoryContainerResultNode : nsINavHistoryResultNode +{ + + /** + * Set this to allow descent into the container. When closed, attempting + * to call getChildren or childCount will result in an error. You should + * set this to false when you are done reading. + * + * For HOST and DAY groupings, doing this is free since the children have + * been precomputed. For queries and bookmark folders, being open means they + * will keep themselves up-to-date by listening for updates and re-querying + * as needed. + */ + attribute boolean containerOpen; + + /** + * Indicates whether the container is closed, loading, or opened. Loading + * implies that the container has been opened asynchronously and has not yet + * fully opened. + */ + readonly attribute unsigned short state; + const unsigned short STATE_CLOSED = 0; + const unsigned short STATE_LOADING = 1; + const unsigned short STATE_OPENED = 2; + + /** + * This indicates whether this node "may" have children, and can be used + * when the container is open or closed. When the container is closed, it + * will give you an exact answer if the node can easily be populated (for + * example, a bookmark folder). If not (for example, a complex history query), + * it will return true. When the container is open, it will always be + * accurate. It is intended to be used to see if we should draw the "+" next + * to a tree item. + */ + readonly attribute boolean hasChildren; + + /** + * This gives you the children of the nodes. It is preferrable to use this + * interface over the array one, since it avoids creating an nsIArray object + * and the interface is already the correct type. + * + * @throws NS_ERROR_NOT_AVAILABLE if containerOpen is false. + */ + readonly attribute unsigned long childCount; + nsINavHistoryResultNode getChild(in unsigned long aIndex); + + /** + * Get the index of a direct child in this container. + * + * @param aNode + * a result node. + * + * @return aNode's index in this container. + * @throws NS_ERROR_NOT_AVAILABLE if containerOpen is false. + * @throws NS_ERROR_INVALID_ARG if aNode isn't a direct child of this + * container. + */ + unsigned long getChildIndex(in nsINavHistoryResultNode aNode); +}; + + +/** + * Used for places queries and as a base for bookmark folders. + * + * Note that if you request places to *not* be expanded in the options that + * generated this node, this item will report it has no children and never try + * to populate itself. + */ +[scriptable, uuid(62817759-4FEE-44A3-B58C-3E2F5AFC9D0A)] +interface nsINavHistoryQueryResultNode : nsINavHistoryContainerResultNode +{ + /** + * Get the query which builds this node's children. + * Only valid for RESULT_TYPE_QUERY nodes. + */ + readonly attribute nsINavHistoryQuery query; + + /** + * Get the options which group this node's children. + * Only valid for RESULT_TYPE_QUERY nodes. + */ + readonly attribute nsINavHistoryQueryOptions queryOptions; + + /** + * For both simple folder queries and folder shortcut queries, this is set to + * the concrete itemId of the folder (i.e. for folder shortcuts it's the + * target folder id). Otherwise, this is set to -1. + */ + readonly attribute long long folderItemId; + + /** + * For both simple folder queries and folder shortcut queries, this is set to + * the concrete guid of the folder (i.e. for folder shortcuts it's the target + * folder guid). Otherwise, this is set to an empty string. + */ + readonly attribute ACString targetFolderGuid; +}; + + +/** + * Allows clients to observe what is happening to a result as it updates itself + * according to history and bookmark system events. Register this observer on a + * result using nsINavHistoryResult::addObserver. + */ +[scriptable, uuid(f62d8b6b-3c4e-4a9f-a897-db605d0b7a0f)] +interface nsINavHistoryResultObserver : nsISupports +{ + /** + * Whether the observer is interested into history details changes. + * Those include visits additions and removals. If the observer doesn't + * provide this attribute, it will default to false. + * In practice, the observer won't receive nodeHistoryDetailsChanged. + * Note: this is only read when the observer is added, it cannot be changed + * dynamically. + */ + readonly attribute boolean skipHistoryDetailsNotifications; + + /** + * Called when 'aItem' is inserted into 'aParent' at index 'aNewIndex'. + * The item previously at index (if any) and everything below it will have + * been shifted down by one. The item may be a container or a leaf. + */ + void nodeInserted(in nsINavHistoryContainerResultNode aParent, + in nsINavHistoryResultNode aNode, + in unsigned long aNewIndex); + + /** + * Called whan 'aItem' is removed from 'aParent' at 'aOldIndex'. The item + * may be a container or a leaf. This function will be called after the item + * has been removed from its parent list, but before anything else (including + * NULLing out the item's parent) has happened. + */ + void nodeRemoved(in nsINavHistoryContainerResultNode aParent, + in nsINavHistoryResultNode aItem, + in unsigned long aOldIndex); + + /** + * Called whan 'aItem' is moved from 'aOldParent' at 'aOldIndex' to + * aNewParent at aNewIndex. The item may be a container or a leaf. + * + * XXX: at the moment, this method is called only when an item is moved + * within the same container. When an item is moved between containers, + * a new node is created for the item, and the itemRemoved/itemAdded methods + * are used. + */ + void nodeMoved(in nsINavHistoryResultNode aNode, + in nsINavHistoryContainerResultNode aOldParent, + in unsigned long aOldIndex, + in nsINavHistoryContainerResultNode aNewParent, + in unsigned long aNewIndex); + + /** + * Called right after aNode's title has changed. + * + * @param aNode + * a result node + * @param aNewTitle + * the new title + */ + void nodeTitleChanged(in nsINavHistoryResultNode aNode, + in AUTF8String aNewTitle); + + /** + * Called right after aNode's uri property has changed. + * + * @param aNode + * a result node + * @param aNewURI + * the old uri + */ + void nodeURIChanged(in nsINavHistoryResultNode aNode, + in AUTF8String aOldURI); + + /** + * Called right after aNode's icon property has changed. + * + * @param aNode + * a result node + * + * @note: The new icon is accessible through aNode.icon. + */ + void nodeIconChanged(in nsINavHistoryResultNode aNode); + + /** + * Called right after aNode's time property or accessCount property, or both, + * have changed. + * + * @param aNode + * a uri result node + * @param aOldVisitDate + * the old visit date + * @param aOldAccessCount + * the old access-count + */ + void nodeHistoryDetailsChanged(in nsINavHistoryResultNode aNode, + in PRTime aOldVisitDate, + in unsigned long aOldAccessCount); + + /** + * Called when the tags set on the uri represented by aNode have changed. + * + * @param aNode + * a uri result node + * + * @note: The new tags list is accessible through aNode.tags. + */ + void nodeTagsChanged(in nsINavHistoryResultNode aNode); + + /** + * Called right after the aNode's keyword property has changed. + * + * @param aNode + * a uri result node + * @param aNewKeyword + * the new keyword + */ + void nodeKeywordChanged(in nsINavHistoryResultNode aNode, + in AUTF8String aNewKeyword); + + /** + * Called right after aNode's dateAdded property has changed. + * + * @param aNode + * a result node + * @param aNewValue + * the new value of the dateAdded property + */ + void nodeDateAddedChanged(in nsINavHistoryResultNode aNode, + in PRTime aNewValue); + + /** + * Called right after aNode's dateModified property has changed. + * + * @param aNode + * a result node + * @param aNewValue + * the new value of the dateModified property + */ + void nodeLastModifiedChanged(in nsINavHistoryResultNode aNode, + in PRTime aNewValue); + + /** + * Called after a container changes state. + * + * @param aContainerNode + * The container that has changed state. + * @param aOldState + * The state that aContainerNode has transitioned out of. + * @param aNewState + * The state that aContainerNode has transitioned into. + */ + void containerStateChanged(in nsINavHistoryContainerResultNode aContainerNode, + in unsigned long aOldState, + in unsigned long aNewState); + + /** + * Called when something significant has happened within the container. The + * contents of the container should be re-built. + * + * @param aContainerNode + * the container node to invalidate + */ + void invalidateContainer(in nsINavHistoryContainerResultNode aContainerNode); + + /** + * This is called to indicate to the UI that the sort has changed to the + * given mode. For trees, for example, this would update the column headers + * to reflect the sorting. For many other types of views, this won't be + * applicable. + * + * @param sortingMode One of nsINavHistoryQueryOptions.SORT_BY_* that + * indicates the new sorting mode. + * + * This only is expected to update the sorting UI. invalidateAll() will also + * get called if the sorting changes to update everything. + */ + void sortingChanged(in unsigned short sortingMode); + + /** + * This is called to indicate that a batch operation is about to start or end. + * The observer could want to disable some events or updates during batches, + * since multiple operations are packed in a short time. + * For example treeviews could temporarily suppress select notifications. + * + * @param aToggleMode + * true if a batch is starting, false if it's ending. + */ + void batching(in boolean aToggleMode); + + /** + * Called by the result when this observer is added. + */ + attribute nsINavHistoryResult result; +}; + + +/** + * The result of a history/bookmark query. + */ +[scriptable, uuid(c2229ce3-2159-4001-859c-7013c52f7619)] +interface nsINavHistoryResult : nsISupports +{ + /** + * Sorts all nodes recursively by the given parameter, one of + * nsINavHistoryQueryOptions.SORT_BY_* This will update the corresponding + * options for this result, so that re-using the current options/queries will + * always give you the current view. + */ + attribute unsigned short sortingMode; + + /** + * Whether or not notifications on result changes are suppressed. + * Initially set to false. + * + * Use this to avoid flickering and to improve performance when you + * do temporary changes to the result structure (e.g. when searching for a + * node recursively). + */ + attribute boolean suppressNotifications; + + /** + * Adds an observer for changes done in the result. + * + * @param aObserver + * a result observer. + * @param aOwnsWeak + * If false, the result will keep an owning reference to the observer, + * which must be removed using removeObserver. + * If true, the result will keep a weak reference to the observer, which + * must implement nsISupportsWeakReference. + * + * @see nsINavHistoryResultObserver + */ + void addObserver(in nsINavHistoryResultObserver aObserver, + [optional] in boolean aOwnsWeak); + + /** + * Removes an observer that was added by addObserver. + * + * @param aObserver + * a result observer that was added by addObserver. + */ + void removeObserver(in nsINavHistoryResultObserver aObserver); + + /** + * This is the root of the results. Remember that you need to open all + * containers for their contents to be valid. + * + * When a result goes out of scope it will continue to observe changes till + * it is cycle collected. While the result waits to be collected it will stay + * in memory, and continue to update itself, potentially causing unwanted + * additional work. When you close the root node the result will stop + * observing changes, so it is good practice to close the root node when you + * are done with a result, since that will avoid unwanted performance hits. + */ + readonly attribute nsINavHistoryContainerResultNode root; + + /** + * Notifies you that a bunch of things are about to change, don't do any + * heavy-duty processing until onEndUpdateBatch is called. + */ + void onBeginUpdateBatch(); + + /** + * Notifies you that we are done doing a bunch of things and you should go + * ahead and update UI, etc. + */ + void onEndUpdateBatch(); +}; + + +/** + * This object encapsulates all the query parameters you're likely to need + * when building up history UI. All parameters are ANDed together. + * + * This is not intended to be a super-general query mechanism. This was designed + * so that most queries can be done in only one SQL query. This is important + * because, if the user has their profile on a networked drive, query latency + * can be non-negligible. + */ + +[scriptable, uuid(dc87ae79-22f1-4dcf-975b-852b01d210cb)] +interface nsINavHistoryQuery : nsISupports +{ + /** + * Time range for results (INCLUSIVE). The *TimeReference is one of the + * constants TIME_RELATIVE_* which indicates how to interpret the + * corresponding time value. + * TIME_RELATIVE_EPOCH (default): + * The time is relative to Jan 1 1970 GMT, (this is a normal PRTime) + * TIME_RELATIVE_TODAY: + * The time is relative to this morning at midnight. Normally used for + * queries relative to today. For example, a "past week" query would be + * today-6 days -> today+1 day + * TIME_RELATIVE_NOW: + * The time is relative to right now. + * + * Note: PRTime is in MICROseconds since 1 Jan 1970. Javascript date objects + * are expressed in MILLIseconds since 1 Jan 1970. + * + * As a special case, a 0 time relative to TIME_RELATIVE_EPOCH indicates that + * the time is not part of the query. This is the default, so an empty query + * will match any time. The has* functions return whether the corresponding + * time is considered. + * + * You can read absolute*Time to get the time value that the currently loaded + * reference points + offset resolve to. + */ + const unsigned long TIME_RELATIVE_EPOCH = 0; + const unsigned long TIME_RELATIVE_TODAY = 1; + const unsigned long TIME_RELATIVE_NOW = 2; + + attribute PRTime beginTime; + attribute unsigned long beginTimeReference; + readonly attribute boolean hasBeginTime; + readonly attribute PRTime absoluteBeginTime; + + attribute PRTime endTime; + attribute unsigned long endTimeReference; + readonly attribute boolean hasEndTime; + readonly attribute PRTime absoluteEndTime; + + /** + * Text search terms. + */ + attribute AString searchTerms; + readonly attribute boolean hasSearchTerms; + + /** + * Set lower or upper limits for how many times an item has been + * visited. The default is -1, and in that case all items are + * matched regardless of their visit count. + */ + attribute long minVisits; + attribute long maxVisits; + + /** + * When the set of transitions is nonempty, results are limited to pages which + * have at least one visit for each of the transition types. + * @note: For searching on more than one transition this can be very slow. + * + * Limit results to the specified list of transition types. + */ + void setTransitions(in Array transitions); + + /** + * Get the transitions set for this query. + */ + Array getTransitions(); + + /** + * Get the count of the set query transitions. + */ + readonly attribute unsigned long transitionCount; + + /** + * When set, returns only bookmarked items, when unset, returns anything. Setting this + * is equivalent to listing all bookmark folders in the 'folders' parameter. + */ + attribute boolean onlyBookmarked; + + /** + * This controls the meaning of 'domain', and whether it is an exact match + * 'domainIsHost' = true, or hierarchical (= false). + */ + attribute boolean domainIsHost; + + /** + * This is the host or domain name (controlled by domainIsHost). When + * domainIsHost, domain only does exact matching on host names. Otherwise, + * it will return anything whose host name ends in 'domain'. + * + * This one is a little different than most. Setting it to an empty string + * is a real query and will match any URI that has no host name (local files + * and such). Set this to NULL (in C++ use SetIsVoid) if you don't want + * domain matching. + */ + attribute AUTF8String domain; + readonly attribute boolean hasDomain; + + /** + * This is a URI to match, to, for example, find out every time you visited + * a given URI. This is an exact match. + */ + attribute nsIURI uri; + readonly attribute boolean hasUri; + + /** + * Test for existence or non-existence of a given annotation. We don't + * currently support >1 annotation name per query. If 'annotationIsNot' is + * true, we test for the non-existence of the specified annotation. + * + * Testing for not annotation will do the same thing as a normal query and + * remove everything that doesn't have that annotation. Asking for things + * that DO have a given annotation is a little different. It also includes + * things that have never been visited. This allows place queries to be + * returned as well as anything else that may have been tagged with an + * annotation. This will only work for RESULTS_AS_URI since there will be + * no visits for these items. + */ + attribute boolean annotationIsNot; + attribute AUTF8String annotation; + readonly attribute boolean hasAnnotation; + + /** + * Limit results to items that are tagged with all of the given tags. This + * attribute must be set to an array of strings. When called as a getter it + * will return an array of strings sorted ascending in lexicographical order. + * The array may be empty in either case. Duplicate tags may be specified + * when setting the attribute, but the getter returns only unique tags. + */ + attribute nsIVariant tags; + + /** + * If 'tagsAreNot' is true, the results are instead limited to items that + * are not tagged with any of the given tags. This attribute is used in + * conjunction with the 'tags' attribute. + */ + attribute boolean tagsAreNot; + + /** + * Limit results to items that are in all of the given folders. + */ + Array getParents(); + readonly attribute unsigned long parentCount; + + /** + * This is not recursive so results will be returned from the first level of + * that folder. + */ + void setParents(in Array aGuids); + + /** + * Creates a new query item with the same parameters of this one. + */ + nsINavHistoryQuery clone(); +}; + +/** + * This object represents the global options for executing a query. + */ +[scriptable, uuid(8198dfa7-8061-4766-95cb-fa86b3c00a47)] +interface nsINavHistoryQueryOptions : nsISupports +{ + /** + * You can ask for the results to be pre-sorted. Since the DB has indices + * of many items, it can produce sorted results almost for free. These should + * be self-explanatory. + * + * Note: re-sorting is slower, as is sorting by title or when you have a + * host name. + * + * For bookmark items, SORT_BY_NONE means sort by the natural bookmark order. + */ + const unsigned short SORT_BY_NONE = 0; + const unsigned short SORT_BY_TITLE_ASCENDING = 1; + const unsigned short SORT_BY_TITLE_DESCENDING = 2; + const unsigned short SORT_BY_DATE_ASCENDING = 3; + const unsigned short SORT_BY_DATE_DESCENDING = 4; + const unsigned short SORT_BY_URI_ASCENDING = 5; + const unsigned short SORT_BY_URI_DESCENDING = 6; + const unsigned short SORT_BY_VISITCOUNT_ASCENDING = 7; + const unsigned short SORT_BY_VISITCOUNT_DESCENDING = 8; + const unsigned short SORT_BY_DATEADDED_ASCENDING = 11; + const unsigned short SORT_BY_DATEADDED_DESCENDING = 12; + const unsigned short SORT_BY_LASTMODIFIED_ASCENDING = 13; + const unsigned short SORT_BY_LASTMODIFIED_DESCENDING = 14; + const unsigned short SORT_BY_TAGS_ASCENDING = 17; + const unsigned short SORT_BY_TAGS_DESCENDING = 18; + const unsigned short SORT_BY_FRECENCY_ASCENDING = 21; + const unsigned short SORT_BY_FRECENCY_DESCENDING = 22; + + /** + * "URI" results, one for each URI visited in the range. Individual result + * nodes will be of type "URI". + */ + const unsigned short RESULTS_AS_URI = 0; + + /** + * "Visit" results, with one for each time a page was visited (this will + * often give you multiple results for one URI). Individual result nodes will + * have type "Visit" + * + * @note This result type is only supported by QUERY_TYPE_HISTORY. + */ + const unsigned short RESULTS_AS_VISIT = 1; + + /** + * This returns query nodes for each predefined date range where we + * had visits. The node contains information how to load its content: + * - visits for the given date range will be loaded. + * + * @note This result type is only supported by QUERY_TYPE_HISTORY. + */ + const unsigned short RESULTS_AS_DATE_QUERY = 3; + + /** + * This returns nsINavHistoryQueryResultNode nodes for each site where we + * have visits. The node contains information how to load its content: + * - last visit for each url in the given host will be loaded. + * + * @note This result type is only supported by QUERY_TYPE_HISTORY. + */ + const unsigned short RESULTS_AS_SITE_QUERY = 4; + + /** + * This returns nsINavHistoryQueryResultNode nodes for each day where we + * have visits. The node contains information how to load its content: + * - list of hosts visited in the given period will be loaded. + * + * @note This result type is only supported by QUERY_TYPE_HISTORY. + */ + const unsigned short RESULTS_AS_DATE_SITE_QUERY = 5; + + /** + * This returns nsINavHistoryQueryResultNode nodes for each tag. + * The node contains information how to load its content: + * - list of bookmarks with the given tag will be loaded. + * + * @note Setting this resultType will force queryType to QUERY_TYPE_BOOKMARKS. + */ + const unsigned short RESULTS_AS_TAGS_ROOT = 6; + + /** + * DEPRECATED: This exists for Sync and also to avoid reusing this number. + */ + const unsigned short RESULTS_AS_TAG_CONTENTS = 7; + + /** + * This returns nsINavHistoryQueryResultNode nodes for each top-level bookmark + * root. + * + * @note Setting this resultType will force queryType to QUERY_TYPE_BOOKMARKS. + */ + const unsigned short RESULTS_AS_ROOTS_QUERY = 8; + + /** + * This returns nsINavHistoryQueryResultNode for each left-pane root. + */ + const unsigned short RESULTS_AS_LEFT_PANE_QUERY = 9; + + /** + * The sorting mode to be used for this query. + * mode is one of SORT_BY_* + */ + attribute unsigned short sortingMode; + + /** + * Sets the result type. One of RESULT_TYPE_* which includes how URIs are + * represented. + */ + attribute unsigned short resultType; + + /** + * This option excludes all URIs and separators from a bookmarks query. + * This would be used if you just wanted a list of bookmark folders and + * queries (such as the left pane of the places page). + * Defaults to false. + */ + attribute boolean excludeItems; + + /** + * Set to true to exclude queries ("place:" URIs) from the query results. + * Simple folder queries (bookmark folder symlinks) will still be included. + * Defaults to false. + */ + attribute boolean excludeQueries; + + /** + * When set, allows items with "place:" URIs to appear as containers, + * with the container's contents filled in from the stored query. + * If not set, these will appear as normal items. Doesn't do anything if + * excludeQueries is set. Defaults to false. + * + * Note that this has no effect on folder links, which are place: URIs + * returned by nsINavBookmarkService.GetFolderURI. These are always expanded + * and will appear as bookmark folders. + */ + attribute boolean expandQueries; + + /** + * Some pages in history are marked "hidden" and thus don't appear by default + * in queries. These include automatic framed visits and redirects. Setting + * this attribute will return all pages, even hidden ones. Does nothing for + * bookmark queries. Defaults to false. + */ + attribute boolean includeHidden; + + /** + * This is the maximum number of results that you want. The query is executed, + * the results are sorted, and then the top 'maxResults' results are taken + * and returned. Set to 0 (the default) to get all results. + * + * THIS DOES NOT WORK IN CONJUNCTION WITH SORTING BY TITLE. This is because + * sorting by title requires us to sort after using locale-sensetive sorting + * (as opposed to letting the database do it for us). + * + * Instead, we get the result ordered by date, pick the maxResult most recent + * ones, and THEN sort by title. + */ + attribute unsigned long maxResults; + + const unsigned short QUERY_TYPE_HISTORY = 0; + const unsigned short QUERY_TYPE_BOOKMARKS = 1; + /* Unified queries are not yet implemented. See bug 378798 */ + const unsigned short QUERY_TYPE_UNIFIED = 2; + + /** + * The type of search to use when querying the DB; This attribute is only + * honored by query nodes. It is silently ignored for simple folder queries. + */ + attribute unsigned short queryType; + + /** + * When this is true, the root container node generated by these options and + * its descendant containers will be opened asynchronously if they support it. + * This is false by default. + * + * @note Currently only bookmark folder containers support being opened + * asynchronously. + */ + attribute boolean asyncEnabled; + + /** + * Creates a new options item with the same parameters of this one. + */ + nsINavHistoryQueryOptions clone(); +}; + +[scriptable, uuid(20c974ff-ee16-4828-9326-1b7c9e036622)] +interface nsINavHistoryService : nsISupports +{ + /** + * System Notifications: + * + * places-init-complete - Sent once the History service is completely + * initialized successfully. + * places-database-locked - Sent if initialization of the History service + * failed due to the inability to open the places.sqlite + * for access reasons. + */ + + /** + * This transition type means the user followed a link and got a new toplevel + * window. + */ + const unsigned long TRANSITION_LINK = 1; + + /** + * This transition type means that the user typed the page's URL in the + * URL bar or selected it from URL bar autocomplete results, clicked on + * it from a history query (from the History sidebar, History menu, + * or history query in the personal toolbar or Places organizer. + */ + const unsigned long TRANSITION_TYPED = 2; + + /** + * This transition is set when the user followed a bookmark to get to the + * page. + */ + const unsigned long TRANSITION_BOOKMARK = 3; + + /** + * This transition type is set when some inner content is loaded. This is + * true of all images on a page, and the contents of the iframe. It is also + * true of any content in a frame if the user did not explicitly follow + * a link to get there. + */ + const unsigned long TRANSITION_EMBED = 4; + + /** + * Set when the transition was a permanent redirect. + */ + const unsigned long TRANSITION_REDIRECT_PERMANENT = 5; + + /** + * Set when the transition was a temporary redirect. + */ + const unsigned long TRANSITION_REDIRECT_TEMPORARY = 6; + + /** + * Set when the transition is a download. + */ + const unsigned long TRANSITION_DOWNLOAD = 7; + + /** + * This transition type means the user followed a link and got a visit in + * a frame. + */ + const unsigned long TRANSITION_FRAMED_LINK = 8; + + /** + * This transition type means the page has been reloaded. + */ + const unsigned long TRANSITION_RELOAD = 9; + + /** + * Set when database is coherent + */ + const unsigned short DATABASE_STATUS_OK = 0; + + /** + * Set when database did not exist and we created a new one. + */ + const unsigned short DATABASE_STATUS_CREATE = 1; + + /** + * Set when database was corrupt and we replaced it with a new one. + */ + const unsigned short DATABASE_STATUS_CORRUPT = 2; + + /** + * Set when database schema has been upgraded. + */ + const unsigned short DATABASE_STATUS_UPGRADED = 3; + + /** + * Set when database couldn't be opened. + */ + const unsigned short DATABASE_STATUS_LOCKED = 4; + + /** + * Insert this value into moz_historyvisits if the visit source is organic. + */ + const unsigned short VISIT_SOURCE_ORGANIC = 0; + + /** + * Insert this value into moz_historyvisits if the visit source is sponsored. + */ + const unsigned short VISIT_SOURCE_SPONSORED = 1; + + /** + * Insert this value into moz_historyvisits if the visit source is bookmarked. + */ + const unsigned short VISIT_SOURCE_BOOKMARKED = 2; + + /** + * Insert this value into moz_historyvisits if the visit source is searched. + */ + const unsigned short VISIT_SOURCE_SEARCHED = 3; + + /** + * Returns the current database status + */ + readonly attribute unsigned short databaseStatus; + + /** + * This is just like markPageAsTyped (in nsIBrowserHistory, also implemented + * by the history service), but for bookmarks. It declares that the given URI + * is being opened as a result of following a bookmark. If this URI is loaded + * soon after this message has been received, that transition will be marked + * as following a bookmark. + */ + void markPageAsFollowedBookmark(in nsIURI aURI); + + /** + * Designates the url as having been explicitly typed in by the user. + * + * @param aURI + * URI of the page to be marked. + */ + void markPageAsTyped(in nsIURI aURI); + + /** + * Designates the url as coming from a link explicitly followed by + * the user (for example by clicking on it). + * + * @param aURI + * URI of the page to be marked. + */ + void markPageAsFollowedLink(in nsIURI aURI); + + /** + * Returns true if this URI would be added to the history. You don't have to + * worry about calling this, adding a visit will always check before + * actually adding the page. This function is public because some components + * may want to check if this page would go in the history (i.e. for + * annotations). + */ + boolean canAddURI(in nsIURI aURI); + + /** + * This returns a new query object that you can pass to executeQuer[y/ies]. + * It will be initialized to all empty (so using it will give you all history). + */ + nsINavHistoryQuery getNewQuery(); + + /** + * This returns a new options object that you can pass to executeQuer[y/ies] + * after setting the desired options. + */ + nsINavHistoryQueryOptions getNewQueryOptions(); + + /** + * Executes a single query. + */ + nsINavHistoryResult executeQuery(in nsINavHistoryQuery aQuery, + in nsINavHistoryQueryOptions options); + + /** + * Converts a query URI-like string to a query object. + */ + void queryStringToQuery(in AUTF8String aQueryString, + out nsINavHistoryQuery aQuery, + out nsINavHistoryQueryOptions options); + + /** + * Converts a query into an equivalent string that can be persisted. Inverse + * of queryStringToQuery() + */ + AUTF8String queryToQueryString(in nsINavHistoryQuery aQuery, + in nsINavHistoryQueryOptions options); + + /** + * True if history is disabled. currently, + * history is disabled if the places.history.enabled pref is false. + */ + readonly attribute boolean historyDisabled; + + /** + * Generate a guid. + * Guids can be used for any places purposes (history, bookmarks, etc.) + * Returns null if the generation of the guid failed. + */ + ACString makeGuid(); + + /** + * Returns a 48-bit hash for a URI spec. + * + * @param aSpec + * The URI spec to hash. + * @param aMode + * The hash mode: `""` (default), `"prefix_lo"`, or `"prefix_hi"`. + */ + unsigned long long hashURL(in ACString aSpec, [optional] in ACString aMode); + + /** + * Resets and recalculates the origin frecency statistics that are kept in the + * moz_meta table. + * + * @param aCallback + * Called when the recalculation is complete. The arguments passed to + * the observer are not defined. + */ + void recalculateOriginFrecencyStats([optional] in nsIObserver aCallback); + + /** + * Whether frecency is in the process of being decayed. The value can also + * be read through the is_frecency_decaying() SQL function exposed by Places + * database connections. + */ + attribute boolean isFrecencyDecaying; + + /** + * This is set to true when a frecency is invalidated and set back to false + * when all the outdated values have been recalculated. + */ + attribute boolean shouldStartFrecencyRecalculation; + + /** + * The database connection used by Places. + */ + readonly attribute mozIStorageConnection DBConnection; + + /** + * Asynchronously executes the statement created from a query. + * + * @see nsINavHistoryService::executeQuery + * @note THIS IS A TEMPORARY API. Don't rely on it, since it will be replaced + * in future versions by a real async querying API. + * @note Results obtained from this method differ from results obtained from + * executeQuery, because there is additional filtering and sorting + * done by the latter. Thus you should use executeQuery, unless you + * are absolutely sure that the returned results are fine for + * your use-case. + */ + mozIStoragePendingStatement asyncExecuteLegacyQuery( + in nsINavHistoryQuery aQuery, + in nsINavHistoryQueryOptions aOptions, + in mozIStorageStatementCallback aCallback); + + /** + * Hook for clients who need to perform actions during/by the end of + * the shutdown of the database. + * May be null if it's too late to get one. + */ + readonly attribute nsIAsyncShutdownClient shutdownClient; + + /** + * Hook for internal clients who need to perform actions just before the + * connection gets closed. + * May be null if it's too late to get one. + */ + readonly attribute nsIAsyncShutdownClient connectionShutdownClient; +}; diff --git a/toolkit/components/places/nsIPlacesPreviewsHelperService.idl b/toolkit/components/places/nsIPlacesPreviewsHelperService.idl new file mode 100644 index 0000000000..7293532ffb --- /dev/null +++ b/toolkit/components/places/nsIPlacesPreviewsHelperService.idl @@ -0,0 +1,20 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * A service which returns information about file paths where the + * previews for URLs are stored. These previews are used by the + * moz-page-thumb protocol + */ + +[scriptable, uuid(bd0a4d3b-ff26-4d4d-9a62-a513e1c1bf92)] +interface nsIPlacesPreviewsHelperService : nsISupports +{ + /** + * Returns the full file path containing the screenshot for a given URL + */ + AString getFilePathForURL(in AString aURL); +}; diff --git a/toolkit/components/places/nsITaggingService.idl b/toolkit/components/places/nsITaggingService.idl new file mode 100644 index 0000000000..acda63b583 --- /dev/null +++ b/toolkit/components/places/nsITaggingService.idl @@ -0,0 +1,66 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; +interface nsIVariant; + +[scriptable, uuid(9759bd0e-78e2-4421-9ed1-c676e1af3513)] +interface nsITaggingService : nsISupports +{ + /** + * Tags a URL with the given set of tags. Current tags set for the URL + * persist. Tags in aTags which are already set for the given URL are + * ignored. + * + * @param aURI + * the URL to tag. + * @param aTags + * Array of tags to set for the given URL. Each element within the + * array can be either a tag name (non-empty string) or a concrete + * itemId of a tag container. + * @param [optional] aSource + * A change source constant from nsINavBookmarksService::SOURCE_*. + * Defaults to SOURCE_DEFAULT if omitted. + */ + void tagURI(in nsIURI aURI, + in nsIVariant aTags, + [optional] in unsigned short aSource); + + /** + * Removes tags from a URL. Tags from aTags which are not set for the + * given URL are ignored. + * + * @param aURI + * the URL to un-tag. + * @param aTags + * Array of tags to unset. Pass null to remove all tags from the given + * url. Each element within the array can be either a tag name + * (non-empty string) or a concrete itemId of a tag container. + * @param [optional] aSource + * A change source constant from nsINavBookmarksService::SOURCE_*. + * Defaults to SOURCE_DEFAULT if omitted. + */ + void untagURI(in nsIURI aURI, + in nsIVariant aTags, + [optional] in unsigned short aSource); + + /** + * Retrieves all tags set for the given URL. + * + * @param aURI + * a URL. + * @returns array of tags (sorted by name). + */ + Array getTagsForURI(in nsIURI aURI); + +}; + +%{C++ + +#define TAGGING_SERVICE_CID "@mozilla.org/browser/tagging-service;1" + +%} diff --git a/toolkit/components/places/nsNavBookmarks.cpp b/toolkit/components/places/nsNavBookmarks.cpp new file mode 100644 index 0000000000..5dcb0c1b44 --- /dev/null +++ b/toolkit/components/places/nsNavBookmarks.cpp @@ -0,0 +1,1717 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsNavBookmarks.h" + +#include "nsNavHistory.h" +#include "nsPlacesMacros.h" +#include "Helpers.h" + +#include "nsAppDirectoryServiceDefs.h" +#include "nsITaggingService.h" +#include "nsNetUtil.h" +#include "nsIProtocolHandler.h" +#include "nsIObserverService.h" +#include "nsUnicharUtils.h" +#include "nsPrintfCString.h" +#include "nsQueryObject.h" +#include "mozIStorageValueArray.h" +#include "mozilla/Preferences.h" +#include "mozilla/ProfilerLabels.h" +#include "mozilla/storage.h" +#include "mozilla/dom/PlacesBookmarkAddition.h" +#include "mozilla/dom/PlacesBookmarkRemoved.h" +#include "mozilla/dom/PlacesBookmarkTags.h" +#include "mozilla/dom/PlacesBookmarkTime.h" +#include "mozilla/dom/PlacesBookmarkTitle.h" +#include "mozilla/dom/PlacesObservers.h" +#include "mozilla/dom/PlacesVisit.h" + +using namespace mozilla; + +// These columns sit to the right of the kGetInfoIndex_* columns. +const int32_t nsNavBookmarks::kGetChildrenIndex_Guid = 18; +const int32_t nsNavBookmarks::kGetChildrenIndex_Position = 19; +const int32_t nsNavBookmarks::kGetChildrenIndex_Type = 20; +const int32_t nsNavBookmarks::kGetChildrenIndex_PlaceID = 21; +const int32_t nsNavBookmarks::kGetChildrenIndex_SyncStatus = 22; + +using namespace mozilla::dom; +using namespace mozilla::places; + +extern "C" { + +// Returns the total number of Sync changes recorded since Places startup for +// all bookmarks. This function uses C linkage because it's called from the +// Rust synced bookmarks mirror, on the storage thread. Using `get_service` to +// access the bookmarks service from Rust trips a thread-safety assertion, so +// we can't use `nsNavBookmarks::GetTotalSyncChanges`. +int64_t NS_NavBookmarksTotalSyncChanges() { + return nsNavBookmarks::sTotalSyncChanges; +} + +} // extern "C" + +PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavBookmarks, gBookmarksService) + +namespace { + +// Returns the sync change counter increment for a change source constant. +inline int64_t DetermineSyncChangeDelta(uint16_t aSource) { + return aSource == nsINavBookmarksService::SOURCE_SYNC ? 0 : 1; +} + +// Returns the sync status for a new item inserted by a change source. +inline int32_t DetermineInitialSyncStatus(uint16_t aSource) { + if (aSource == nsINavBookmarksService::SOURCE_SYNC) { + return nsINavBookmarksService::SYNC_STATUS_NORMAL; + } + if (aSource == nsINavBookmarksService::SOURCE_RESTORE_ON_STARTUP) { + return nsINavBookmarksService::SYNC_STATUS_UNKNOWN; + } + return nsINavBookmarksService::SYNC_STATUS_NEW; +} + +// Indicates whether an item has been uploaded to the server and +// needs a tombstone on deletion. +inline bool NeedsTombstone(const BookmarkData& aBookmark) { + return aBookmark.syncStatus == nsINavBookmarksService::SYNC_STATUS_NORMAL; +} + +inline nsresult GetTags(nsIURI* aURI, nsTArray& aResult) { + nsresult rv; + nsCOMPtr taggingService = + do_GetService("@mozilla.org/browser/tagging-service;1", &rv); + + if (NS_FAILED(rv)) { + return rv; + } + + return taggingService->GetTagsForURI(aURI, aResult); +} + +} // namespace + +nsNavBookmarks::nsNavBookmarks() : mCanNotify(false) { + NS_ASSERTION(!gBookmarksService, + "Attempting to create two instances of the service!"); + gBookmarksService = this; +} + +nsNavBookmarks::~nsNavBookmarks() { + NS_ASSERTION(gBookmarksService == this, + "Deleting a non-singleton instance of the service"); + if (gBookmarksService == this) gBookmarksService = nullptr; +} + +NS_IMPL_ISUPPORTS(nsNavBookmarks, nsINavBookmarksService, nsIObserver, + nsISupportsWeakReference) + +Atomic nsNavBookmarks::sLastInsertedItemId(0); + +void // static +nsNavBookmarks::StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId) { + MOZ_ASSERT(aTable.EqualsLiteral("moz_bookmarks")); + sLastInsertedItemId = aLastInsertedId; +} + +Atomic nsNavBookmarks::sTotalSyncChanges(0); + +void // static +nsNavBookmarks::NoteSyncChange() { + sTotalSyncChanges++; +} + +nsresult nsNavBookmarks::Init() { + mDB = Database::GetDatabase(); + NS_ENSURE_STATE(mDB); + + nsCOMPtr os = mozilla::services::GetObserverService(); + if (os) { + (void)os->AddObserver(this, TOPIC_PLACES_CONNECTION_CLOSED, true); + } + + mCanNotify = true; + + // DO NOT PUT STUFF HERE that can fail. See observer comment above. + + return NS_OK; +} + +nsresult nsNavBookmarks::AdjustIndices(int64_t aFolderId, int32_t aStartIndex, + int32_t aEndIndex, int32_t aDelta) { + NS_ASSERTION( + aStartIndex >= 0 && aEndIndex <= INT32_MAX && aStartIndex <= aEndIndex, + "Bad indices"); + + nsCOMPtr stmt = mDB->GetStatement( + "UPDATE moz_bookmarks SET position = position + :delta " + "WHERE parent = :parent " + "AND position BETWEEN :from_index AND :to_index"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt32ByName("delta"_ns, aDelta); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("from_index"_ns, aStartIndex); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("to_index"_ns, aEndIndex); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult nsNavBookmarks::AdjustSeparatorsSyncCounter(int64_t aFolderId, + int32_t aStartIndex, + int64_t aSyncChangeDelta) { + MOZ_ASSERT(aStartIndex >= 0, "Bad start position"); + if (!aSyncChangeDelta) { + return NS_OK; + } + + nsCOMPtr stmt = mDB->GetStatement( + "UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + :delta " + "WHERE parent = :parent AND position >= :start_index " + "AND type = :item_type "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("delta"_ns, aSyncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("start_index"_ns, aStartIndex); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("item_type"_ns, TYPE_SEPARATOR); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::GetTagsFolder(int64_t* aRoot) { + int64_t id = mDB->GetTagsFolderId(); + NS_ENSURE_TRUE(id > 0, NS_ERROR_UNEXPECTED); + *aRoot = id; + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::GetTotalSyncChanges(int64_t* aTotalSyncChanges) { + *aTotalSyncChanges = sTotalSyncChanges; + return NS_OK; +} + +nsresult nsNavBookmarks::InsertBookmarkInDB( + int64_t aPlaceId, enum ItemType aItemType, int64_t aParentId, + int32_t aIndex, const nsACString& aTitle, PRTime aDateAdded, + PRTime aLastModified, const nsACString& aParentGuid, int64_t aGrandParentId, + nsIURI* aURI, uint16_t aSource, int64_t* _itemId, nsACString& _guid) { + // Check for a valid itemId. + MOZ_ASSERT(_itemId && (*_itemId == -1 || *_itemId > 0)); + // Check for a valid placeId. + MOZ_ASSERT(aPlaceId && (aPlaceId == -1 || aPlaceId > 0)); + + nsCOMPtr stmt = mDB->GetStatement( + "INSERT INTO moz_bookmarks " + "(id, fk, type, parent, position, title, " + "dateAdded, lastModified, guid, syncStatus, syncChangeCounter) " + "VALUES (:item_id, :page_id, :item_type, :parent, :item_index, " + ":item_title, :date_added, :last_modified, " + ":item_guid, :sync_status, :change_counter)"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv; + if (*_itemId != -1) + rv = stmt->BindInt64ByName("item_id"_ns, *_itemId); + else + rv = stmt->BindNullByName("item_id"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + if (aPlaceId != -1) + rv = stmt->BindInt64ByName("page_id"_ns, aPlaceId); + else + rv = stmt->BindNullByName("page_id"_ns); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->BindInt32ByName("item_type"_ns, aItemType); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("parent"_ns, aParentId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("item_index"_ns, aIndex); + NS_ENSURE_SUCCESS(rv, rv); + + if (aTitle.IsEmpty()) + rv = stmt->BindNullByName("item_title"_ns); + else + rv = stmt->BindUTF8StringByName("item_title"_ns, aTitle); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->BindInt64ByName("date_added"_ns, aDateAdded); + NS_ENSURE_SUCCESS(rv, rv); + + if (aLastModified) { + rv = stmt->BindInt64ByName("last_modified"_ns, aLastModified); + } else { + rv = stmt->BindInt64ByName("last_modified"_ns, aDateAdded); + } + NS_ENSURE_SUCCESS(rv, rv); + + // Could use IsEmpty because our callers check for GUID validity, + // but it doesn't hurt. + bool hasExistingGuid = _guid.Length() == 12; + if (hasExistingGuid) { + MOZ_ASSERT(IsValidGUID(_guid)); + rv = stmt->BindUTF8StringByName("item_guid"_ns, _guid); + NS_ENSURE_SUCCESS(rv, rv); + } else { + nsAutoCString guid; + rv = GenerateGUID(guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("item_guid"_ns, guid); + NS_ENSURE_SUCCESS(rv, rv); + _guid.Assign(guid); + } + + int64_t syncChangeDelta = DetermineSyncChangeDelta(aSource); + rv = stmt->BindInt64ByName("change_counter"_ns, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + uint16_t syncStatus = DetermineInitialSyncStatus(aSource); + rv = stmt->BindInt32ByName("sync_status"_ns, syncStatus); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Remove stale tombstones if we're reinserting an item. + if (hasExistingGuid) { + rv = RemoveTombstone(_guid); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (*_itemId == -1) { + *_itemId = sLastInsertedItemId; + } + + if (aParentId > 0) { + // Update last modified date of the ancestors. + // TODO (bug 408991): Doing this for all ancestors would be slow without a + // nested tree, so for now update only the parent. + rv = SetItemDateInternal(LAST_MODIFIED, syncChangeDelta, aParentId, + aDateAdded); + NS_ENSURE_SUCCESS(rv, rv); + } + + int64_t tagsRootId = mDB->GetTagsFolderId(); + bool isTagging = aGrandParentId == tagsRootId; + if (isTagging) { + // If we're tagging a bookmark, increment the change counter for all + // bookmarks with the URI. + rv = AddSyncChangesForBookmarksWithURI(aURI, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Mark all affected separators as changed + rv = AdjustSeparatorsSyncCounter(aParentId, aIndex + 1, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + // Add a cache entry since we know everything about this bookmark. + BookmarkData bookmark; + bookmark.id = *_itemId; + bookmark.guid.Assign(_guid); + if (!aTitle.IsEmpty()) { + bookmark.title.Assign(aTitle); + } + bookmark.position = aIndex; + bookmark.placeId = aPlaceId; + bookmark.parentId = aParentId; + bookmark.type = aItemType; + bookmark.dateAdded = aDateAdded; + if (aLastModified) + bookmark.lastModified = aLastModified; + else + bookmark.lastModified = aDateAdded; + if (aURI) { + rv = aURI->GetSpec(bookmark.url); + NS_ENSURE_SUCCESS(rv, rv); + } + bookmark.parentGuid = aParentGuid; + bookmark.grandParentId = aGrandParentId; + bookmark.syncStatus = syncStatus; + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::InsertBookmark(int64_t aFolder, nsIURI* aURI, int32_t aIndex, + const nsACString& aTitle, + const nsACString& aGUID, uint16_t aSource, + int64_t* aNewBookmarkId) { + NS_ENSURE_ARG(aURI); + NS_ENSURE_ARG_POINTER(aNewBookmarkId); + NS_ENSURE_ARG_MIN(aIndex, nsINavBookmarksService::DEFAULT_INDEX); + + if (!aGUID.IsEmpty() && !IsValidGUID(aGUID)) return NS_ERROR_INVALID_ARG; + + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + int64_t placeId; + nsAutoCString placeGuid; + nsresult rv = history->GetOrCreateIdForPage(aURI, &placeId, placeGuid); + NS_ENSURE_SUCCESS(rv, rv); + + // Get the correct index for insertion. This also ensures the parent exists. + int32_t index, folderCount; + int64_t grandParentId; + nsAutoCString folderGuid; + rv = FetchFolderInfo(aFolder, &folderCount, folderGuid, &grandParentId); + NS_ENSURE_SUCCESS(rv, rv); + if (aIndex == nsINavBookmarksService::DEFAULT_INDEX || + aIndex >= folderCount) { + index = folderCount; + } else { + index = aIndex; + // Create space for the insertion. + rv = AdjustIndices(aFolder, index, INT32_MAX, 1); + NS_ENSURE_SUCCESS(rv, rv); + } + + *aNewBookmarkId = -1; + PRTime dateAdded = RoundedPRNow(); + nsAutoCString guid(aGUID); + nsCString title; + TruncateTitle(aTitle, title); + + rv = InsertBookmarkInDB(placeId, BOOKMARK, aFolder, index, title, dateAdded, + 0, folderGuid, grandParentId, aURI, aSource, + aNewBookmarkId, guid); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + if (!mCanNotify) { + return NS_OK; + } + + Sequence> notifications; + nsAutoCString utf8spec; + aURI->GetSpec(utf8spec); + int64_t tagsRootId = mDB->GetTagsFolderId(); + + RefPtr bookmark = new PlacesBookmarkAddition(); + bookmark->mItemType = TYPE_BOOKMARK; + bookmark->mId = *aNewBookmarkId; + bookmark->mParentId = aFolder; + bookmark->mIndex = index; + bookmark->mUrl.Assign(NS_ConvertUTF8toUTF16(utf8spec)); + bookmark->mTitle.Assign(NS_ConvertUTF8toUTF16(title)); + bookmark->mDateAdded = dateAdded / 1000; + bookmark->mGuid.Assign(guid); + bookmark->mParentGuid.Assign(folderGuid); + bookmark->mSource = aSource; + bookmark->mIsTagging = grandParentId == tagsRootId; + bool success = !!notifications.AppendElement(bookmark.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + // If the bookmark has been added to a tag container, notify all + // bookmark-folder result nodes which contain a bookmark for the new + // bookmark's url. + if (grandParentId == tagsRootId) { + // Notify a tags change to all bookmarks for this URI. + nsTArray bookmarks; + rv = GetBookmarksForURI(aURI, bookmarks); + NS_ENSURE_SUCCESS(rv, rv); + + nsTArray tags; + rv = GetTags(aURI, tags); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < bookmarks.Length(); ++i) { + // Check that bookmarks doesn't include the current tag itemId. + MOZ_ASSERT(bookmarks[i].id != *aNewBookmarkId); + RefPtr tagsChanged = new PlacesBookmarkTags(); + tagsChanged->mId = bookmarks[i].id; + tagsChanged->mItemType = TYPE_BOOKMARK; + tagsChanged->mUrl.Assign(NS_ConvertUTF8toUTF16(utf8spec)); + tagsChanged->mGuid = bookmarks[i].guid; + tagsChanged->mParentGuid = bookmarks[i].parentGuid; + tagsChanged->mTags.Assign(tags); + tagsChanged->mLastModified = bookmarks[i].lastModified / 1000; + tagsChanged->mSource = aSource; + tagsChanged->mIsTagging = false; + success = !!notifications.AppendElement(tagsChanged.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + } + } + + PlacesObservers::NotifyListeners(notifications); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::RemoveItem(int64_t aItemId, uint16_t aSource) { + AUTO_PROFILER_LABEL("nsNavBookmarks::RemoveItem", OTHER); + + BookmarkData bookmark; + nsresult rv = FetchItemInfo(aItemId, bookmark); + NS_ENSURE_SUCCESS(rv, rv); + // Check we're not trying to remove a root. + NS_ENSURE_ARG(bookmark.parentId > 0 && bookmark.grandParentId > 0); + + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + if (bookmark.type == TYPE_FOLDER) { + // Remove all of the folder's children. + rv = RemoveFolderChildren(bookmark.id, aSource); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsCOMPtr stmt = + mDB->GetStatement("DELETE FROM moz_bookmarks WHERE id = :item_id"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = stmt->BindInt64ByName("item_id"_ns, bookmark.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Fix indices in the parent. + if (bookmark.position != DEFAULT_INDEX) { + rv = AdjustIndices(bookmark.parentId, bookmark.position + 1, INT32_MAX, -1); + NS_ENSURE_SUCCESS(rv, rv); + } + + int64_t syncChangeDelta = DetermineSyncChangeDelta(aSource); + + // Add a tombstone for synced items. + if (syncChangeDelta) { + rv = InsertTombstone(bookmark); + NS_ENSURE_SUCCESS(rv, rv); + } + + bookmark.lastModified = RoundedPRNow(); + rv = SetItemDateInternal(LAST_MODIFIED, syncChangeDelta, bookmark.parentId, + bookmark.lastModified); + NS_ENSURE_SUCCESS(rv, rv); + + // Mark all affected separators as changed + rv = AdjustSeparatorsSyncCounter(bookmark.parentId, bookmark.position, + syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + if (bookmark.grandParentId == tagsRootId) { + // If we're removing a tag, increment the change counter for all bookmarks + // with the URI. + rv = AddSyncChangesForBookmarksWithURL(bookmark.url, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr uri; + if (bookmark.type == TYPE_BOOKMARK) { + // A broken url should not interrupt the removal process. + (void)NS_NewURI(getter_AddRefs(uri), bookmark.url); + // We cannot assert since some automated tests are checking this path. + NS_WARNING_ASSERTION(uri, "Invalid URI in RemoveItem"); + } + + if (!mCanNotify) { + return NS_OK; + } + + Sequence> notifications; + RefPtr bookmarkRef = new PlacesBookmarkRemoved(); + bookmarkRef->mItemType = bookmark.type; + bookmarkRef->mId = bookmark.id; + bookmarkRef->mParentId = bookmark.parentId; + bookmarkRef->mIndex = bookmark.position; + if (bookmark.type == TYPE_BOOKMARK) { + bookmarkRef->mUrl.Assign(NS_ConvertUTF8toUTF16(bookmark.url)); + } + bookmarkRef->mTitle.Assign(NS_ConvertUTF8toUTF16(bookmark.title)); + bookmarkRef->mGuid.Assign(bookmark.guid); + bookmarkRef->mParentGuid.Assign(bookmark.parentGuid); + bookmarkRef->mSource = aSource; + bookmarkRef->mIsTagging = + bookmark.parentId == tagsRootId || bookmark.grandParentId == tagsRootId; + bookmarkRef->mIsDescendantRemoval = false; + bool success = !!notifications.AppendElement(bookmarkRef.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + if (bookmark.type == TYPE_BOOKMARK && bookmark.grandParentId == tagsRootId && + uri) { + // If the removed bookmark was child of a tag container, notify a tags + // change to all bookmarks for this URI. + nsTArray bookmarks; + rv = GetBookmarksForURI(uri, bookmarks); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString utf8spec; + uri->GetSpec(utf8spec); + + nsTArray tags; + rv = GetTags(uri, tags); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < bookmarks.Length(); ++i) { + RefPtr tagsChanged = new PlacesBookmarkTags(); + tagsChanged->mId = bookmarks[i].id; + tagsChanged->mItemType = TYPE_BOOKMARK; + tagsChanged->mUrl.Assign(NS_ConvertUTF8toUTF16(utf8spec)); + tagsChanged->mGuid = bookmarks[i].guid; + tagsChanged->mParentGuid = bookmarks[i].parentGuid; + tagsChanged->mTags.Assign(tags); + tagsChanged->mLastModified = bookmarks[i].lastModified / 1000; + tagsChanged->mSource = aSource; + tagsChanged->mIsTagging = false; + success = !!notifications.AppendElement(tagsChanged.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + } + } + + PlacesObservers::NotifyListeners(notifications); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::CreateFolder(int64_t aParent, const nsACString& aTitle, + int32_t aIndex, const nsACString& aGUID, + uint16_t aSource, int64_t* aNewFolderId) { + // NOTE: aParent can be null for root creation, so not checked + NS_ENSURE_ARG_POINTER(aNewFolderId); + NS_ENSURE_ARG_MIN(aIndex, nsINavBookmarksService::DEFAULT_INDEX); + if (!aGUID.IsEmpty() && !IsValidGUID(aGUID)) return NS_ERROR_INVALID_ARG; + + // Get the correct index for insertion. This also ensures the parent exists. + int32_t index = aIndex, folderCount; + int64_t grandParentId; + nsAutoCString folderGuid; + nsresult rv = + FetchFolderInfo(aParent, &folderCount, folderGuid, &grandParentId); + NS_ENSURE_SUCCESS(rv, rv); + + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + if (aIndex == nsINavBookmarksService::DEFAULT_INDEX || + aIndex >= folderCount) { + index = folderCount; + } else { + // Create space for the insertion. + rv = AdjustIndices(aParent, index, INT32_MAX, 1); + NS_ENSURE_SUCCESS(rv, rv); + } + + *aNewFolderId = -1; + PRTime dateAdded = RoundedPRNow(); + nsAutoCString guid(aGUID); + nsCString title; + TruncateTitle(aTitle, title); + + rv = InsertBookmarkInDB(-1, FOLDER, aParent, index, title, dateAdded, 0, + folderGuid, grandParentId, nullptr, aSource, + aNewFolderId, guid); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + + if (mCanNotify) { + Sequence> events; + RefPtr folder = new PlacesBookmarkAddition(); + folder->mItemType = TYPE_FOLDER; + folder->mId = *aNewFolderId; + folder->mParentId = aParent; + folder->mIndex = index; + folder->mTitle.Assign(NS_ConvertUTF8toUTF16(title)); + folder->mDateAdded = dateAdded / 1000; + folder->mGuid.Assign(guid); + folder->mParentGuid.Assign(folderGuid); + folder->mSource = aSource; + folder->mIsTagging = aParent == tagsRootId; + bool success = !!events.AppendElement(folder.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + PlacesObservers::NotifyListeners(events); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::GetDescendantChildren( + int64_t aFolderId, const nsACString& aFolderGuid, int64_t aGrandParentId, + nsTArray& aFolderChildrenArray) { + // New children will be added from this index on. + uint32_t startIndex = aFolderChildrenArray.Length(); + nsresult rv; + { + // Collect children informations. + // Select all children of a given folder, sorted by position. + // This is a LEFT JOIN because not all bookmarks types have a place. + // We construct a result where the first columns exactly match + // kGetInfoIndex_* order, and additionally contains columns for position, + // item_child, and folder_child from moz_bookmarks. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT h.id, h.url, b.title, h.rev_host, h.visit_count, " + "h.last_visit_date, null, b.id, b.dateAdded, b.lastModified, " + "b.parent, null, h.frecency, h.hidden, h.guid, null, null, null, " + "b.guid, b.position, b.type, b.fk, b.syncStatus " + "FROM moz_bookmarks b " + "LEFT JOIN moz_places h ON b.fk = h.id " + "WHERE b.parent = :parent " + "ORDER BY b.position ASC"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = stmt->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasMore)) && hasMore) { + BookmarkData child; + rv = stmt->GetInt64(nsNavHistory::kGetInfoIndex_ItemId, &child.id); + NS_ENSURE_SUCCESS(rv, rv); + child.parentId = aFolderId; + child.grandParentId = aGrandParentId; + child.parentGuid = aFolderGuid; + rv = stmt->GetInt32(kGetChildrenIndex_Type, &child.type); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(kGetChildrenIndex_PlaceID, &child.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt32(kGetChildrenIndex_Position, &child.position); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetUTF8String(kGetChildrenIndex_Guid, child.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt32(kGetChildrenIndex_SyncStatus, &child.syncStatus); + NS_ENSURE_SUCCESS(rv, rv); + + if (child.type == TYPE_BOOKMARK) { + rv = stmt->GetUTF8String(nsNavHistory::kGetInfoIndex_URL, child.url); + NS_ENSURE_SUCCESS(rv, rv); + } + + bool isNull; + rv = stmt->GetIsNull(nsNavHistory::kGetInfoIndex_Title, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = + stmt->GetUTF8String(nsNavHistory::kGetInfoIndex_Title, child.title); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Append item to children's array. + aFolderChildrenArray.AppendElement(child); + } + } + + // Recursively call GetDescendantChildren for added folders. + // We start at startIndex since previous folders are checked + // by previous calls to this method. + uint32_t childCount = aFolderChildrenArray.Length(); + for (uint32_t i = startIndex; i < childCount; ++i) { + if (aFolderChildrenArray[i].type == TYPE_FOLDER) { + // nsTarray assumes that all children can be memmove()d, thus we can't + // just pass aFolderChildrenArray[i].guid to a method that will change + // the array itself. Otherwise, since it's passed by reference, after a + // memmove() it could point to garbage and cause intermittent crashes. + nsCString guid = aFolderChildrenArray[i].guid; + GetDescendantChildren(aFolderChildrenArray[i].id, guid, aFolderId, + aFolderChildrenArray); + } + } + + return NS_OK; +} + +nsresult nsNavBookmarks::RemoveFolderChildren(int64_t aFolderId, + uint16_t aSource) { + AUTO_PROFILER_LABEL("nsNavBookmarks::RemoveFolderChilder", OTHER); + NS_ENSURE_ARG_MIN(aFolderId, 1); + + BookmarkData folder; + nsresult rv = FetchItemInfo(aFolderId, folder); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_ARG(folder.type == TYPE_FOLDER); + NS_ENSURE_ARG(folder.parentId != 0); + + // Fill folder children array recursively. + nsTArray folderChildrenArray; + rv = GetDescendantChildren(folder.id, folder.guid, folder.parentId, + folderChildrenArray); + NS_ENSURE_SUCCESS(rv, rv); + + // Build a string of folders whose children will be removed. + nsCString foldersToRemove; + for (uint32_t i = 0; i < folderChildrenArray.Length(); ++i) { + BookmarkData& child = folderChildrenArray[i]; + + if (child.type == TYPE_FOLDER) { + foldersToRemove.Append(','); + foldersToRemove.AppendInt(child.id); + } + } + + int64_t syncChangeDelta = DetermineSyncChangeDelta(aSource); + + // Delete items from the database now. + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + nsCOMPtr deleteStatement = + mDB->GetStatement(nsLiteralCString("DELETE FROM moz_bookmarks " + "WHERE parent IN (:parent") + + foldersToRemove + ")"_ns); + NS_ENSURE_STATE(deleteStatement); + mozStorageStatementScoper deleteStatementScoper(deleteStatement); + + rv = deleteStatement->BindInt64ByName("parent"_ns, folder.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = deleteStatement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Clean up orphan items annotations. + nsCOMPtr conn = mDB->MainConn(); + if (!conn) { + return NS_ERROR_UNEXPECTED; + } + rv = conn->ExecuteSimpleSQL( + nsLiteralCString("DELETE FROM moz_items_annos " + "WHERE id IN (" + "SELECT a.id from moz_items_annos a " + "LEFT JOIN moz_bookmarks b ON a.item_id = b.id " + "WHERE b.id ISNULL)")); + NS_ENSURE_SUCCESS(rv, rv); + + // Set the lastModified date. + rv = SetItemDateInternal(LAST_MODIFIED, syncChangeDelta, folder.id, + RoundedPRNow()); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + + if (syncChangeDelta) { + nsTArray tombstones(folderChildrenArray.Length()); + PRTime dateRemoved = RoundedPRNow(); + + for (uint32_t i = 0; i < folderChildrenArray.Length(); ++i) { + BookmarkData& child = folderChildrenArray[i]; + if (NeedsTombstone(child)) { + // Write tombstones for synced children. + TombstoneData childTombstone = {child.guid, dateRemoved}; + tombstones.AppendElement(childTombstone); + } + bool isUntagging = child.grandParentId == tagsRootId; + if (isUntagging) { + // Bump the change counter for all tagged bookmarks when removing a tag + // folder. + rv = AddSyncChangesForBookmarksWithURL(child.url, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + rv = InsertTombstones(tombstones); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + Sequence> notifications; + // Call observers in reverse order to serve children before their parent. + for (int32_t i = folderChildrenArray.Length() - 1; i >= 0; --i) { + BookmarkData& child = folderChildrenArray[i]; + + nsCOMPtr uri; + if (child.type == TYPE_BOOKMARK) { + // A broken url should not interrupt the removal process. + (void)NS_NewURI(getter_AddRefs(uri), child.url); + // We cannot assert since some automated tests are checking this path. + NS_WARNING_ASSERTION(uri, "Invalid URI in RemoveFolderChildren"); + } + + if (!mCanNotify) { + return NS_OK; + } + + RefPtr bookmark = new PlacesBookmarkRemoved(); + bookmark->mItemType = TYPE_BOOKMARK; + bookmark->mId = child.id; + bookmark->mParentId = child.parentId; + bookmark->mIndex = child.position; + bookmark->mUrl.Assign(NS_ConvertUTF8toUTF16(child.url)); + bookmark->mTitle.Assign(NS_ConvertUTF8toUTF16(child.title)); + bookmark->mGuid.Assign(child.guid); + bookmark->mParentGuid.Assign(child.parentGuid); + bookmark->mSource = aSource; + bookmark->mIsTagging = (child.grandParentId == tagsRootId); + bookmark->mIsDescendantRemoval = (child.grandParentId != tagsRootId); + bool success = !!notifications.AppendElement(bookmark.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + + if (child.type == TYPE_BOOKMARK && child.grandParentId == tagsRootId && + uri) { + // If the removed bookmark was a child of a tag container, notify all + // bookmark-folder result nodes which contain a bookmark for the removed + // bookmark's url. + nsTArray bookmarks; + rv = GetBookmarksForURI(uri, bookmarks); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString utf8spec; + uri->GetSpec(utf8spec); + + nsTArray tags; + rv = GetTags(uri, tags); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < bookmarks.Length(); ++i) { + RefPtr tagsChanged = new PlacesBookmarkTags(); + tagsChanged->mId = bookmarks[i].id; + tagsChanged->mItemType = TYPE_BOOKMARK; + tagsChanged->mUrl.Assign(NS_ConvertUTF8toUTF16(utf8spec)); + tagsChanged->mGuid = bookmarks[i].guid; + tagsChanged->mParentGuid = bookmarks[i].parentGuid; + tagsChanged->mTags.Assign(tags); + tagsChanged->mLastModified = bookmarks[i].lastModified / 1000; + tagsChanged->mSource = aSource; + tagsChanged->mIsTagging = false; + success = !!notifications.AppendElement(tagsChanged.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + } + } + } + + if (notifications.Length()) { + PlacesObservers::NotifyListeners(notifications); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::FetchItemInfo(int64_t aItemId, + BookmarkData& _bookmark) { + // LEFT JOIN since not all bookmarks have an associated place. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT b.id, h.url, b.title, b.position, b.fk, b.parent, b.type, " + "b.dateAdded, b.lastModified, b.guid, t.guid, t.parent, " + "b.syncStatus " + "FROM moz_bookmarks b " + "LEFT JOIN moz_bookmarks t ON t.id = b.parent " + "LEFT JOIN moz_places h ON h.id = b.fk " + "WHERE b.id = :item_id"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("item_id"_ns, aItemId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasResult) { + return NS_ERROR_INVALID_ARG; + } + + _bookmark.id = aItemId; + rv = stmt->GetUTF8String(1, _bookmark.url); + NS_ENSURE_SUCCESS(rv, rv); + + bool isNull; + rv = stmt->GetIsNull(2, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = stmt->GetUTF8String(2, _bookmark.title); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = stmt->GetInt32(3, &_bookmark.position); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(4, &_bookmark.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(5, &_bookmark.parentId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt32(6, &_bookmark.type); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(7, reinterpret_cast(&_bookmark.dateAdded)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(8, reinterpret_cast(&_bookmark.lastModified)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetUTF8String(9, _bookmark.guid); + NS_ENSURE_SUCCESS(rv, rv); + // Getting properties of the root would show no parent. + rv = stmt->GetIsNull(10, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = stmt->GetUTF8String(10, _bookmark.parentGuid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(11, &_bookmark.grandParentId); + NS_ENSURE_SUCCESS(rv, rv); + } else { + _bookmark.grandParentId = -1; + } + rv = stmt->GetInt32(12, &_bookmark.syncStatus); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult nsNavBookmarks::FetchItemInfo(const nsCString& aGUID, + BookmarkData& _bookmark) { + // LEFT JOIN since not all bookmarks have an associated place. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT b.id, h.url, b.title, b.position, b.fk, b.parent, b.type, " + "b.dateAdded, b.lastModified, t.guid, t.parent, " + "b.syncStatus " + "FROM moz_bookmarks b " + "LEFT JOIN moz_bookmarks t ON t.id = b.parent " + "LEFT JOIN moz_places h ON h.id = b.fk " + "WHERE b.guid = :item_guid"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindUTF8StringByName("item_guid"_ns, aGUID); + NS_ENSURE_SUCCESS(rv, rv); + + _bookmark.guid = aGUID; + + bool hasResult; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasResult) { + return NS_ERROR_INVALID_ARG; + } + + rv = stmt->GetInt64(0, &_bookmark.id); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->GetUTF8String(1, _bookmark.url); + NS_ENSURE_SUCCESS(rv, rv); + + bool isNull; + rv = stmt->GetIsNull(2, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = stmt->GetUTF8String(2, _bookmark.title); + NS_ENSURE_SUCCESS(rv, rv); + } + rv = stmt->GetInt32(3, &_bookmark.position); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(4, &_bookmark.placeId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(5, &_bookmark.parentId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt32(6, &_bookmark.type); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(7, reinterpret_cast(&_bookmark.dateAdded)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(8, reinterpret_cast(&_bookmark.lastModified)); + NS_ENSURE_SUCCESS(rv, rv); + // Getting properties of the root would show no parent. + rv = stmt->GetIsNull(9, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = stmt->GetUTF8String(9, _bookmark.parentGuid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(10, &_bookmark.grandParentId); + NS_ENSURE_SUCCESS(rv, rv); + } else { + _bookmark.grandParentId = -1; + } + rv = stmt->GetInt32(11, &_bookmark.syncStatus); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult nsNavBookmarks::SetItemDateInternal(enum BookmarkDate aDateType, + int64_t aSyncChangeDelta, + int64_t aItemId, PRTime aValue) { + aValue = RoundToMilliseconds(aValue); + + nsCOMPtr stmt = mDB->GetStatement( + "UPDATE moz_bookmarks SET lastModified = :date, " + "syncChangeCounter = syncChangeCounter + :delta " + "WHERE id = :item_id"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("date"_ns, aValue); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("item_id"_ns, aItemId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("delta"_ns, aSyncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // note, we are not notifying the observers + // that the item has changed. + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::SetItemLastModified(int64_t aItemId, PRTime aLastModified, + uint16_t aSource) { + NS_ENSURE_ARG_MIN(aItemId, 1); + + BookmarkData bookmark; + nsresult rv = FetchItemInfo(aItemId, bookmark); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + bool isTagging = bookmark.grandParentId == tagsRootId; + int64_t syncChangeDelta = DetermineSyncChangeDelta(aSource); + + // Round here so that we notify with the right value. + bookmark.lastModified = RoundToMilliseconds(aLastModified); + + if (isTagging) { + // If we're changing a tag, bump the change counter for all tagged + // bookmarks. We use a separate code path to avoid a transaction for + // non-tags. + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + rv = SetItemDateInternal(LAST_MODIFIED, syncChangeDelta, bookmark.id, + bookmark.lastModified); + NS_ENSURE_SUCCESS(rv, rv); + + rv = AddSyncChangesForBookmarksWithURL(bookmark.url, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + } else { + rv = SetItemDateInternal(LAST_MODIFIED, syncChangeDelta, bookmark.id, + bookmark.lastModified); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Note: mDBSetItemDateAdded also sets lastModified to aDateAdded. + + if (mCanNotify) { + Sequence> events; + RefPtr timeChanged = new PlacesBookmarkTime(); + timeChanged->mId = bookmark.id; + timeChanged->mItemType = bookmark.type; + timeChanged->mUrl.Assign(NS_ConvertUTF8toUTF16(bookmark.url)); + timeChanged->mGuid = bookmark.guid; + timeChanged->mParentGuid = bookmark.parentGuid; + timeChanged->mDateAdded = bookmark.dateAdded / 1000; + timeChanged->mLastModified = bookmark.lastModified / 1000; + timeChanged->mSource = aSource; + timeChanged->mIsTagging = + bookmark.parentId == tagsRootId || bookmark.grandParentId == tagsRootId; + bool success = !!events.AppendElement(timeChanged.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + PlacesObservers::NotifyListeners(events); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::AddSyncChangesForBookmarksWithURL( + const nsACString& aURL, int64_t aSyncChangeDelta) { + if (!aSyncChangeDelta) { + return NS_OK; + } + nsCOMPtr uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), aURL); + if (NS_WARN_IF(NS_FAILED(rv))) { + // Ignore sync changes for invalid URLs. + return NS_OK; + } + return AddSyncChangesForBookmarksWithURI(uri, aSyncChangeDelta); +} + +nsresult nsNavBookmarks::AddSyncChangesForBookmarksWithURI( + nsIURI* aURI, int64_t aSyncChangeDelta) { + if (NS_WARN_IF(!aURI) || !aSyncChangeDelta) { + // Ignore sync changes for invalid URIs. + return NS_OK; + } + + nsCOMPtr statement = mDB->GetStatement( + "UPDATE moz_bookmarks SET " + "syncChangeCounter = syncChangeCounter + :delta " + "WHERE type = :type AND " + "fk = (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND " + "url = :url)"); + NS_ENSURE_STATE(statement); + mozStorageStatementScoper scoper(statement); + + nsresult rv = statement->BindInt64ByName("delta"_ns, aSyncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64ByName("type"_ns, + nsINavBookmarksService::TYPE_BOOKMARK); + NS_ENSURE_SUCCESS(rv, rv); + rv = URIBinder::Bind(statement, "url"_ns, aURI); + NS_ENSURE_SUCCESS(rv, rv); + + return statement->Execute(); +} + +nsresult nsNavBookmarks::AddSyncChangesForBookmarksInFolder( + int64_t aFolderId, int64_t aSyncChangeDelta) { + if (!aSyncChangeDelta) { + return NS_OK; + } + + nsCOMPtr statement = mDB->GetStatement( + "UPDATE moz_bookmarks SET " + "syncChangeCounter = syncChangeCounter + :delta " + "WHERE type = :type AND " + "fk = (SELECT fk FROM moz_bookmarks WHERE parent = :parent)"); + NS_ENSURE_STATE(statement); + mozStorageStatementScoper scoper(statement); + + nsresult rv = statement->BindInt64ByName("delta"_ns, aSyncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64ByName("type"_ns, + nsINavBookmarksService::TYPE_BOOKMARK); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult nsNavBookmarks::InsertTombstone(const BookmarkData& aBookmark) { + if (!NeedsTombstone(aBookmark)) { + return NS_OK; + } + nsCOMPtr stmt = mDB->GetStatement( + "INSERT INTO moz_bookmarks_deleted (guid, dateRemoved) " + "VALUES (:guid, :date_removed)"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindUTF8StringByName("guid"_ns, aBookmark.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByName("date_removed"_ns, RoundedPRNow()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult nsNavBookmarks::InsertTombstones( + const nsTArray& aTombstones) { + if (aTombstones.IsEmpty()) { + return NS_OK; + } + + nsCOMPtr conn = mDB->MainConn(); + NS_ENSURE_STATE(conn); + + int32_t variableLimit = 0; + nsresult rv = conn->GetVariableLimit(&variableLimit); + NS_ENSURE_SUCCESS(rv, rv); + + size_t maxRowsPerChunk = variableLimit / 2; + for (uint32_t startIndex = 0; startIndex < aTombstones.Length(); + startIndex += maxRowsPerChunk) { + size_t rowsPerChunk = + std::min(maxRowsPerChunk, aTombstones.Length() - startIndex); + + // Build a query to insert all tombstones in a single statement, chunking to + // avoid the SQLite bound parameter limit. + nsAutoCString tombstonesToInsert; + tombstonesToInsert.AppendLiteral("VALUES (?, ?)"); + for (uint32_t i = 1; i < rowsPerChunk; ++i) { + tombstonesToInsert.AppendLiteral(", (?, ?)"); + } +#ifdef DEBUG + MOZ_ASSERT(tombstonesToInsert.CountChar('?') == rowsPerChunk * 2, + "Expected one binding param per column for each tombstone"); +#endif + + nsCOMPtr stmt = + mDB->GetStatement(nsLiteralCString("INSERT INTO moz_bookmarks_deleted " + "(guid, dateRemoved) ") + + tombstonesToInsert); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + uint32_t paramIndex = 0; + for (uint32_t i = 0; i < rowsPerChunk; ++i) { + const TombstoneData& tombstone = aTombstones[startIndex + i]; + rv = stmt->BindUTF8StringByIndex(paramIndex++, tombstone.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt64ByIndex(paramIndex++, tombstone.dateRemoved); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::RemoveTombstone(const nsACString& aGUID) { + nsCOMPtr stmt = + mDB->GetStatement("DELETE FROM moz_bookmarks_deleted WHERE guid = :guid"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindUTF8StringByName("guid"_ns, aGUID); + NS_ENSURE_SUCCESS(rv, rv); + + return stmt->Execute(); +} + +NS_IMETHODIMP +nsNavBookmarks::SetItemTitle(int64_t aItemId, const nsACString& aTitle, + uint16_t aSource) { + NS_ENSURE_ARG_MIN(aItemId, 1); + + BookmarkData bookmark; + nsresult rv = FetchItemInfo(aItemId, bookmark); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + bool isChangingTagFolder = bookmark.parentId == tagsRootId; + int64_t syncChangeDelta = DetermineSyncChangeDelta(aSource); + + nsAutoCString title; + TruncateTitle(aTitle, title); + + if (isChangingTagFolder) { + // If we're changing the title of a tag folder, bump the change counter + // for all tagged bookmarks. We use a separate code path to avoid a + // transaction for non-tags. + mozStorageTransaction transaction(mDB->MainConn(), false); + + // XXX Handle the error, bug 1696133. + Unused << NS_WARN_IF(NS_FAILED(transaction.Start())); + + rv = SetItemTitleInternal(bookmark, title, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + rv = AddSyncChangesForBookmarksInFolder(bookmark.id, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + } else { + rv = SetItemTitleInternal(bookmark, title, syncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (mCanNotify) { + Sequence> events; + RefPtr titleChanged = new PlacesBookmarkTitle(); + titleChanged->mId = bookmark.id; + titleChanged->mItemType = bookmark.type; + titleChanged->mUrl.Assign(NS_ConvertUTF8toUTF16(bookmark.url)); + titleChanged->mGuid = bookmark.guid; + titleChanged->mParentGuid = bookmark.parentGuid; + titleChanged->mTitle.Assign(NS_ConvertUTF8toUTF16(title)); + titleChanged->mLastModified = bookmark.lastModified / 1000; + titleChanged->mSource = aSource; + titleChanged->mIsTagging = + bookmark.parentId == tagsRootId || bookmark.grandParentId == tagsRootId; + bool success = !!events.AppendElement(titleChanged.forget(), fallible); + MOZ_RELEASE_ASSERT(success); + PlacesObservers::NotifyListeners(events); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::SetItemTitleInternal(BookmarkData& aBookmark, + const nsACString& aTitle, + int64_t aSyncChangeDelta) { + nsCOMPtr statement = mDB->GetStatement( + "UPDATE moz_bookmarks SET " + "title = :item_title, lastModified = :date, " + "syncChangeCounter = syncChangeCounter + :delta " + "WHERE id = :item_id"); + NS_ENSURE_STATE(statement); + mozStorageStatementScoper scoper(statement); + + nsresult rv; + if (aTitle.IsEmpty()) { + rv = statement->BindNullByName("item_title"_ns); + } else { + rv = statement->BindUTF8StringByName("item_title"_ns, aTitle); + } + NS_ENSURE_SUCCESS(rv, rv); + aBookmark.lastModified = RoundToMilliseconds(RoundedPRNow()); + rv = statement->BindInt64ByName("date"_ns, aBookmark.lastModified); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64ByName("item_id"_ns, aBookmark.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt64ByName("delta"_ns, aSyncChangeDelta); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavBookmarks::GetItemTitle(int64_t aItemId, nsACString& _title) { + NS_ENSURE_ARG_MIN(aItemId, 1); + + BookmarkData bookmark; + nsresult rv = FetchItemInfo(aItemId, bookmark); + NS_ENSURE_SUCCESS(rv, rv); + + _title = bookmark.title; + return NS_OK; +} + +nsresult nsNavBookmarks::ResultNodeForContainer( + const nsCString& aGUID, nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aNode) { + BookmarkData bookmark; + nsresult rv = FetchItemInfo(aGUID, bookmark); + NS_ENSURE_SUCCESS(rv, rv); + + if (bookmark.type == TYPE_FOLDER) { // TYPE_FOLDER + *aNode = + new nsNavHistoryFolderResultNode(bookmark.title, aOptions, bookmark.id); + } else { + return NS_ERROR_INVALID_ARG; + } + + (*aNode)->mDateAdded = bookmark.dateAdded; + (*aNode)->mLastModified = bookmark.lastModified; + (*aNode)->mBookmarkGuid = bookmark.guid; + (*aNode)->GetAsFolder()->mTargetFolderGuid = bookmark.guid; + + NS_ADDREF(*aNode); + return NS_OK; +} + +nsresult nsNavBookmarks::QueryFolderChildren( + int64_t aFolderId, nsNavHistoryQueryOptions* aOptions, + nsCOMArray* aChildren) { + NS_ENSURE_ARG_POINTER(aOptions); + NS_ENSURE_ARG_POINTER(aChildren); + + // Select all children of a given folder, sorted by position. + // This is a LEFT JOIN because not all bookmarks types have a place. + // We construct a result where the first columns exactly match those returned + // by mDBGetURLPageInfo, and additionally contains columns for position, + // item_child, and folder_child from moz_bookmarks. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT h.id, h.url, b.title, h.rev_host, h.visit_count, " + "h.last_visit_date, null, b.id, b.dateAdded, b.lastModified, " + "b.parent, null, h.frecency, h.hidden, h.guid, null, null, null, " + "b.guid, b.position, b.type, b.fk " + "FROM moz_bookmarks b " + "LEFT JOIN moz_places h ON b.fk = h.id " + "WHERE b.parent = :parent " + "AND (NOT :excludeItems OR " + "b.type = :folder OR " + "h.url_hash BETWEEN hash('place', 'prefix_lo') AND hash('place', " + "'prefix_hi')) " + "ORDER BY b.position ASC"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("folder"_ns, TYPE_FOLDER); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("excludeItems"_ns, aOptions->ExcludeItems()); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr row = do_QueryInterface(stmt, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t index = -1; + bool hasResult; + while (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + rv = ProcessFolderNodeRow(row, aOptions, aChildren, index); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::ProcessFolderNodeRow( + mozIStorageValueArray* aRow, nsNavHistoryQueryOptions* aOptions, + nsCOMArray* aChildren, int32_t& aCurrentIndex) { + NS_ENSURE_ARG_POINTER(aRow); + NS_ENSURE_ARG_POINTER(aOptions); + NS_ENSURE_ARG_POINTER(aChildren); + + // The results will be in order of aCurrentIndex. Even if we don't add a node + // because it was excluded, we need to count its index, so do that before + // doing anything else. + aCurrentIndex++; + + int32_t itemType; + nsresult rv = aRow->GetInt32(kGetChildrenIndex_Type, &itemType); + NS_ENSURE_SUCCESS(rv, rv); + int64_t id; + rv = aRow->GetInt64(nsNavHistory::kGetInfoIndex_ItemId, &id); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr node; + + if (itemType == TYPE_BOOKMARK) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + rv = history->RowToResult(aRow, aOptions, getter_AddRefs(node)); + NS_ENSURE_SUCCESS(rv, rv); + uint32_t nodeType; + node->GetType(&nodeType); + if (nodeType == nsINavHistoryResultNode::RESULT_TYPE_QUERY && + aOptions->ExcludeQueries()) { + return NS_OK; + } + } else if (itemType == TYPE_FOLDER) { + nsAutoCString title; + bool isNull; + rv = aRow->GetIsNull(nsNavHistory::kGetInfoIndex_Title, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = aRow->GetUTF8String(nsNavHistory::kGetInfoIndex_Title, title); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Don't use options from the parent to build the new folder node, it will + // inherit those later when it's inserted in the result. + node = new nsNavHistoryFolderResultNode(title, + new nsNavHistoryQueryOptions(), id); + + rv = aRow->GetUTF8String(kGetChildrenIndex_Guid, node->mBookmarkGuid); + NS_ENSURE_SUCCESS(rv, rv); + node->GetAsFolder()->mTargetFolderGuid = node->mBookmarkGuid; + + rv = aRow->GetInt64(nsNavHistory::kGetInfoIndex_ItemDateAdded, + reinterpret_cast(&node->mDateAdded)); + NS_ENSURE_SUCCESS(rv, rv); + rv = aRow->GetInt64(nsNavHistory::kGetInfoIndex_ItemLastModified, + reinterpret_cast(&node->mLastModified)); + NS_ENSURE_SUCCESS(rv, rv); + } else { + // This is a separator. + node = new nsNavHistorySeparatorResultNode(); + + node->mItemId = id; + rv = aRow->GetUTF8String(kGetChildrenIndex_Guid, node->mBookmarkGuid); + NS_ENSURE_SUCCESS(rv, rv); + rv = aRow->GetInt64(nsNavHistory::kGetInfoIndex_ItemDateAdded, + reinterpret_cast(&node->mDateAdded)); + NS_ENSURE_SUCCESS(rv, rv); + rv = aRow->GetInt64(nsNavHistory::kGetInfoIndex_ItemLastModified, + reinterpret_cast(&node->mLastModified)); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Store the index of the node within this container. Note that this is not + // moz_bookmarks.position. + node->mBookmarkIndex = aCurrentIndex; + + NS_ENSURE_TRUE(aChildren->AppendObject(node), NS_ERROR_OUT_OF_MEMORY); + return NS_OK; +} + +nsresult nsNavBookmarks::QueryFolderChildrenAsync( + nsNavHistoryFolderResultNode* aNode, + mozIStoragePendingStatement** _pendingStmt) { + NS_ENSURE_ARG_POINTER(aNode); + NS_ENSURE_ARG_POINTER(_pendingStmt); + + // Select all children of a given folder, sorted by position. + // This is a LEFT JOIN because not all bookmarks types have a place. + // We construct a result where the first columns exactly match those returned + // by mDBGetURLPageInfo, and additionally contains columns for position, + // item_child, and folder_child from moz_bookmarks. + nsCOMPtr stmt = mDB->GetAsyncStatement( + "SELECT h.id, h.url, b.title, h.rev_host, h.visit_count, " + "h.last_visit_date, null, b.id, b.dateAdded, b.lastModified, " + "b.parent, null, h.frecency, h.hidden, h.guid, null, null, null, " + "b.guid, b.position, b.type, b.fk " + "FROM moz_bookmarks b " + "LEFT JOIN moz_places h ON b.fk = h.id " + "WHERE b.parent = :parent " + "AND (NOT :excludeItems OR " + "b.type = :folder OR " + "h.url_hash BETWEEN hash('place', 'prefix_lo') AND hash('place', " + "'prefix_hi')) " + "ORDER BY b.position ASC"); + NS_ENSURE_STATE(stmt); + + nsresult rv = stmt->BindInt64ByName("parent"_ns, aNode->mTargetFolderItemId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("folder"_ns, TYPE_FOLDER); + NS_ENSURE_SUCCESS(rv, rv); + rv = + stmt->BindInt32ByName("excludeItems"_ns, aNode->mOptions->ExcludeItems()); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr pendingStmt; + rv = stmt->ExecuteAsync(aNode, getter_AddRefs(pendingStmt)); + NS_ENSURE_SUCCESS(rv, rv); + + NS_IF_ADDREF(*_pendingStmt = pendingStmt); + return NS_OK; +} + +nsresult nsNavBookmarks::FetchFolderInfo(int64_t aFolderId, + int32_t* _folderCount, + nsACString& _guid, + int64_t* _parentId) { + *_folderCount = 0; + *_parentId = -1; + + // This query has to always return results, so it can't be written as a join, + // though a left join of 2 subqueries would have the same cost. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT count(*), " + "(SELECT guid FROM moz_bookmarks WHERE id = :parent), " + "(SELECT parent FROM moz_bookmarks WHERE id = :parent) " + "FROM moz_bookmarks " + "WHERE parent = :parent"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("parent"_ns, aFolderId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasResult; + rv = stmt->ExecuteStep(&hasResult); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(hasResult, NS_ERROR_UNEXPECTED); + + // Ensure that the folder we are looking for exists. + // Can't rely only on parent, since the root has parent 0, that doesn't exist. + bool isNull; + rv = stmt->GetIsNull(2, &isNull); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && (!isNull || aFolderId == 0), + NS_ERROR_INVALID_ARG); + + rv = stmt->GetInt32(0, _folderCount); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = stmt->GetUTF8String(1, _guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(2, _parentId); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult nsNavBookmarks::GetBookmarksForURI( + nsIURI* aURI, nsTArray& aBookmarks) { + NS_ENSURE_ARG(aURI); + + // Double ordering covers possible lastModified ties, that could happen when + // importing, syncing or due to extensions. + // Note: not using a JOIN is cheaper in this case. + nsCOMPtr stmt = mDB->GetStatement( + "/* do not warn (bug 1175249) */ " + "SELECT b.id, b.guid, b.parent, b.lastModified, t.guid, t.parent, " + "b.syncStatus " + "FROM moz_bookmarks b " + "JOIN moz_bookmarks t on t.id = b.parent " + "WHERE b.fk = (SELECT id FROM moz_places WHERE url_hash = " + "hash(:page_url) AND url = :page_url) " + "ORDER BY b.lastModified DESC, b.id DESC "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = URIBinder::Bind(stmt, "page_url"_ns, aURI); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t tagsRootId = mDB->GetTagsFolderId(); + + bool more; + nsAutoString tags; + while (NS_SUCCEEDED((rv = stmt->ExecuteStep(&more))) && more) { + // Skip tags. + int64_t grandParentId; + nsresult rv = stmt->GetInt64(5, &grandParentId); + NS_ENSURE_SUCCESS(rv, rv); + if (grandParentId == tagsRootId) { + continue; + } + + BookmarkData bookmark; + bookmark.grandParentId = grandParentId; + rv = stmt->GetInt64(0, &bookmark.id); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetUTF8String(1, bookmark.guid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(2, &bookmark.parentId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt64(3, reinterpret_cast(&bookmark.lastModified)); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetUTF8String(4, bookmark.parentGuid); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetInt32(6, &bookmark.syncStatus); + NS_ENSURE_SUCCESS(rv, rv); + + aBookmarks.AppendElement(bookmark); + } + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsIObserver + +NS_IMETHODIMP +nsNavBookmarks::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + + if (strcmp(aTopic, TOPIC_PLACES_CONNECTION_CLOSED) == 0) { + // Don't even try to notify observers from this point on, the category + // cache would init services that could try to use our APIs. + mCanNotify = false; + } + + return NS_OK; +} diff --git a/toolkit/components/places/nsNavBookmarks.h b/toolkit/components/places/nsNavBookmarks.h new file mode 100644 index 0000000000..1f5d1e717b --- /dev/null +++ b/toolkit/components/places/nsNavBookmarks.h @@ -0,0 +1,312 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsNavBookmarks_h_ +#define nsNavBookmarks_h_ + +#include "nsINavBookmarksService.h" +#include "nsNavHistory.h" +#include "nsToolkitCompsCID.h" +#include "nsCategoryCache.h" +#include "nsTHashtable.h" +#include "mozilla/Attributes.h" +#include "prtime.h" + +class nsNavBookmarks; + +namespace mozilla { +namespace places { + +enum BookmarkStatementId { + DB_FIND_REDIRECTED_BOOKMARK = 0, + DB_GET_BOOKMARKS_FOR_URI +}; + +struct BookmarkData { + int64_t id = -1; + nsCString url; + nsCString title; + int32_t position = -1; + int64_t placeId = -1; + int64_t parentId = -1; + int64_t grandParentId = -1; + int32_t type = 0; + int32_t syncStatus = nsINavBookmarksService::SYNC_STATUS_UNKNOWN; + nsCString serviceCID; + PRTime dateAdded = 0; + PRTime lastModified = 0; + nsCString guid; + nsCString parentGuid; +}; + +struct ItemVisitData { + BookmarkData bookmark; + int64_t visitId; + uint32_t transitionType; + PRTime time; +}; + +struct TombstoneData { + nsCString guid; + PRTime dateRemoved; +}; + +typedef void (nsNavBookmarks::*ItemVisitMethod)(const ItemVisitData&); + +enum BookmarkDate { LAST_MODIFIED }; + +} // namespace places +} // namespace mozilla + +class nsNavBookmarks final : public nsINavBookmarksService, + public nsIObserver, + public nsSupportsWeakReference { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSINAVBOOKMARKSSERVICE + NS_DECL_NSIOBSERVER + + nsNavBookmarks(); + + /** + * Obtains the service's object. + */ + static already_AddRefed GetSingleton(); + + /** + * Initializes the service's object. This should only be called once. + */ + nsresult Init(); + + static nsNavBookmarks* GetBookmarksService() { + if (!gBookmarksService) { + nsCOMPtr serv = + do_GetService(NS_NAVBOOKMARKSSERVICE_CONTRACTID); + NS_ENSURE_TRUE(serv, nullptr); + NS_ASSERTION(gBookmarksService, + "Should have static instance pointer now"); + } + return gBookmarksService; + } + + typedef mozilla::places::BookmarkData BookmarkData; + typedef mozilla::places::ItemVisitData ItemVisitData; + typedef mozilla::places::BookmarkStatementId BookmarkStatementId; + + nsresult OnVisit(nsIURI* aURI, int64_t aVisitId, PRTime aTime, + int64_t aSessionId, int64_t aReferringId, + uint32_t aTransitionType, const nsACString& aGUID, + bool aHidden, uint32_t aVisitCount, uint32_t aTyped, + const nsAString& aLastKnownTitle); + + nsresult ResultNodeForContainer(const nsCString& aGUID, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aNode); + + // Find all the children of a folder, using the given query and options. + // For each child, a ResultNode is created and added to |children|. + // The results are ordered by folder position. + nsresult QueryFolderChildren(int64_t aFolderId, + nsNavHistoryQueryOptions* aOptions, + nsCOMArray* children); + + /** + * Turns aRow into a node and appends it to aChildren if it is appropriate to + * do so. + * + * @param aRow + * A Storage statement (in the case of synchronous execution) or row of + * a result set (in the case of asynchronous execution). + * @param aOptions + * The options of the parent folder node. These are the options used + * to fill the parent node. + * @param aChildren + * The children of the parent folder node. + * @param aCurrentIndex + * The index of aRow within the results. When called on the first row, + * this should be set to -1. + */ + nsresult ProcessFolderNodeRow(mozIStorageValueArray* aRow, + nsNavHistoryQueryOptions* aOptions, + nsCOMArray* aChildren, + int32_t& aCurrentIndex); + + /** + * The async version of QueryFolderChildren. + * + * @param aNode + * The folder node that will receive the children. + * @param _pendingStmt + * The Storage pending statement that will be used to control async + * execution. + */ + nsresult QueryFolderChildrenAsync(nsNavHistoryFolderResultNode* aNode, + mozIStoragePendingStatement** _pendingStmt); + + /** + * Fetches information about the specified id from the database. + * + * @param aItemId + * Id of the item to fetch information for. + * @param aBookmark + * BookmarkData to store the information. + */ + nsresult FetchItemInfo(int64_t aItemId, BookmarkData& _bookmark); + + /** + * Fetches information about the specified GUID from the database. + * + * @param aGUID + * GUID of the item to fetch information for. + * @param aBookmark + * BookmarkData to store the information. + */ + nsresult FetchItemInfo(const nsCString& aGUID, BookmarkData& _bookmark); + + /** + * Notifies that a bookmark has been visited. + * + * @param aItemId + * The visited item id. + * @param aData + * Details about the new visit. + */ + void NotifyItemVisited(const ItemVisitData& aData); + + static const int32_t kGetChildrenIndex_Guid; + static const int32_t kGetChildrenIndex_Position; + static const int32_t kGetChildrenIndex_Type; + static const int32_t kGetChildrenIndex_PlaceID; + static const int32_t kGetChildrenIndex_SyncStatus; + + static mozilla::Atomic sLastInsertedItemId; + static void StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId); + + static mozilla::Atomic sTotalSyncChanges; + static void NoteSyncChange(); + + private: + static nsNavBookmarks* gBookmarksService; + + ~nsNavBookmarks(); + + nsresult AdjustIndices(int64_t aFolder, int32_t aStartIndex, + int32_t aEndIndex, int32_t aDelta); + + nsresult AdjustSeparatorsSyncCounter(int64_t aFolderId, int32_t aStartIndex, + int64_t aSyncChangeDelta); + + /** + * Fetches properties of a folder. + * + * @param aFolderId + * Folder to count children for. + * @param _folderCount + * Number of children in the folder. + * @param _guid + * Unique id of the folder. + * @param _parentId + * Id of the parent of the folder. + * + * @throws If folder does not exist. + */ + nsresult FetchFolderInfo(int64_t aFolderId, int32_t* _folderCount, + nsACString& _guid, int64_t* _parentId); + + nsresult AddSyncChangesForBookmarksWithURL(const nsACString& aURL, + int64_t aSyncChangeDelta); + + // Bumps the change counter for all bookmarks with |aURI|. This is used to + // update tagged bookmarks when adding or changing a tag entry. + nsresult AddSyncChangesForBookmarksWithURI(nsIURI* aURI, + int64_t aSyncChangeDelta); + + // Bumps the change counter for all bookmarked URLs within |aFolderId|. This + // is used to update tagged bookmarks when changing or removing a tag folder. + nsresult AddSyncChangesForBookmarksInFolder(int64_t aFolderId, + int64_t aSyncChangeDelta); + + // Inserts a tombstone for a removed synced item. + nsresult InsertTombstone(const BookmarkData& aBookmark); + + // Inserts tombstones for removed synced items. + nsresult InsertTombstones( + const nsTArray& aTombstones); + + // Removes a stale synced bookmark tombstone. + nsresult RemoveTombstone(const nsACString& aGUID); + + nsresult SetItemTitleInternal(BookmarkData& aBookmark, + const nsACString& aTitle, + int64_t aSyncChangeDelta); + + /** + * This is an handle to the Places database. + */ + RefPtr mDB; + + nsresult SetItemDateInternal(enum mozilla::places::BookmarkDate aDateType, + int64_t aSyncChangeDelta, int64_t aItemId, + PRTime aValue); + + MOZ_CAN_RUN_SCRIPT + nsresult RemoveFolderChildren(int64_t aFolderId, uint16_t aSource); + + // Recursive method to build an array of folder's children + nsresult GetDescendantChildren(int64_t aFolderId, + const nsACString& aFolderGuid, + int64_t aGrandParentId, + nsTArray& aFolderChildrenArray); + + enum ItemType { + BOOKMARK = TYPE_BOOKMARK, + FOLDER = TYPE_FOLDER, + SEPARATOR = TYPE_SEPARATOR, + }; + + /** + * Helper to insert a bookmark in the database. + * + * @param aItemId + * The itemId to insert, pass -1 to generate a new one. + * @param aPlaceId + * The placeId to which this bookmark refers to, pass nullptr for + * items that don't refer to an URI (eg. folders, separators, ...). + * @param aItemType + * The type of the new bookmark, see TYPE_* constants. + * @param aParentId + * The itemId of the parent folder. + * @param aIndex + * The position inside the parent folder. + * @param aTitle + * The title for the new bookmark. + * Pass a void string to set a NULL title. + * @param aDateAdded + * The date for the insertion. + * @param [optional] aLastModified + * The last modified date for the insertion. + * It defaults to aDateAdded. + * + * @return The new item id that has been inserted. + * + * @note This will also update last modified date of the parent folder. + */ + nsresult InsertBookmarkInDB(int64_t aPlaceId, enum ItemType aItemType, + int64_t aParentId, int32_t aIndex, + const nsACString& aTitle, PRTime aDateAdded, + PRTime aLastModified, + const nsACString& aParentGuid, + int64_t aGrandParentId, nsIURI* aURI, + uint16_t aSource, int64_t* _itemId, + nsACString& _guid); + + nsresult GetBookmarksForURI(nsIURI* aURI, nsTArray& _bookmarks); + + // Used to enable and disable the observer notifications. + bool mCanNotify; +}; + +#endif // nsNavBookmarks_h_ diff --git a/toolkit/components/places/nsNavHistory.cpp b/toolkit/components/places/nsNavHistory.cpp new file mode 100644 index 0000000000..05da3536ce --- /dev/null +++ b/toolkit/components/places/nsNavHistory.cpp @@ -0,0 +1,3033 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include + +#include "mozilla/Components.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/intl/LocaleService.h" + +#include "nsNavHistory.h" + +#include "mozIPlacesAutoComplete.h" +#include "nsNavBookmarks.h" +#include "nsFaviconService.h" +#include "nsPlacesMacros.h" +#include "nsPlacesTriggers.h" +#include "mozilla/intl/AppDateTimeFormat.h" +#include "History.h" +#include "Helpers.h" +#include "NotifyRankingChanged.h" + +#include "mozIStorageValueArray.h" +#include "nsTArray.h" +#include "nsNetUtil.h" +#include "nsPrintfCString.h" +#include "nsPromiseFlatString.h" +#include "nsString.h" +#include "nsUnicharUtils.h" +#include "prsystem.h" +#include "prtime.h" +#include "nsEscape.h" +#include "nsIEffectiveTLDService.h" +#include "nsIClassInfoImpl.h" +#include "nsIIDNService.h" +#include "nsQueryObject.h" +#include "nsThreadUtils.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsMathUtils.h" +#include "nsReadableUtils.h" +#include "mozilla/storage.h" +#include "mozilla/Preferences.h" +#include + +using namespace mozilla; +using namespace mozilla::places; + +// The maximum number of things that we will store in the recent events list +// before calling ExpireNonrecentEvents. This number should be big enough so it +// is very difficult to get that many unconsumed events (for example, typed but +// never visited) in the RECENT_EVENT_THRESHOLD. Otherwise, we'll start +// checking each one for every page visit, which will be somewhat slower. +#define RECENT_EVENT_QUEUE_MAX_LENGTH 128 + +// preference ID strings +#define PREF_HISTORY_ENABLED "places.history.enabled" +#define PREF_MATCH_DIACRITICS "places.search.matchDiacritics" + +#define PREF_FREC_NUM_VISITS "places.frecency.numVisits" +#define PREF_FREC_NUM_VISITS_DEF 10 +#define PREF_FREC_FIRST_BUCKET_CUTOFF "places.frecency.firstBucketCutoff" +#define PREF_FREC_FIRST_BUCKET_CUTOFF_DEF 4 +#define PREF_FREC_SECOND_BUCKET_CUTOFF "places.frecency.secondBucketCutoff" +#define PREF_FREC_SECOND_BUCKET_CUTOFF_DEF 14 +#define PREF_FREC_THIRD_BUCKET_CUTOFF "places.frecency.thirdBucketCutoff" +#define PREF_FREC_THIRD_BUCKET_CUTOFF_DEF 31 +#define PREF_FREC_FOURTH_BUCKET_CUTOFF "places.frecency.fourthBucketCutoff" +#define PREF_FREC_FOURTH_BUCKET_CUTOFF_DEF 90 +#define PREF_FREC_FIRST_BUCKET_WEIGHT "places.frecency.firstBucketWeight" +#define PREF_FREC_FIRST_BUCKET_WEIGHT_DEF 100 +#define PREF_FREC_SECOND_BUCKET_WEIGHT "places.frecency.secondBucketWeight" +#define PREF_FREC_SECOND_BUCKET_WEIGHT_DEF 70 +#define PREF_FREC_THIRD_BUCKET_WEIGHT "places.frecency.thirdBucketWeight" +#define PREF_FREC_THIRD_BUCKET_WEIGHT_DEF 50 +#define PREF_FREC_FOURTH_BUCKET_WEIGHT "places.frecency.fourthBucketWeight" +#define PREF_FREC_FOURTH_BUCKET_WEIGHT_DEF 30 +#define PREF_FREC_DEFAULT_BUCKET_WEIGHT "places.frecency.defaultBucketWeight" +#define PREF_FREC_DEFAULT_BUCKET_WEIGHT_DEF 10 +#define PREF_FREC_EMBED_VISIT_BONUS "places.frecency.embedVisitBonus" +#define PREF_FREC_EMBED_VISIT_BONUS_DEF 0 +#define PREF_FREC_FRAMED_LINK_VISIT_BONUS "places.frecency.framedLinkVisitBonus" +#define PREF_FREC_FRAMED_LINK_VISIT_BONUS_DEF 0 +#define PREF_FREC_LINK_VISIT_BONUS "places.frecency.linkVisitBonus" +#define PREF_FREC_LINK_VISIT_BONUS_DEF 100 +#define PREF_FREC_TYPED_VISIT_BONUS "places.frecency.typedVisitBonus" +#define PREF_FREC_TYPED_VISIT_BONUS_DEF 2000 +#define PREF_FREC_BOOKMARK_VISIT_BONUS "places.frecency.bookmarkVisitBonus" +#define PREF_FREC_BOOKMARK_VISIT_BONUS_DEF 75 +#define PREF_FREC_DOWNLOAD_VISIT_BONUS "places.frecency.downloadVisitBonus" +#define PREF_FREC_DOWNLOAD_VISIT_BONUS_DEF 0 +#define PREF_FREC_PERM_REDIRECT_VISIT_BONUS \ + "places.frecency.permRedirectVisitBonus" +#define PREF_FREC_PERM_REDIRECT_VISIT_BONUS_DEF 0 +#define PREF_FREC_TEMP_REDIRECT_VISIT_BONUS \ + "places.frecency.tempRedirectVisitBonus" +#define PREF_FREC_TEMP_REDIRECT_VISIT_BONUS_DEF 0 +#define PREF_FREC_REDIR_SOURCE_VISIT_BONUS \ + "places.frecency.redirectSourceVisitBonus" +#define PREF_FREC_REDIR_SOURCE_VISIT_BONUS_DEF 25 +#define PREF_FREC_DEFAULT_VISIT_BONUS "places.frecency.defaultVisitBonus" +#define PREF_FREC_DEFAULT_VISIT_BONUS_DEF 0 +#define PREF_FREC_UNVISITED_BOOKMARK_BONUS \ + "places.frecency.unvisitedBookmarkBonus" +#define PREF_FREC_UNVISITED_BOOKMARK_BONUS_DEF 140 +#define PREF_FREC_UNVISITED_TYPED_BONUS "places.frecency.unvisitedTypedBonus" +#define PREF_FREC_UNVISITED_TYPED_BONUS_DEF 200 +#define PREF_FREC_RELOAD_VISIT_BONUS "places.frecency.reloadVisitBonus" +#define PREF_FREC_RELOAD_VISIT_BONUS_DEF 0 + +// In order to avoid calling PR_now() too often we use a cached "now" value +// for repeating stuff. These are milliseconds between "now" cache refreshes. +#define RENEW_CACHED_NOW_TIMEOUT ((int32_t)3 * PR_MSEC_PER_SEC) + +// These macros are used when splitting history by date. +// These are the day containers and catch-all final container. +#define HISTORY_ADDITIONAL_DATE_CONT_NUM 3 +// We use a guess of the number of months considering all of them 30 days +// long, but we split only the last 6 months. +#define HISTORY_DATE_CONT_NUM(_daysFromOldestVisit) \ + (HISTORY_ADDITIONAL_DATE_CONT_NUM + \ + std::min(6, (int32_t)ceilf((float)_daysFromOldestVisit / 30))) +// Max number of containers, used to initialize the params hash. +#define HISTORY_DATE_CONT_LENGTH 8 + +// Initial length of the recent events cache. +#define RECENT_EVENTS_INITIAL_CACHE_LENGTH 64 + +// Observed topics. +#define TOPIC_IDLE_DAILY "idle-daily" +#define TOPIC_PREF_CHANGED "nsPref:changed" +#define TOPIC_PROFILE_TEARDOWN "profile-change-teardown" +#define TOPIC_PROFILE_CHANGE "profile-before-change" +#define TOPIC_APP_LOCALES_CHANGED "intl:app-locales-changed" + +static const char* kObservedPrefs[] = {PREF_HISTORY_ENABLED, + PREF_MATCH_DIACRITICS, + PREF_FREC_NUM_VISITS, + PREF_FREC_FIRST_BUCKET_CUTOFF, + PREF_FREC_SECOND_BUCKET_CUTOFF, + PREF_FREC_THIRD_BUCKET_CUTOFF, + PREF_FREC_FOURTH_BUCKET_CUTOFF, + PREF_FREC_FIRST_BUCKET_WEIGHT, + PREF_FREC_SECOND_BUCKET_WEIGHT, + PREF_FREC_THIRD_BUCKET_WEIGHT, + PREF_FREC_FOURTH_BUCKET_WEIGHT, + PREF_FREC_DEFAULT_BUCKET_WEIGHT, + PREF_FREC_EMBED_VISIT_BONUS, + PREF_FREC_FRAMED_LINK_VISIT_BONUS, + PREF_FREC_LINK_VISIT_BONUS, + PREF_FREC_TYPED_VISIT_BONUS, + PREF_FREC_BOOKMARK_VISIT_BONUS, + PREF_FREC_DOWNLOAD_VISIT_BONUS, + PREF_FREC_PERM_REDIRECT_VISIT_BONUS, + PREF_FREC_TEMP_REDIRECT_VISIT_BONUS, + PREF_FREC_REDIR_SOURCE_VISIT_BONUS, + PREF_FREC_DEFAULT_VISIT_BONUS, + PREF_FREC_UNVISITED_BOOKMARK_BONUS, + PREF_FREC_UNVISITED_TYPED_BONUS, + nullptr}; + +NS_IMPL_ADDREF(nsNavHistory) +NS_IMPL_RELEASE(nsNavHistory) + +NS_IMPL_CLASSINFO(nsNavHistory, nullptr, nsIClassInfo::SINGLETON, + NS_NAVHISTORYSERVICE_CID) +NS_INTERFACE_MAP_BEGIN(nsNavHistory) + NS_INTERFACE_MAP_ENTRY(nsINavHistoryService) + NS_INTERFACE_MAP_ENTRY(nsIObserver) + NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference) + NS_INTERFACE_MAP_ENTRY(mozIStorageVacuumParticipant) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsINavHistoryService) + NS_IMPL_QUERY_CLASSINFO(nsNavHistory) +NS_INTERFACE_MAP_END + +// We don't care about flattening everything +NS_IMPL_CI_INTERFACE_GETTER(nsNavHistory, nsINavHistoryService) + +namespace { + +static nsCString GetSimpleBookmarksQueryParent( + const RefPtr& aQuery, + const RefPtr& aOptions); +static void ParseSearchTermsFromQuery(const RefPtr& aQuery, + nsTArray* aTerms); + +void GetTagsSqlFragment(int64_t aTagsFolder, const nsACString& aRelation, + bool aHasSearchTerms, nsACString& _sqlFragment) { + if (!aHasSearchTerms) + _sqlFragment.AssignLiteral("null"); + else { + // This subquery DOES NOT order tags for performance reasons. + _sqlFragment.Assign( + nsLiteralCString("(SELECT GROUP_CONCAT(t_t.title, ',') " + "FROM moz_bookmarks b_t " + "JOIN moz_bookmarks t_t ON t_t.id = +b_t.parent " + "WHERE b_t.fk = ") + + aRelation + + nsLiteralCString(" " + "AND t_t.parent = ") + + nsPrintfCString("%" PRId64, aTagsFolder) + + nsLiteralCString(" " + ")")); + } + + _sqlFragment.AppendLiteral(" AS tags "); +} + +} // namespace + +// Queries rows indexes to bind or get values, if adding a new one, be sure to +// update nsNavBookmarks statements and its kGetChildrenIndex_* constants +const int32_t nsNavHistory::kGetInfoIndex_PageID = 0; +const int32_t nsNavHistory::kGetInfoIndex_URL = 1; +const int32_t nsNavHistory::kGetInfoIndex_Title = 2; +const int32_t nsNavHistory::kGetInfoIndex_RevHost = 3; +const int32_t nsNavHistory::kGetInfoIndex_VisitCount = 4; +const int32_t nsNavHistory::kGetInfoIndex_VisitDate = 5; +const int32_t nsNavHistory::kGetInfoIndex_FaviconURL = 6; +const int32_t nsNavHistory::kGetInfoIndex_ItemId = 7; +const int32_t nsNavHistory::kGetInfoIndex_ItemDateAdded = 8; +const int32_t nsNavHistory::kGetInfoIndex_ItemLastModified = 9; +const int32_t nsNavHistory::kGetInfoIndex_ItemParentId = 10; +const int32_t nsNavHistory::kGetInfoIndex_ItemTags = 11; +const int32_t nsNavHistory::kGetInfoIndex_Frecency = 12; +const int32_t nsNavHistory::kGetInfoIndex_Hidden = 13; +const int32_t nsNavHistory::kGetInfoIndex_Guid = 14; +const int32_t nsNavHistory::kGetInfoIndex_VisitId = 15; +const int32_t nsNavHistory::kGetInfoIndex_FromVisitId = 16; +const int32_t nsNavHistory::kGetInfoIndex_VisitType = 17; +// These columns are followed by corresponding constants in nsNavBookmarks.cpp, +// which must be kept in sync: +// nsNavBookmarks::kGetChildrenIndex_Guid = 18; +// nsNavBookmarks::kGetChildrenIndex_Position = 19; +// nsNavBookmarks::kGetChildrenIndex_Type = 20; +// nsNavBookmarks::kGetChildrenIndex_PlaceID = 21; + +PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavHistory, gHistoryService) + +nsNavHistory::nsNavHistory() + : mCachedNow(0), + mRecentTyped(RECENT_EVENTS_INITIAL_CACHE_LENGTH), + mRecentLink(RECENT_EVENTS_INITIAL_CACHE_LENGTH), + mRecentBookmark(RECENT_EVENTS_INITIAL_CACHE_LENGTH), + mHistoryEnabled(true), + mMatchDiacritics(false), + mNumVisitsForFrecency(10), + mTagsFolder(-1), + mLastCachedStartOfDay(INT64_MAX), + mLastCachedEndOfDay(0) { + NS_ASSERTION(!gHistoryService, + "Attempting to create two instances of the service!"); + gHistoryService = this; +} + +nsNavHistory::~nsNavHistory() { + MOZ_ASSERT(NS_IsMainThread(), "Must be called on the main thread"); + + // remove the static reference to the service. Check to make sure its us + // in case somebody creates an extra instance of the service. + NS_ASSERTION(gHistoryService == this, + "Deleting a non-singleton instance of the service"); + + if (gHistoryService == this) gHistoryService = nullptr; +} + +nsresult nsNavHistory::Init() { + LoadPrefs(); + + mDB = Database::GetDatabase(); + NS_ENSURE_STATE(mDB); + + /***************************************************************************** + *** IMPORTANT NOTICE! + *** + *** Nothing after these add observer calls should return anything but NS_OK. + *** If a failure code is returned, this nsNavHistory object will be held onto + *** by the observer service and the preference service. + ****************************************************************************/ + + // Observe preferences changes. + Preferences::AddWeakObservers(this, kObservedPrefs); + + nsCOMPtr obsSvc = services::GetObserverService(); + if (obsSvc) { + (void)obsSvc->AddObserver(this, TOPIC_PLACES_CONNECTION_CLOSED, true); + (void)obsSvc->AddObserver(this, TOPIC_IDLE_DAILY, true); + (void)obsSvc->AddObserver(this, TOPIC_APP_LOCALES_CHANGED, true); + } + + // Don't add code that can fail here! Do it up above, before we add our + // observers. + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetDatabaseStatus(uint16_t* aDatabaseStatus) { + NS_ENSURE_ARG_POINTER(aDatabaseStatus); + *aDatabaseStatus = mDB->GetDatabaseStatus(); + return NS_OK; +} + +uint32_t nsNavHistory::GetRecentFlags(nsIURI* aURI) { + uint32_t result = 0; + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "Unable to get aURI's spec"); + + if (NS_SUCCEEDED(rv)) { + if (CheckIsRecentEvent(&mRecentTyped, spec)) result |= RECENT_TYPED; + if (CheckIsRecentEvent(&mRecentLink, spec)) result |= RECENT_ACTIVATED; + if (CheckIsRecentEvent(&mRecentBookmark, spec)) result |= RECENT_BOOKMARKED; + } + + return result; +} + +nsresult nsNavHistory::GetIdForPage(nsIURI* aURI, int64_t* _pageId, + nsCString& _GUID) { + *_pageId = 0; + + nsCOMPtr stmt = mDB->GetStatement( + "SELECT id, url, title, rev_host, visit_count, guid " + "FROM moz_places " + "WHERE url_hash = hash(:page_url) AND url = :page_url "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = URIBinder::Bind(stmt, "page_url"_ns, aURI); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasEntry = false; + rv = stmt->ExecuteStep(&hasEntry); + NS_ENSURE_SUCCESS(rv, rv); + + if (hasEntry) { + rv = stmt->GetInt64(0, _pageId); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->GetUTF8String(5, _GUID); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult nsNavHistory::GetOrCreateIdForPage(nsIURI* aURI, int64_t* _pageId, + nsCString& _GUID) { + nsresult rv = GetIdForPage(aURI, _pageId, _GUID); + NS_ENSURE_SUCCESS(rv, rv); + + if (*_pageId != 0) { + return NS_OK; + } + + { + // Create a new hidden, untyped and unvisited entry. + nsCOMPtr stmt = mDB->GetStatement( + "INSERT INTO moz_places (url, url_hash, rev_host, hidden, frecency, " + "guid) " + "VALUES (:page_url, hash(:page_url), :rev_host, :hidden, :frecency, " + ":guid) "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + rv = URIBinder::Bind(stmt, "page_url"_ns, aURI); + NS_ENSURE_SUCCESS(rv, rv); + // host (reversed with trailing period) + nsAutoString revHost; + rv = GetReversedHostname(aURI, revHost); + // Not all URI types have hostnames, so this is optional. + if (NS_SUCCEEDED(rv)) { + rv = stmt->BindStringByName("rev_host"_ns, revHost); + } else { + rv = stmt->BindNullByName("rev_host"_ns); + } + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("hidden"_ns, 1); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString spec; + rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindInt32ByName("frecency"_ns, IsQueryURI(spec) ? 0 : -1); + NS_ENSURE_SUCCESS(rv, rv); + rv = GenerateGUID(_GUID); + NS_ENSURE_SUCCESS(rv, rv); + rv = stmt->BindUTF8StringByName("guid"_ns, _GUID); + NS_ENSURE_SUCCESS(rv, rv); + + rv = stmt->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + *_pageId = sLastInsertedPlaceId; + } + + { + // Trigger the updates to the moz_origins tables + nsCOMPtr stmt = + mDB->GetStatement("DELETE FROM moz_updateoriginsinsert_temp"); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + } + + return NS_OK; +} + +void nsNavHistory::LoadPrefs() { + // History preferences. + mHistoryEnabled = Preferences::GetBool(PREF_HISTORY_ENABLED, true); + mMatchDiacritics = Preferences::GetBool(PREF_MATCH_DIACRITICS, false); + + // Frecency preferences. +#define FRECENCY_PREF(_prop, _pref) \ + _prop = Preferences::GetInt(_pref, _pref##_DEF) + + FRECENCY_PREF(mNumVisitsForFrecency, PREF_FREC_NUM_VISITS); + FRECENCY_PREF(mFirstBucketCutoffInDays, PREF_FREC_FIRST_BUCKET_CUTOFF); + FRECENCY_PREF(mSecondBucketCutoffInDays, PREF_FREC_SECOND_BUCKET_CUTOFF); + FRECENCY_PREF(mThirdBucketCutoffInDays, PREF_FREC_THIRD_BUCKET_CUTOFF); + FRECENCY_PREF(mFourthBucketCutoffInDays, PREF_FREC_FOURTH_BUCKET_CUTOFF); + FRECENCY_PREF(mEmbedVisitBonus, PREF_FREC_EMBED_VISIT_BONUS); + FRECENCY_PREF(mFramedLinkVisitBonus, PREF_FREC_FRAMED_LINK_VISIT_BONUS); + FRECENCY_PREF(mLinkVisitBonus, PREF_FREC_LINK_VISIT_BONUS); + FRECENCY_PREF(mTypedVisitBonus, PREF_FREC_TYPED_VISIT_BONUS); + FRECENCY_PREF(mBookmarkVisitBonus, PREF_FREC_BOOKMARK_VISIT_BONUS); + FRECENCY_PREF(mDownloadVisitBonus, PREF_FREC_DOWNLOAD_VISIT_BONUS); + FRECENCY_PREF(mPermRedirectVisitBonus, PREF_FREC_PERM_REDIRECT_VISIT_BONUS); + FRECENCY_PREF(mTempRedirectVisitBonus, PREF_FREC_TEMP_REDIRECT_VISIT_BONUS); + FRECENCY_PREF(mRedirectSourceVisitBonus, PREF_FREC_REDIR_SOURCE_VISIT_BONUS); + FRECENCY_PREF(mDefaultVisitBonus, PREF_FREC_DEFAULT_VISIT_BONUS); + FRECENCY_PREF(mUnvisitedBookmarkBonus, PREF_FREC_UNVISITED_BOOKMARK_BONUS); + FRECENCY_PREF(mUnvisitedTypedBonus, PREF_FREC_UNVISITED_TYPED_BONUS); + FRECENCY_PREF(mReloadVisitBonus, PREF_FREC_RELOAD_VISIT_BONUS); + FRECENCY_PREF(mFirstBucketWeight, PREF_FREC_FIRST_BUCKET_WEIGHT); + FRECENCY_PREF(mSecondBucketWeight, PREF_FREC_SECOND_BUCKET_WEIGHT); + FRECENCY_PREF(mThirdBucketWeight, PREF_FREC_THIRD_BUCKET_WEIGHT); + FRECENCY_PREF(mFourthBucketWeight, PREF_FREC_FOURTH_BUCKET_WEIGHT); + FRECENCY_PREF(mDefaultWeight, PREF_FREC_DEFAULT_BUCKET_WEIGHT); + +#undef FRECENCY_PREF +} + +void nsNavHistory::UpdateDaysOfHistory(PRTime visitTime) { + if (sDaysOfHistory == 0) { + sDaysOfHistory = 1; + } + + if (visitTime > mLastCachedEndOfDay || visitTime < mLastCachedStartOfDay) { + InvalidateDaysOfHistory(); + } +} + +NS_IMETHODIMP +nsNavHistory::RecalculateOriginFrecencyStats(nsIObserver* aCallback) { + RefPtr self(this); + nsMainThreadPtrHandle callback( + !aCallback ? nullptr + : new nsMainThreadPtrHolder( + "nsNavHistory::RecalculateOriginFrecencyStats callback", + aCallback)); + + nsCOMPtr target(do_GetInterface(mDB->MainConn())); + NS_ENSURE_STATE(target); + nsresult rv = target->Dispatch(NS_NewRunnableFunction( + "nsNavHistory::RecalculateOriginFrecencyStats", [self, callback] { + Unused << self->mDB->RecalculateOriginFrecencyStatsInternal(); + Unused << NS_DispatchToMainThread(NS_NewRunnableFunction( + "nsNavHistory::RecalculateOriginFrecencyStats callback", + [callback] { + if (callback) { + Unused << callback->Observe(nullptr, "", nullptr); + } + })); + })); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +Atomic nsNavHistory::sLastInsertedPlaceId(0); +Atomic nsNavHistory::sLastInsertedVisitId(0); +Atomic nsNavHistory::sIsFrecencyDecaying(false); +Atomic nsNavHistory::sShouldStartFrecencyRecalculation(false); + +void // static +nsNavHistory::StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId) { + if (aTable.EqualsLiteral("moz_places")) { + nsNavHistory::sLastInsertedPlaceId = aLastInsertedId; + } else if (aTable.EqualsLiteral("moz_historyvisits")) { + nsNavHistory::sLastInsertedVisitId = aLastInsertedId; + } else { + MOZ_ASSERT(false, "Trying to store the insert id for an unknown table?"); + } +} + +Atomic nsNavHistory::sDaysOfHistory(-1); + +void // static +nsNavHistory::InvalidateDaysOfHistory() { + sDaysOfHistory = -1; +} + +int32_t nsNavHistory::GetDaysOfHistory() { + MOZ_ASSERT(NS_IsMainThread(), "This can only be called on the main thread"); + + if (sDaysOfHistory != -1) return sDaysOfHistory; + + // SQLite doesn't have a CEIL() function, so we must do that later. + // We should also take into account timers resolution, that may be as bad as + // 16ms on Windows, so in some cases the difference may be 0, if the + // check is done near the visit. Thus remember to check for NULL separately. + nsCOMPtr stmt = mDB->GetStatement( + "SELECT CAST(( " + "strftime('%s','now','localtime','utc') - " + "(SELECT MIN(visit_date)/1000000 FROM moz_historyvisits) " + ") AS DOUBLE) " + "/86400, " + "strftime('%s','now','localtime','+1 day','start of day','utc') * " + "1000000"); + NS_ENSURE_TRUE(stmt, 0); + mozStorageStatementScoper scoper(stmt); + + bool hasResult; + if (NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) { + // If we get NULL, then there are no visits, otherwise there must always be + // at least 1 day of history. + bool hasNoVisits; + (void)stmt->GetIsNull(0, &hasNoVisits); + sDaysOfHistory = + hasNoVisits + ? 0 + : std::max(1, static_cast(ceil(stmt->AsDouble(0)))); + mLastCachedStartOfDay = + NormalizeTime(nsINavHistoryQuery::TIME_RELATIVE_TODAY, 0); + mLastCachedEndOfDay = stmt->AsInt64(1) - 1; // Start of tomorrow - 1. + } + + return sDaysOfHistory; +} + +PRTime nsNavHistory::GetNow() { + if (!mCachedNow) { + mCachedNow = PR_Now(); + if (!mExpireNowTimer) mExpireNowTimer = NS_NewTimer(); + if (mExpireNowTimer) + mExpireNowTimer->InitWithNamedFuncCallback( + expireNowTimerCallback, this, RENEW_CACHED_NOW_TIMEOUT, + nsITimer::TYPE_ONE_SHOT, "nsNavHistory::GetNow"); + } + return mCachedNow; +} + +void nsNavHistory::expireNowTimerCallback(nsITimer* aTimer, void* aClosure) { + nsNavHistory* history = static_cast(aClosure); + if (history) { + history->mCachedNow = 0; + history->mExpireNowTimer = nullptr; + } +} + +/** + * Code borrowed from mozilla/xpfe/components/history/src/nsGlobalHistory.cpp + * Pass in a pre-normalized now and a date, and we'll find the difference since + * midnight on each of the days. + */ +static PRTime NormalizeTimeRelativeToday(PRTime aTime) { + // round to midnight this morning + PRExplodedTime explodedTime; + PR_ExplodeTime(aTime, PR_LocalTimeParameters, &explodedTime); + + // set to midnight (0:00) + explodedTime.tm_min = explodedTime.tm_hour = explodedTime.tm_sec = + explodedTime.tm_usec = 0; + + return PR_ImplodeTime(&explodedTime); +} + +// nsNavHistory::NormalizeTime +// +// Converts a nsINavHistoryQuery reference+offset time into a PRTime +// relative to the epoch. +// +// It is important that this function NOT use the current time optimization. +// It is called to update queries, and we really need to know what right +// now is because those incoming values will also have current times that +// we will have to compare against. + +PRTime // static +nsNavHistory::NormalizeTime(uint32_t aRelative, PRTime aOffset) { + PRTime ref; + switch (aRelative) { + case nsINavHistoryQuery::TIME_RELATIVE_EPOCH: + return aOffset; + case nsINavHistoryQuery::TIME_RELATIVE_TODAY: + ref = NormalizeTimeRelativeToday(PR_Now()); + break; + case nsINavHistoryQuery::TIME_RELATIVE_NOW: + ref = PR_Now(); + break; + default: + MOZ_ASSERT_UNREACHABLE("Invalid relative time"); + return 0; + } + return ref + aOffset; +} + +// nsNavHistory::DomainNameFromURI +// +// This does the www.mozilla.org -> mozilla.org and +// foo.theregister.co.uk -> theregister.co.uk conversion +void nsNavHistory::DomainNameFromURI(nsIURI* aURI, nsACString& aDomainName) { + // lazily get the effective tld service + if (!mTLDService) + mTLDService = do_GetService(NS_EFFECTIVETLDSERVICE_CONTRACTID); + + if (mTLDService) { + // get the base domain for a given hostname. + // e.g. for "images.bbc.co.uk", this would be "bbc.co.uk". + nsresult rv = mTLDService->GetBaseDomain(aURI, 0, aDomainName); + if (NS_SUCCEEDED(rv)) return; + } + + // just return the original hostname + // (it's also possible the host is an IP address) + aURI->GetAsciiHost(aDomainName); +} + +bool nsNavHistory::hasHistoryEntries() { return GetDaysOfHistory() > 0; } + +// Call this method before visiting a URL in order to help determine the +// transition type of the visit. +// +// @see MarkPageAsTyped + +NS_IMETHODIMP +nsNavHistory::MarkPageAsFollowedBookmark(nsIURI* aURI) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aURI); + + // don't add when history is disabled + if (IsHistoryDisabled()) return NS_OK; + + nsAutoCString uriString; + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); + + mRecentBookmark.InsertOrUpdate(uriString, GetNow()); + + if (mRecentBookmark.Count() > RECENT_EVENT_QUEUE_MAX_LENGTH) + ExpireNonrecentEvents(&mRecentBookmark); + + return NS_OK; +} + +// nsNavHistory::CanAddURI +// +// Filter out unwanted URIs such as "chrome:", "mailbox:", etc. +// +// The model is if we don't know differently then add which basically means +// we are suppose to try all the things we know not to allow in and then if +// we don't bail go on and allow it in. + +NS_IMETHODIMP +nsNavHistory::CanAddURI(nsIURI* aURI, bool* canAdd) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aURI); + NS_ENSURE_ARG_POINTER(canAdd); + + // If history is disabled, don't add any entry. + *canAdd = !IsHistoryDisabled() && BaseHistory::CanStore(aURI); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetNewQuery(nsINavHistoryQuery** _retval) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG_POINTER(_retval); + + RefPtr query = new nsNavHistoryQuery(); + query.forget(_retval); + return NS_OK; +} + +// nsNavHistory::GetNewQueryOptions + +NS_IMETHODIMP +nsNavHistory::GetNewQueryOptions(nsINavHistoryQueryOptions** _retval) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG_POINTER(_retval); + + RefPtr queryOptions = + new nsNavHistoryQueryOptions(); + queryOptions.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::ExecuteQuery(nsINavHistoryQuery* aQuery, + nsINavHistoryQueryOptions* aOptions, + nsINavHistoryResult** _retval) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aQuery); + NS_ENSURE_ARG(aOptions); + NS_ENSURE_ARG_POINTER(_retval); + + // Clone the input query and options, because the caller might change the + // objects, but we always want to reflect the original parameters. + nsCOMPtr queryClone; + aQuery->Clone(getter_AddRefs(queryClone)); + NS_ENSURE_STATE(queryClone); + RefPtr query = do_QueryObject(queryClone); + NS_ENSURE_STATE(query); + nsCOMPtr optionsClone; + aOptions->Clone(getter_AddRefs(optionsClone)); + NS_ENSURE_STATE(optionsClone); + RefPtr options = do_QueryObject(optionsClone); + NS_ENSURE_STATE(options); + + // Create the root node. + RefPtr rootNode; + + nsCString folderGuid = GetSimpleBookmarksQueryParent(query, options); + if (!folderGuid.IsEmpty()) { + // In the simple case where we're just querying children of a single + // bookmark folder, we can more efficiently generate results. + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); + RefPtr tempRootNode; + nsresult rv = bookmarks->ResultNodeForContainer( + folderGuid, options, getter_AddRefs(tempRootNode)); + if (NS_SUCCEEDED(rv)) { + rootNode = tempRootNode->GetAsContainer(); + } else { + NS_WARNING("Generating a generic empty node for a broken query!"); + // This is a perf hack to generate an empty query that skips filtering. + options->SetExcludeItems(true); + } + } + + if (!rootNode) { + // Either this is not a folder shortcut, or is a broken one. In both cases + // just generate a query node. + nsAutoCString queryUri; + nsresult rv = QueryToQueryString(query, options, queryUri); + NS_ENSURE_SUCCESS(rv, rv); + rootNode = + new nsNavHistoryQueryResultNode(""_ns, 0, queryUri, query, options); + } + + // Create the result that will hold nodes. Inject batching status into it. + RefPtr result = + new nsNavHistoryResult(rootNode, query, options); + result.forget(_retval); + return NS_OK; +} + +// determine from our nsNavHistoryQuery array and nsNavHistoryQueryOptions +// if this is the place query from the history menu. +// from browser-menubar.inc, our history menu query is: +// place:sort=4&maxResults=10 +// note, any maxResult > 0 will still be considered a history menu query +// or if this is the place query from the old "Most Visited" item in some +// profiles: folder: place:sort=8&maxResults=10 note, any maxResult > 0 will +// still be considered a Most Visited menu query +static bool IsOptimizableHistoryQuery( + const RefPtr& aQuery, + const RefPtr& aOptions, uint16_t aSortMode) { + if (aOptions->QueryType() != nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY) + return false; + + if (aOptions->ResultType() != nsINavHistoryQueryOptions::RESULTS_AS_URI) + return false; + + if (aOptions->SortingMode() != aSortMode) return false; + + if (aOptions->MaxResults() <= 0) return false; + + if (aOptions->ExcludeItems()) return false; + + if (aOptions->IncludeHidden()) return false; + + if (aQuery->MinVisits() != -1 || aQuery->MaxVisits() != -1) return false; + + if (aQuery->BeginTime() || aQuery->BeginTimeReference()) return false; + + if (aQuery->EndTime() || aQuery->EndTimeReference()) return false; + + if (!aQuery->SearchTerms().IsEmpty()) return false; + + if (aQuery->OnlyBookmarked()) return false; + + if (aQuery->DomainIsHost() || !aQuery->Domain().IsEmpty()) return false; + + if (aQuery->AnnotationIsNot() || !aQuery->Annotation().IsEmpty()) + return false; + + if (aQuery->Parents().Length() > 0) return false; + + if (aQuery->Tags().Length() > 0) return false; + + if (aQuery->Transitions().Length() > 0) return false; + + return true; +} + +static bool NeedToFilterResultSet(const RefPtr& aQuery, + nsNavHistoryQueryOptions* aOptions) { + return aOptions->ExcludeQueries(); +} + +// ** Helper class for ConstructQueryString **/ + +class PlacesSQLQueryBuilder { + public: + PlacesSQLQueryBuilder(const nsCString& aConditions, + const RefPtr& aQuery, + const RefPtr& aOptions, + bool aUseLimit, nsNavHistory::StringHash& aAddParams, + bool aHasSearchTerms); + + nsresult GetQueryString(nsCString& aQueryString); + + private: + nsresult Select(); + + nsresult SelectAsURI(); + nsresult SelectAsVisit(); + nsresult SelectAsDay(); + nsresult SelectAsSite(); + nsresult SelectAsTag(); + nsresult SelectAsRoots(); + nsresult SelectAsLeftPane(); + + nsresult Where(); + nsresult GroupBy(); + nsresult OrderBy(); + nsresult Limit(); + + void OrderByColumnIndexAsc(int32_t aIndex); + void OrderByColumnIndexDesc(int32_t aIndex); + // Use these if you want a case insensitive sorting. + void OrderByTextColumnIndexAsc(int32_t aIndex); + void OrderByTextColumnIndexDesc(int32_t aIndex); + + const nsCString& mConditions; + bool mUseLimit; + bool mHasSearchTerms; + + uint16_t mResultType; + uint16_t mQueryType; + bool mIncludeHidden; + uint16_t mSortingMode; + uint32_t mMaxResults; + + nsCString mQueryString; + nsCString mGroupBy; + bool mHasDateColumns; + bool mSkipOrderBy; + + nsNavHistory::StringHash& mAddParams; +}; + +PlacesSQLQueryBuilder::PlacesSQLQueryBuilder( + const nsCString& aConditions, const RefPtr& aQuery, + const RefPtr& aOptions, bool aUseLimit, + nsNavHistory::StringHash& aAddParams, bool aHasSearchTerms) + : mConditions(aConditions), + mUseLimit(aUseLimit), + mHasSearchTerms(aHasSearchTerms), + mResultType(aOptions->ResultType()), + mQueryType(aOptions->QueryType()), + mIncludeHidden(aOptions->IncludeHidden()), + mSortingMode(aOptions->SortingMode()), + mMaxResults(aOptions->MaxResults()), + mSkipOrderBy(false), + mAddParams(aAddParams) { + mHasDateColumns = + (mQueryType == nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS); + // Force the default sorting mode for tag queries. + if (mSortingMode == nsINavHistoryQueryOptions::SORT_BY_NONE && + aQuery->Tags().Length() > 0) { + mSortingMode = nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING; + } +} + +nsresult PlacesSQLQueryBuilder::GetQueryString(nsCString& aQueryString) { + nsresult rv = Select(); + NS_ENSURE_SUCCESS(rv, rv); + rv = Where(); + NS_ENSURE_SUCCESS(rv, rv); + rv = GroupBy(); + NS_ENSURE_SUCCESS(rv, rv); + rv = OrderBy(); + NS_ENSURE_SUCCESS(rv, rv); + rv = Limit(); + NS_ENSURE_SUCCESS(rv, rv); + + aQueryString = mQueryString; + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::Select() { + nsresult rv; + + switch (mResultType) { + case nsINavHistoryQueryOptions::RESULTS_AS_URI: + rv = SelectAsURI(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_VISIT: + rv = SelectAsVisit(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY: + case nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY: + rv = SelectAsDay(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_SITE_QUERY: + rv = SelectAsSite(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT: + rv = SelectAsTag(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY: + rv = SelectAsRoots(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + case nsINavHistoryQueryOptions::RESULTS_AS_LEFT_PANE_QUERY: + rv = SelectAsLeftPane(); + NS_ENSURE_SUCCESS(rv, rv); + break; + + default: + MOZ_ASSERT_UNREACHABLE("Invalid result type"); + } + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsURI() { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + nsAutoCString tagsSqlFragment; + + switch (mQueryType) { + case nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY: + GetTagsSqlFragment(history->GetTagsFolder(), "h.id"_ns, mHasSearchTerms, + tagsSqlFragment); + + mQueryString = nsLiteralCString( + "SELECT h.id, h.url, h.title AS page_title, " + "h.rev_host, h.visit_count, " + "h.last_visit_date, null, null, null, null, null, ") + + tagsSqlFragment + + nsLiteralCString( + ", h.frecency, h.hidden, h.guid, " + "null, null, null " + "FROM moz_places h " + // WHERE 1 is a no-op since additonal conditions will + // start with AND. + "WHERE 1 " + "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " + "{ADDITIONAL_CONDITIONS} "); + break; + + case nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS: + + GetTagsSqlFragment(history->GetTagsFolder(), "b.fk"_ns, mHasSearchTerms, + tagsSqlFragment); + mQueryString = + nsLiteralCString( + "SELECT b.fk, h.url, b.title AS page_title, " + "h.rev_host, h.visit_count, h.last_visit_date, null, b.id, " + "b.dateAdded, b.lastModified, b.parent, ") + + tagsSqlFragment + + nsLiteralCString( + ", h.frecency, h.hidden, h.guid," + "null, null, null, b.guid, b.position, b.type, b.fk " + "FROM moz_bookmarks b " + "JOIN moz_places h ON b.fk = h.id " + "WHERE NOT EXISTS " + "(SELECT id FROM moz_bookmarks " + "WHERE id = b.parent AND parent = ") + + nsPrintfCString("%" PRId64, history->GetTagsFolder()) + + nsLiteralCString( + ") " + "AND NOT h.url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi') " + "{ADDITIONAL_CONDITIONS}"); + break; + + default: + return NS_ERROR_NOT_IMPLEMENTED; + } + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsVisit() { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + nsAutoCString tagsSqlFragment; + GetTagsSqlFragment(history->GetTagsFolder(), "h.id"_ns, mHasSearchTerms, + tagsSqlFragment); + mQueryString = + nsLiteralCString( + "SELECT h.id, h.url, h.title AS page_title, h.rev_host, " + "h.visit_count, " + "v.visit_date, null, null, null, null, null, ") + + tagsSqlFragment + + nsLiteralCString( + ", h.frecency, h.hidden, h.guid, " + "v.id, v.from_visit, v.visit_type " + "FROM moz_places h " + "JOIN moz_historyvisits v ON h.id = v.place_id " + // WHERE 1 is a no-op since additonal conditions will start with AND. + "WHERE 1 " + "{QUERY_OPTIONS_VISITS} {QUERY_OPTIONS_PLACES} " + "{ADDITIONAL_CONDITIONS} "); + + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsDay() { + mSkipOrderBy = true; + + // Sort child queries based on sorting mode if it's provided, otherwise + // fallback to default sort by title ascending. + uint16_t sortingMode = nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING; + if (mSortingMode != nsINavHistoryQueryOptions::SORT_BY_NONE && + mResultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY) + sortingMode = mSortingMode; + + uint16_t resultType = + mResultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY + ? (uint16_t)nsINavHistoryQueryOptions::RESULTS_AS_URI + : (uint16_t)nsINavHistoryQueryOptions::RESULTS_AS_SITE_QUERY; + + // beginTime will become the node's time property, we don't use endTime + // because it could overlap, and we use time to sort containers and find + // insert position in a result. + mQueryString = nsPrintfCString( + "SELECT null, " + "'place:type=%d&sort=%d&beginTime='||beginTime||'&endTime='||endTime, " + "dayTitle, null, null, beginTime, null, null, null, null, null, null, " + "null, null, null " + "FROM (", // TOUTER BEGIN + resultType, sortingMode); + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + + int32_t daysOfHistory = history->GetDaysOfHistory(); + for (int32_t i = 0; i <= HISTORY_DATE_CONT_NUM(daysOfHistory); i++) { + nsAutoCString dateName; + // Timeframes are calculated as BeginTime <= container < EndTime. + // Notice times can't be relative to now, since to recognize a query we + // must ensure it won't change based on the time it is built. + // So, to select till now, we really select till start of tomorrow, that is + // a fixed timestamp. + // These are used as limits for the inside containers. + nsAutoCString sqlFragmentContainerBeginTime, sqlFragmentContainerEndTime; + // These are used to query if the container should be visible. + nsAutoCString sqlFragmentSearchBeginTime, sqlFragmentSearchEndTime; + switch (i) { + case 0: + // Today + history->GetStringFromName("finduri-AgeInDays-is-0", dateName); + // From start of today + sqlFragmentContainerBeginTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','utc')*1000000)"); + // To now (tomorrow) + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','+1 " + "day','utc')*1000000)"); + // Search for the same timeframe. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = sqlFragmentContainerEndTime; + break; + case 1: + // Yesterday + history->GetStringFromName("finduri-AgeInDays-is-1", dateName); + // From start of yesterday + sqlFragmentContainerBeginTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','-1 " + "day','utc')*1000000)"); + // To start of today + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','utc')*1000000)"); + // Search for the same timeframe. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = sqlFragmentContainerEndTime; + break; + case 2: + // Last 7 days + history->GetAgeInDaysString(7, "finduri-AgeInDays-last-is", dateName); + // From start of 7 days ago + sqlFragmentContainerBeginTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','-7 " + "days','utc')*1000000)"); + // To now (tomorrow) + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','+1 " + "day','utc')*1000000)"); + // This is an overlapped container, but we show it only if there are + // visits older than yesterday. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','-1 " + "day','utc')*1000000)"); + break; + case 3: + // This month + history->GetStringFromName("finduri-AgeInMonths-is-0", dateName); + // From start of this month + sqlFragmentContainerBeginTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of " + "month','utc')*1000000)"); + // To now (tomorrow) + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','+1 " + "day','utc')*1000000)"); + // This is an overlapped container, but we show it only if there are + // visits older than 7 days ago. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of day','-7 " + "days','utc')*1000000)"); + break; + default: + if (i == HISTORY_ADDITIONAL_DATE_CONT_NUM + 6) { + // Older than 6 months + history->GetAgeInDaysString(6, "finduri-AgeInMonths-isgreater", + dateName); + // From start of epoch + sqlFragmentContainerBeginTime = + "(datetime(0, 'unixepoch')*1000000)"_ns; + // To start of 6 months ago ( 5 months + this month). + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of month','-5 " + "months','utc')*1000000)"); + // Search for the same timeframe. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = sqlFragmentContainerEndTime; + break; + } + int32_t MonthIndex = i - HISTORY_ADDITIONAL_DATE_CONT_NUM; + // Previous months' titles are month's name if inside this year, + // month's name and year for previous years. + PRExplodedTime tm; + PR_ExplodeTime(PR_Now(), PR_LocalTimeParameters, &tm); + uint16_t currentYear = tm.tm_year; + // Set day before month, setting month without day could cause issues. + // For example setting month to February when today is 30, since + // February has not 30 days, will return March instead. + // Also, we use day 2 instead of day 1, so that the GMT month is always + // the same as the local month. (Bug 603002) + tm.tm_mday = 2; + tm.tm_month -= MonthIndex; + // Notice we use GMTParameters because we just want to get the first + // day of each month. Using LocalTimeParameters would instead force us + // to apply a DST correction that we don't really need here. + PR_NormalizeTime(&tm, PR_GMTParameters); + // If the container is for a past year, add the year to its title, + // otherwise just show the month name. + if (tm.tm_year < currentYear) { + nsNavHistory::GetMonthYear(tm, dateName); + } else { + nsNavHistory::GetMonthName(tm, dateName); + } + + // From start of MonthIndex + 1 months ago + sqlFragmentContainerBeginTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of month','-"); + sqlFragmentContainerBeginTime.AppendInt(MonthIndex); + sqlFragmentContainerBeginTime.AppendLiteral(" months','utc')*1000000)"); + // To start of MonthIndex months ago + sqlFragmentContainerEndTime = nsLiteralCString( + "(strftime('%s','now','localtime','start of month','-"); + sqlFragmentContainerEndTime.AppendInt(MonthIndex - 1); + sqlFragmentContainerEndTime.AppendLiteral(" months','utc')*1000000)"); + // Search for the same timeframe. + sqlFragmentSearchBeginTime = sqlFragmentContainerBeginTime; + sqlFragmentSearchEndTime = sqlFragmentContainerEndTime; + break; + } + + nsPrintfCString dateParam("dayTitle%d", i); + mAddParams.InsertOrUpdate(dateParam, dateName); + + nsPrintfCString dayRange( + "SELECT :%s AS dayTitle, " + "%s AS beginTime, " + "%s AS endTime " + "WHERE EXISTS ( " + "SELECT id FROM moz_historyvisits " + "WHERE visit_date >= %s " + "AND visit_date < %s " + "AND visit_type NOT IN (0,%d,%d) " + "{QUERY_OPTIONS_VISITS} " + "LIMIT 1 " + ") ", + dateParam.get(), sqlFragmentContainerBeginTime.get(), + sqlFragmentContainerEndTime.get(), sqlFragmentSearchBeginTime.get(), + sqlFragmentSearchEndTime.get(), nsINavHistoryService::TRANSITION_EMBED, + nsINavHistoryService::TRANSITION_FRAMED_LINK); + + mQueryString.Append(dayRange); + + if (i < HISTORY_DATE_CONT_NUM(daysOfHistory)) + mQueryString.AppendLiteral(" UNION ALL "); + } + + mQueryString.AppendLiteral(") "); // TOUTER END + + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsSite() { + nsAutoCString localFiles; + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + + history->GetStringFromName("localhost", localFiles); + mAddParams.InsertOrUpdate("localhost"_ns, localFiles); + + // If there are additional conditions the query has to join on visits too. + nsAutoCString visitsJoin; + nsAutoCString additionalConditions; + nsAutoCString timeConstraints; + if (!mConditions.IsEmpty()) { + visitsJoin.AssignLiteral("JOIN moz_historyvisits v ON v.place_id = h.id "); + additionalConditions.AssignLiteral( + "{QUERY_OPTIONS_VISITS} " + "{QUERY_OPTIONS_PLACES} " + "{ADDITIONAL_CONDITIONS} "); + timeConstraints.AssignLiteral( + "||'&beginTime='||:begin_time||" + "'&endTime='||:end_time"); + } + + mQueryString = nsPrintfCString( + "SELECT null, 'place:type=%d&sort=%d&domain=&domainIsHost=true'%s, " + ":localhost, :localhost, null, null, null, null, null, null, null, " + "null, null, null " + "WHERE EXISTS ( " + "SELECT h.id FROM moz_places h " + "%s " + "WHERE h.hidden = 0 " + "AND h.visit_count > 0 " + "AND h.url_hash BETWEEN hash('file', 'prefix_lo') AND " + "hash('file', 'prefix_hi') " + "%s " + "LIMIT 1 " + ") " + "UNION ALL " + "SELECT null, " + "'place:type=%d&sort=%d&domain='||host||'&domainIsHost=true'%s, " + "host, host, null, null, null, null, null, null, null, " + "null, null, null " + "FROM ( " + "SELECT get_unreversed_host(h.rev_host) AS host " + "FROM moz_places h " + "%s " + "WHERE h.hidden = 0 " + "AND h.rev_host <> '.' " + "AND h.visit_count > 0 " + "%s " + "GROUP BY h.rev_host " + "ORDER BY host ASC " + ") ", + nsINavHistoryQueryOptions::RESULTS_AS_URI, mSortingMode, + timeConstraints.get(), visitsJoin.get(), additionalConditions.get(), + nsINavHistoryQueryOptions::RESULTS_AS_URI, mSortingMode, + timeConstraints.get(), visitsJoin.get(), additionalConditions.get()); + + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsTag() { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + + // This allows sorting by date fields what is not possible with + // other history queries. + mHasDateColumns = true; + + // TODO (Bug 1449939): This is likely wrong, since the tag name should + // probably be urlencoded, and we have no util for that in SQL, yet. + // We could encode the tag when the user sets it though. + mQueryString = nsPrintfCString( + "SELECT null, 'place:tag=' || title, " + "title, null, null, null, null, null, dateAdded, " + "lastModified, null, null, null, null, null, null " + "FROM moz_bookmarks " + "WHERE parent = %" PRId64, + history->GetTagsFolder()); + + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsRoots() { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + + nsAutoCString toolbarTitle; + nsAutoCString menuTitle; + nsAutoCString unfiledTitle; + + history->GetStringFromName("BookmarksToolbarFolderTitle", toolbarTitle); + mAddParams.InsertOrUpdate("BookmarksToolbarFolderTitle"_ns, toolbarTitle); + history->GetStringFromName("BookmarksMenuFolderTitle", menuTitle); + mAddParams.InsertOrUpdate("BookmarksMenuFolderTitle"_ns, menuTitle); + history->GetStringFromName("OtherBookmarksFolderTitle", unfiledTitle); + mAddParams.InsertOrUpdate("OtherBookmarksFolderTitle"_ns, unfiledTitle); + + nsAutoCString mobileString; + + if (Preferences::GetBool(MOBILE_BOOKMARKS_PREF, false)) { + nsAutoCString mobileTitle; + history->GetStringFromName("MobileBookmarksFolderTitle", mobileTitle); + mAddParams.InsertOrUpdate("MobileBookmarksFolderTitle"_ns, mobileTitle); + + mobileString = nsLiteralCString( + "," + "(null, 'place:parent=" MOBILE_ROOT_GUID + "', :MobileBookmarksFolderTitle, null, null, null, " + "null, null, 0, 0, null, null, null, null, " + "'" MOBILE_BOOKMARKS_VIRTUAL_GUID "', null) "); + } + + mQueryString = + nsLiteralCString( + "SELECT * FROM (" + "VALUES(null, 'place:parent=" TOOLBAR_ROOT_GUID + "', :BookmarksToolbarFolderTitle, null, null, null, " + "null, null, 0, 0, null, null, null, null, 'toolbar____v', null), " + "(null, 'place:parent=" MENU_ROOT_GUID + "', :BookmarksMenuFolderTitle, null, null, null, " + "null, null, 0, 0, null, null, null, null, 'menu_______v', null), " + "(null, 'place:parent=" UNFILED_ROOT_GUID + "', :OtherBookmarksFolderTitle, null, null, null, " + "null, null, 0, 0, null, null, null, null, 'unfiled____v', null) ") + + mobileString + ")"_ns; + + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::SelectAsLeftPane() { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + + nsAutoCString historyTitle; + nsAutoCString downloadsTitle; + nsAutoCString tagsTitle; + nsAutoCString allBookmarksTitle; + + history->GetStringFromName("OrganizerQueryHistory", historyTitle); + mAddParams.InsertOrUpdate("OrganizerQueryHistory"_ns, historyTitle); + history->GetStringFromName("OrganizerQueryDownloads", downloadsTitle); + mAddParams.InsertOrUpdate("OrganizerQueryDownloads"_ns, downloadsTitle); + history->GetStringFromName("TagsFolderTitle", tagsTitle); + mAddParams.InsertOrUpdate("TagsFolderTitle"_ns, tagsTitle); + history->GetStringFromName("OrganizerQueryAllBookmarks", allBookmarksTitle); + mAddParams.InsertOrUpdate("OrganizerQueryAllBookmarks"_ns, allBookmarksTitle); + + mQueryString = nsPrintfCString( + "SELECT * FROM (" + "VALUES" + "(null, 'place:type=%d&sort=%d', :OrganizerQueryHistory, null, null, " + "null, " + "null, null, 0, 0, null, null, null, null, 'history____v', null), " + "(null, 'place:transition=%d&sort=%d', :OrganizerQueryDownloads, null, " + "null, null, " + "null, null, 0, 0, null, null, null, null, 'downloads__v', null), " + "(null, 'place:type=%d&sort=%d', :TagsFolderTitle, null, null, null, " + "null, null, 0, 0, null, null, null, null, 'tags_______v', null), " + "(null, 'place:type=%d', :OrganizerQueryAllBookmarks, null, null, null, " + "null, null, 0, 0, null, null, null, null, 'allbms_____v', null) " + ")", + nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY, + nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING, + nsINavHistoryService::TRANSITION_DOWNLOAD, + nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING, + nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT, + nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING, + nsINavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY); + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::Where() { + // Set query options + nsAutoCString additionalVisitsConditions; + nsAutoCString additionalPlacesConditions; + + if (!mIncludeHidden) { + additionalPlacesConditions += "AND hidden = 0 "_ns; + } + + if (mQueryType == nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY) { + // last_visit_date is updated for any kind of visit, so it's a good + // indicator whether the page has visits. + additionalPlacesConditions += "AND last_visit_date NOTNULL "_ns; + } + + if (mResultType == nsINavHistoryQueryOptions::RESULTS_AS_URI && + !additionalVisitsConditions.IsEmpty()) { + // URI results don't join on visits. + nsAutoCString tmp = additionalVisitsConditions; + additionalVisitsConditions = + "AND EXISTS (SELECT 1 FROM moz_historyvisits WHERE place_id = h.id "; + additionalVisitsConditions.Append(tmp); + additionalVisitsConditions.AppendLiteral("LIMIT 1)"); + } + + mQueryString.ReplaceSubstring("{QUERY_OPTIONS_VISITS}", + additionalVisitsConditions.get()); + mQueryString.ReplaceSubstring("{QUERY_OPTIONS_PLACES}", + additionalPlacesConditions.get()); + + // If we used WHERE already, we inject the conditions + // in place of {ADDITIONAL_CONDITIONS} + if (mQueryString.Find("{ADDITIONAL_CONDITIONS}") != kNotFound) { + nsAutoCString innerCondition; + // If we have condition AND it + if (!mConditions.IsEmpty()) { + innerCondition = " AND ("; + innerCondition += mConditions; + innerCondition += ")"; + } + mQueryString.ReplaceSubstring("{ADDITIONAL_CONDITIONS}", + innerCondition.get()); + + } else if (!mConditions.IsEmpty()) { + mQueryString += "WHERE "; + mQueryString += mConditions; + } + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::GroupBy() { + mQueryString += mGroupBy; + return NS_OK; +} + +nsresult PlacesSQLQueryBuilder::OrderBy() { + if (mSkipOrderBy) return NS_OK; + + // Sort clause: we will sort later, but if it comes out of the DB sorted, + // our later sort will be basically free. The DB can sort these for free + // most of the time anyway, because it has indices over these items. + switch (mSortingMode) { + case nsINavHistoryQueryOptions::SORT_BY_NONE: + // Ensure sorting does not change based on tables status. + if (mResultType == nsINavHistoryQueryOptions::RESULTS_AS_URI) { + if (mQueryType == nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS) + mQueryString += " ORDER BY b.id ASC "_ns; + else if (mQueryType == nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY) + mQueryString += " ORDER BY h.id ASC "_ns; + } + break; + case nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING: + case nsINavHistoryQueryOptions::SORT_BY_TITLE_DESCENDING: + // If the user wants few results, we limit them by date, necessitating + // a sort by date here (see the IDL definition for maxResults). + // Otherwise we will do actual sorting by title, but since we could need + // to special sort for some locale we will repeat a second sorting at the + // end in nsNavHistoryResult, that should be faster since the list will be + // almost ordered. + if (mMaxResults > 0) + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_VisitDate); + else if (mSortingMode == + nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING) + OrderByTextColumnIndexAsc(nsNavHistory::kGetInfoIndex_Title); + else + OrderByTextColumnIndexDesc(nsNavHistory::kGetInfoIndex_Title); + break; + case nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING: + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_VisitDate); + break; + case nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING: + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_VisitDate); + break; + case nsINavHistoryQueryOptions::SORT_BY_URI_ASCENDING: + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_URL); + break; + case nsINavHistoryQueryOptions::SORT_BY_URI_DESCENDING: + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_URL); + break; + case nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING: + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_VisitCount); + break; + case nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING: + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_VisitCount); + break; + case nsINavHistoryQueryOptions::SORT_BY_DATEADDED_ASCENDING: + if (mHasDateColumns) + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_ItemDateAdded); + break; + case nsINavHistoryQueryOptions::SORT_BY_DATEADDED_DESCENDING: + if (mHasDateColumns) + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_ItemDateAdded); + break; + case nsINavHistoryQueryOptions::SORT_BY_LASTMODIFIED_ASCENDING: + if (mHasDateColumns) + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_ItemLastModified); + break; + case nsINavHistoryQueryOptions::SORT_BY_LASTMODIFIED_DESCENDING: + if (mHasDateColumns) + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_ItemLastModified); + break; + case nsINavHistoryQueryOptions::SORT_BY_TAGS_ASCENDING: + case nsINavHistoryQueryOptions::SORT_BY_TAGS_DESCENDING: + break; // Sort later in nsNavHistoryQueryResultNode::FillChildren() + case nsINavHistoryQueryOptions::SORT_BY_FRECENCY_ASCENDING: + OrderByColumnIndexAsc(nsNavHistory::kGetInfoIndex_Frecency); + break; + case nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING: + OrderByColumnIndexDesc(nsNavHistory::kGetInfoIndex_Frecency); + break; + default: + MOZ_ASSERT_UNREACHABLE("Invalid sorting mode"); + } + return NS_OK; +} + +void PlacesSQLQueryBuilder::OrderByColumnIndexAsc(int32_t aIndex) { + mQueryString += nsPrintfCString(" ORDER BY %d ASC", aIndex + 1); +} + +void PlacesSQLQueryBuilder::OrderByColumnIndexDesc(int32_t aIndex) { + mQueryString += nsPrintfCString(" ORDER BY %d DESC", aIndex + 1); +} + +void PlacesSQLQueryBuilder::OrderByTextColumnIndexAsc(int32_t aIndex) { + mQueryString += + nsPrintfCString(" ORDER BY %d COLLATE NOCASE ASC", aIndex + 1); +} + +void PlacesSQLQueryBuilder::OrderByTextColumnIndexDesc(int32_t aIndex) { + mQueryString += + nsPrintfCString(" ORDER BY %d COLLATE NOCASE DESC", aIndex + 1); +} + +nsresult PlacesSQLQueryBuilder::Limit() { + if (mUseLimit && mMaxResults > 0) { + mQueryString += " LIMIT "_ns; + mQueryString.AppendInt(mMaxResults); + mQueryString.Append(' '); + } + return NS_OK; +} + +nsresult nsNavHistory::ConstructQueryString( + const RefPtr& aQuery, + const RefPtr& aOptions, nsCString& queryString, + bool& aParamsPresent, nsNavHistory::StringHash& aAddParams) { + // For information about visit_type see nsINavHistoryService.idl. + // visitType == 0 is undefined (see bug #375777 for details). + // Some sites, especially Javascript-heavy ones, load things in frames to + // display them, resulting in a lot of these entries. This is the reason + // why such visits are filtered out. + nsresult rv; + aParamsPresent = false; + + int32_t sortingMode = aOptions->SortingMode(); + NS_ASSERTION( + sortingMode >= nsINavHistoryQueryOptions::SORT_BY_NONE && + sortingMode <= nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING, + "Invalid sortingMode found while building query!"); + + bool hasSearchTerms = !aQuery->SearchTerms().IsEmpty(); + + nsAutoCString tagsSqlFragment; + GetTagsSqlFragment(GetTagsFolder(), "h.id"_ns, hasSearchTerms, + tagsSqlFragment); + + if (IsOptimizableHistoryQuery( + aQuery, aOptions, + nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING) || + IsOptimizableHistoryQuery( + aQuery, aOptions, + nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING)) { + // Generate an optimized query for the history menu and the old most visited + // bookmark that was inserted into profiles. + queryString = + nsLiteralCString( + "SELECT h.id, h.url, h.title AS page_title, h.rev_host, " + "h.visit_count, h.last_visit_date, " + "null, null, null, null, null, ") + + tagsSqlFragment + + nsLiteralCString( + ", h.frecency, h.hidden, h.guid, " + "null, null, null " + "FROM moz_places h " + "WHERE h.hidden = 0 " + "AND EXISTS (SELECT id FROM moz_historyvisits WHERE place_id = " + "h.id " + "AND visit_type NOT IN ") + + nsPrintfCString("(0,%d,%d) ", nsINavHistoryService::TRANSITION_EMBED, + nsINavHistoryService::TRANSITION_FRAMED_LINK) + + nsLiteralCString( + "LIMIT 1) " + "{QUERY_OPTIONS} "); + + queryString.AppendLiteral("ORDER BY "); + if (sortingMode == nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING) + queryString.AppendLiteral("last_visit_date DESC "); + else + queryString.AppendLiteral("visit_count DESC "); + + queryString.AppendLiteral("LIMIT "); + queryString.AppendInt(aOptions->MaxResults()); + + nsAutoCString additionalQueryOptions; + + queryString.ReplaceSubstring("{QUERY_OPTIONS}", + additionalQueryOptions.get()); + return NS_OK; + } + + // If the query is a tag query, the type is bookmarks. + if (!aQuery->Tags().IsEmpty()) { + aOptions->SetQueryType(nsNavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS); + } + + nsAutoCString conditions; + nsCString queryClause; + rv = QueryToSelectClause(aQuery, aOptions, &queryClause); + NS_ENSURE_SUCCESS(rv, rv); + if (!queryClause.IsEmpty()) { + // TODO: This should be set on a case basis, not blindly. + aParamsPresent = true; + conditions += queryClause; + } + + // Determine whether we can push maxResults constraints into the query + // as LIMIT, or if we need to do result count clamping later + // using FilterResultSet() + bool useLimitClause = !NeedToFilterResultSet(aQuery, aOptions); + + PlacesSQLQueryBuilder queryStringBuilder( + conditions, aQuery, aOptions, useLimitClause, aAddParams, hasSearchTerms); + rv = queryStringBuilder.GetQueryString(queryString); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +// nsNavHistory::GetQueryResults +// +// Call this to get the results from a complex query. This is used by +// nsNavHistoryQueryResultNode to populate its children. For simple bookmark +// queries, use nsNavBookmarks::QueryFolderChildren. +// +// THIS DOES NOT DO SORTING. You will need to sort the container yourself +// when you get the results. This is because sorting depends on tree +// statistics that will be built from the perspective of the tree. See +// nsNavHistoryQueryResultNode::FillChildren +// +// FIXME: This only does keyword searching for the first query, and does +// it ANDed with the all the rest of the queries. + +nsresult nsNavHistory::GetQueryResults( + nsNavHistoryQueryResultNode* aResultNode, + const RefPtr& aQuery, + const RefPtr& aOptions, + nsCOMArray* aResults) { + NS_ENSURE_ARG_POINTER(aQuery); + NS_ENSURE_ARG_POINTER(aOptions); + NS_ASSERTION(aResults->Count() == 0, "Initial result array must be empty"); + + nsCString queryString; + bool paramsPresent = false; + nsNavHistory::StringHash addParams(HISTORY_DATE_CONT_LENGTH); + nsresult rv = ConstructQueryString(aQuery, aOptions, queryString, + paramsPresent, addParams); + NS_ENSURE_SUCCESS(rv, rv); + + // create statement + nsCOMPtr statement = mDB->GetStatement(queryString); +#ifdef DEBUG + if (!statement) { + nsCOMPtr conn = mDB->MainConn(); + if (conn) { + nsAutoCString lastErrorString; + (void)conn->GetLastErrorString(lastErrorString); + int32_t lastError = 0; + (void)conn->GetLastError(&lastError); + printf( + "Places failed to create a statement from this query:\n%s\nStorage " + "error (%d): %s\n", + queryString.get(), lastError, lastErrorString.get()); + } + } +#endif + NS_ENSURE_STATE(statement); + mozStorageStatementScoper scoper(statement); + + if (paramsPresent) { + rv = BindQueryClauseParameters(statement, aQuery, aOptions); + NS_ENSURE_SUCCESS(rv, rv); + } + + for (const auto& entry : addParams) { + nsresult rv = + statement->BindUTF8StringByName(entry.GetKey(), entry.GetData()); + if (NS_FAILED(rv)) { + break; + } + } + + // Optimize the case where there is no need for any post-query filtering. + if (NeedToFilterResultSet(aQuery, aOptions)) { + // Generate the top-level results. + nsCOMArray toplevel; + rv = ResultsAsList(statement, aOptions, &toplevel); + NS_ENSURE_SUCCESS(rv, rv); + + FilterResultSet(aResultNode, toplevel, aResults, aQuery, aOptions); + } else { + rv = ResultsAsList(statement, aOptions, aResults); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetHistoryDisabled(bool* _retval) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG_POINTER(_retval); + + *_retval = IsHistoryDisabled(); + return NS_OK; +} + +// Call this method before visiting a URL in order to help determine the +// transition type of the visit. +// +// @see MarkPageAsFollowedBookmark + +NS_IMETHODIMP +nsNavHistory::MarkPageAsTyped(nsIURI* aURI) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aURI); + + // don't add when history is disabled + if (IsHistoryDisabled()) return NS_OK; + + nsAutoCString uriString; + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); + + mRecentTyped.InsertOrUpdate(uriString, GetNow()); + + if (mRecentTyped.Count() > RECENT_EVENT_QUEUE_MAX_LENGTH) + ExpireNonrecentEvents(&mRecentTyped); + + return NS_OK; +} + +// Call this method before visiting a URL in order to help determine the +// transition type of the visit. +// +// @see MarkPageAsTyped + +NS_IMETHODIMP +nsNavHistory::MarkPageAsFollowedLink(nsIURI* aURI) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aURI); + + // don't add when history is disabled + if (IsHistoryDisabled()) return NS_OK; + + nsAutoCString uriString; + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); + + mRecentLink.InsertOrUpdate(uriString, GetNow()); + + if (mRecentLink.Count() > RECENT_EVENT_QUEUE_MAX_LENGTH) + ExpireNonrecentEvents(&mRecentLink); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetIsFrecencyDecaying(bool* _out) { + NS_ENSURE_ARG_POINTER(_out); + *_out = nsNavHistory::sIsFrecencyDecaying; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::SetIsFrecencyDecaying(bool aVal) { + nsNavHistory::sIsFrecencyDecaying = aVal; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetShouldStartFrecencyRecalculation(bool* _out) { + NS_ENSURE_ARG_POINTER(_out); + *_out = nsNavHistory::sShouldStartFrecencyRecalculation; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::SetShouldStartFrecencyRecalculation(bool aVal) { + nsNavHistory::sShouldStartFrecencyRecalculation = aVal; + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// mozIStorageVacuumParticipant + +NS_IMETHODIMP +nsNavHistory::GetDatabaseConnection( + mozIStorageAsyncConnection** _DBConnection) { + NS_ENSURE_ARG_POINTER(_DBConnection); + nsCOMPtr connection = mDB->MainConn(); + connection.forget(_DBConnection); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetUseIncrementalVacuum(bool* _useIncremental) { + *_useIncremental = false; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetExpectedDatabasePageSize(int32_t* _expectedPageSize) { + NS_ENSURE_STATE(mDB); + NS_ENSURE_STATE(mDB->MainConn()); + return mDB->MainConn()->GetDefaultPageSize(_expectedPageSize); +} + +NS_IMETHODIMP +nsNavHistory::OnBeginVacuum(bool* _vacuumGranted) { + // TODO: Check if we have to deny the vacuum in some heavy-load case. + // We could maybe want to do that during batches? + *_vacuumGranted = true; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::OnEndVacuum(bool aSucceeded) { + NS_WARNING_ASSERTION(aSucceeded, "Places.sqlite vacuum failed."); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetDBConnection(mozIStorageConnection** _DBConnection) { + NS_ENSURE_ARG_POINTER(_DBConnection); + nsCOMPtr connection = mDB->MainConn(); + connection.forget(_DBConnection); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetShutdownClient(nsIAsyncShutdownClient** _shutdownClient) { + NS_ENSURE_ARG_POINTER(_shutdownClient); + nsCOMPtr client = mDB->GetClientsShutdown(); + if (!client) { + return NS_ERROR_UNEXPECTED; + } + client.forget(_shutdownClient); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::GetConnectionShutdownClient( + nsIAsyncShutdownClient** _shutdownClient) { + NS_ENSURE_ARG_POINTER(_shutdownClient); + nsCOMPtr client = mDB->GetConnectionShutdown(); + if (!client) { + return NS_ERROR_UNEXPECTED; + } + client.forget(_shutdownClient); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::AsyncExecuteLegacyQuery(nsINavHistoryQuery* aQuery, + nsINavHistoryQueryOptions* aOptions, + mozIStorageStatementCallback* aCallback, + mozIStoragePendingStatement** _stmt) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + NS_ENSURE_ARG(aQuery); + NS_ENSURE_ARG(aOptions); + NS_ENSURE_ARG(aCallback); + NS_ENSURE_ARG_POINTER(_stmt); + + RefPtr query = do_QueryObject(aQuery); + NS_ENSURE_STATE(query); + RefPtr options = do_QueryObject(aOptions); + NS_ENSURE_ARG(options); + + nsCString queryString; + bool paramsPresent = false; + nsNavHistory::StringHash addParams(HISTORY_DATE_CONT_LENGTH); + nsresult rv = ConstructQueryString(query, options, queryString, paramsPresent, + addParams); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr statement = + mDB->GetAsyncStatement(queryString); + NS_ENSURE_STATE(statement); + +#ifdef DEBUG + if (NS_FAILED(rv)) { + nsCOMPtr conn = mDB->MainConn(); + if (conn) { + nsAutoCString lastErrorString; + (void)mDB->MainConn()->GetLastErrorString(lastErrorString); + int32_t lastError = 0; + (void)mDB->MainConn()->GetLastError(&lastError); + printf( + "Places failed to create a statement from this query:\n%s\nStorage " + "error (%d): %s\n", + queryString.get(), lastError, lastErrorString.get()); + } + } +#endif + NS_ENSURE_SUCCESS(rv, rv); + + if (paramsPresent) { + rv = BindQueryClauseParameters(statement, query, options); + NS_ENSURE_SUCCESS(rv, rv); + } + + for (const auto& entry : addParams) { + nsresult rv = + statement->BindUTF8StringByName(entry.GetKey(), entry.GetData()); + if (NS_FAILED(rv)) { + break; + } + } + + rv = statement->ExecuteAsync(aCallback, _stmt); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +//////////////////////////////////////////////////////////////////////////////// +//// nsIObserver + +NS_IMETHODIMP +nsNavHistory::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread"); + if (strcmp(aTopic, TOPIC_PROFILE_TEARDOWN) == 0 || + strcmp(aTopic, TOPIC_PROFILE_CHANGE) == 0 || + strcmp(aTopic, TOPIC_SIMULATE_PLACES_SHUTDOWN) == 0) { + // These notifications are used by tests to simulate a Places shutdown. + // They should just be forwarded to the Database handle. + mDB->Observe(aSubject, aTopic, aData); + } + + else if (strcmp(aTopic, TOPIC_PREF_CHANGED) == 0) { + LoadPrefs(); + } + + else if (strcmp(aTopic, TOPIC_APP_LOCALES_CHANGED) == 0) { + mBundle = nullptr; + } + + return NS_OK; +} + +// Query stuff ***************************************************************** + +// Helper class for QueryToSelectClause +// +// This class helps to build part of the WHERE clause. + +class ConditionBuilder { + public: + ConditionBuilder& Condition(const char* aStr) { + if (!mClause.IsEmpty()) mClause.AppendLiteral(" AND "); + Str(aStr); + return *this; + } + + ConditionBuilder& Str(const char* aStr) { + mClause.Append(' '); + mClause.Append(aStr); + mClause.Append(' '); + return *this; + } + + ConditionBuilder& Param(const char* aParam) { + mClause.Append(' '); + mClause.Append(aParam); + mClause.Append(' '); + return *this; + } + + void GetClauseString(nsCString& aResult) { aResult = mClause; } + + private: + nsCString mClause; +}; + +// nsNavHistory::QueryToSelectClause +// +// THE BEHAVIOR SHOULD BE IN SYNC WITH BindQueryClauseParameters +// +// I don't check return values from the query object getters because there's +// no way for those to fail. + +nsresult nsNavHistory::QueryToSelectClause( + const RefPtr& aQuery, + const RefPtr& aOptions, nsCString* aClause) { + bool hasIt; + // We don't use the value from options here - we post filter if that + // is set. + bool excludeQueries = false; + + ConditionBuilder clause; + + if ((NS_SUCCEEDED(aQuery->GetHasBeginTime(&hasIt)) && hasIt) || + (NS_SUCCEEDED(aQuery->GetHasEndTime(&hasIt)) && hasIt)) { + clause.Condition( + "EXISTS (SELECT 1 FROM moz_historyvisits " + "WHERE place_id = h.id"); + // begin time + if (NS_SUCCEEDED(aQuery->GetHasBeginTime(&hasIt)) && hasIt) + clause.Condition("visit_date >=").Param(":begin_time"); + // end time + if (NS_SUCCEEDED(aQuery->GetHasEndTime(&hasIt)) && hasIt) + clause.Condition("visit_date <=").Param(":end_time"); + clause.Str(" LIMIT 1)"); + } + + // search terms + int32_t searchBehavior = mozIPlacesAutoComplete::BEHAVIOR_HISTORY | + mozIPlacesAutoComplete::BEHAVIOR_BOOKMARK; + if (!aQuery->SearchTerms().IsEmpty()) { + // Re-use the autocomplete_match function. Setting the behavior to match + // history or typed history or bookmarks or open pages will match almost + // everything. + clause.Condition("AUTOCOMPLETE_MATCH(") + .Param(":search_string") + .Str(", h.url, page_title, tags, ") + .Str(nsPrintfCString("1, 1, 1, 1, %d, %d", + mozIPlacesAutoComplete::MATCH_ANYWHERE_UNMODIFIED, + searchBehavior) + .get()) + .Str(", NULL)"); + // Serching by terms implicitly exclude queries. + excludeQueries = true; + } + + // min and max visit count + if (aQuery->MinVisits() >= 0) + clause.Condition("h.visit_count >=").Param(":min_visits"); + + if (aQuery->MaxVisits() >= 0) + clause.Condition("h.visit_count <=").Param(":max_visits"); + + // only bookmarked, has no affect on bookmarks-only queries + if (aOptions->QueryType() != + nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS && + aQuery->OnlyBookmarked()) + clause.Condition("EXISTS (SELECT b.fk FROM moz_bookmarks b WHERE b.type = ") + .Str(nsPrintfCString("%d", nsNavBookmarks::TYPE_BOOKMARK).get()) + .Str("AND b.fk = h.id)"); + + // domain + if (!aQuery->Domain().IsVoid()) { + bool domainIsHost = false; + aQuery->GetDomainIsHost(&domainIsHost); + if (domainIsHost) + clause.Condition("h.rev_host =").Param(":domain_lower"); + else + // see domain setting in BindQueryClauseParameters for why we do this + clause.Condition("h.rev_host >=") + .Param(":domain_lower") + .Condition("h.rev_host <") + .Param(":domain_upper"); + } + + // URI + if (aQuery->Uri()) { + clause.Condition("h.url_hash = hash(") + .Param(":uri") + .Str(")") + .Condition("h.url =") + .Param(":uri"); + } + + // annotation + if (!aQuery->Annotation().IsEmpty()) { + clause.Condition(""); + if (aQuery->AnnotationIsNot()) clause.Str("NOT"); + clause + .Str( + "EXISTS " + "(SELECT h.id " + "FROM moz_annos anno " + "JOIN moz_anno_attributes annoname " + "ON anno.anno_attribute_id = annoname.id " + "WHERE anno.place_id = h.id " + "AND annoname.name = ") + .Param(":anno") + .Str(")"); + // annotation-based queries don't get the common conditions, so you get + // all URLs with that annotation + } + + // tags + const nsTArray& tags = aQuery->Tags(); + if (tags.Length() > 0) { + clause.Condition("h.id"); + if (aQuery->TagsAreNot()) clause.Str("NOT"); + clause + .Str( + "IN " + "(SELECT bms.fk " + "FROM moz_bookmarks bms " + "JOIN moz_bookmarks tags ON bms.parent = tags.id " + "WHERE tags.parent =") + .Param(":tags_folder") + .Str("AND lower(tags.title) IN ("); + for (uint32_t i = 0; i < tags.Length(); ++i) { + nsPrintfCString param(":tag%d_", i); + clause.Param(param.get()); + if (i < tags.Length() - 1) clause.Str(","); + } + clause.Str(")"); + if (!aQuery->TagsAreNot()) { + clause.Str("GROUP BY bms.fk HAVING count(*) >=").Param(":tag_count"); + } + clause.Str(")"); + } + + // transitions + const nsTArray& transitions = aQuery->Transitions(); + for (uint32_t i = 0; i < transitions.Length(); ++i) { + nsPrintfCString param(":transition%d_", i); + clause + .Condition( + "h.id IN (SELECT place_id FROM moz_historyvisits " + "WHERE visit_type = ") + .Param(param.get()) + .Str(")"); + } + + // parents + const nsTArray& parents = aQuery->Parents(); + if (parents.Length() > 0) { + aOptions->SetQueryType(nsNavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS); + clause.Condition( + "b.parent IN( " + "WITH RECURSIVE parents(id) AS ( " + "SELECT id FROM moz_bookmarks WHERE GUID IN ("); + + for (uint32_t i = 0; i < parents.Length(); ++i) { + nsPrintfCString param(":parentguid%d_", i); + clause.Param(param.get()); + if (i < parents.Length() - 1) { + clause.Str(","); + } + } + clause.Str( + ") " + "UNION ALL " + "SELECT b2.id " + "FROM moz_bookmarks b2 " + "JOIN parents p ON b2.parent = p.id " + "WHERE b2.type = 2 " + ") " + "SELECT id FROM parents " + ")"); + } + + if (excludeQueries) { + // Serching by terms implicitly exclude queries and folder shortcuts. + clause.Condition( + "NOT h.url_hash BETWEEN hash('place', 'prefix_lo') AND " + "hash('place', 'prefix_hi')"); + } + + clause.GetClauseString(*aClause); + return NS_OK; +} + +// nsNavHistory::BindQueryClauseParameters +// +// THE BEHAVIOR SHOULD BE IN SYNC WITH QueryToSelectClause + +nsresult nsNavHistory::BindQueryClauseParameters( + mozIStorageBaseStatement* statement, + const RefPtr& aQuery, + const RefPtr& aOptions) { + nsresult rv; + + bool hasIt; + // begin time + if (NS_SUCCEEDED(aQuery->GetHasBeginTime(&hasIt)) && hasIt) { + PRTime time = + NormalizeTime(aQuery->BeginTimeReference(), aQuery->BeginTime()); + rv = statement->BindInt64ByName("begin_time"_ns, time); + NS_ENSURE_SUCCESS(rv, rv); + } + + // end time + if (NS_SUCCEEDED(aQuery->GetHasEndTime(&hasIt)) && hasIt) { + PRTime time = NormalizeTime(aQuery->EndTimeReference(), aQuery->EndTime()); + rv = statement->BindInt64ByName("end_time"_ns, time); + NS_ENSURE_SUCCESS(rv, rv); + } + + // search terms + if (!aQuery->SearchTerms().IsEmpty()) { + rv = statement->BindStringByName("search_string"_ns, aQuery->SearchTerms()); + NS_ENSURE_SUCCESS(rv, rv); + } + + // min and max visit count + int32_t visits = aQuery->MinVisits(); + if (visits >= 0) { + rv = statement->BindInt32ByName("min_visits"_ns, visits); + NS_ENSURE_SUCCESS(rv, rv); + } + + visits = aQuery->MaxVisits(); + if (visits >= 0) { + rv = statement->BindInt32ByName("max_visits"_ns, visits); + NS_ENSURE_SUCCESS(rv, rv); + } + + // domain (see GetReversedHostname for more info on reversed host names) + if (!aQuery->Domain().IsVoid()) { + nsString revDomain; + GetReversedHostname(NS_ConvertUTF8toUTF16(aQuery->Domain()), revDomain); + + if (aQuery->DomainIsHost()) { + rv = statement->BindStringByName("domain_lower"_ns, revDomain); + NS_ENSURE_SUCCESS(rv, rv); + } else { + // for "mozilla.org" do query >= "gro.allizom." AND < "gro.allizom/" + // which will get everything starting with "gro.allizom." while using the + // index (using SUBSTRING() causes indexes to be discarded). + NS_ASSERTION(revDomain[revDomain.Length() - 1] == '.', + "Invalid rev. host"); + rv = statement->BindStringByName("domain_lower"_ns, revDomain); + NS_ENSURE_SUCCESS(rv, rv); + revDomain.Truncate(revDomain.Length() - 1); + revDomain.Append(char16_t('/')); + rv = statement->BindStringByName("domain_upper"_ns, revDomain); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + // URI + if (aQuery->Uri()) { + rv = URIBinder::Bind(statement, "uri"_ns, aQuery->Uri()); + NS_ENSURE_SUCCESS(rv, rv); + } + + // annotation + if (!aQuery->Annotation().IsEmpty()) { + rv = statement->BindUTF8StringByName("anno"_ns, aQuery->Annotation()); + NS_ENSURE_SUCCESS(rv, rv); + } + + // tags + const nsTArray& tags = aQuery->Tags(); + if (tags.Length() > 0) { + for (uint32_t i = 0; i < tags.Length(); ++i) { + nsPrintfCString paramName("tag%d_", i); + nsString utf16Tag = tags[i]; + ToLowerCase(utf16Tag); + NS_ConvertUTF16toUTF8 tag(utf16Tag); + rv = statement->BindUTF8StringByName(paramName, tag); + NS_ENSURE_SUCCESS(rv, rv); + } + int64_t tagsFolder = GetTagsFolder(); + rv = statement->BindInt64ByName("tags_folder"_ns, tagsFolder); + NS_ENSURE_SUCCESS(rv, rv); + if (!aQuery->TagsAreNot()) { + rv = statement->BindInt32ByName("tag_count"_ns, tags.Length()); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + // transitions + const nsTArray& transitions = aQuery->Transitions(); + for (uint32_t i = 0; i < transitions.Length(); ++i) { + nsPrintfCString paramName("transition%d_", i); + rv = statement->BindInt64ByName(paramName, transitions[i]); + NS_ENSURE_SUCCESS(rv, rv); + } + + // parents + const nsTArray& parents = aQuery->Parents(); + for (uint32_t i = 0; i < parents.Length(); ++i) { + nsPrintfCString paramName("parentguid%d_", i); + rv = statement->BindUTF8StringByName(paramName, parents[i]); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +// nsNavHistory::ResultsAsList +// + +nsresult nsNavHistory::ResultsAsList( + mozIStorageStatement* statement, nsNavHistoryQueryOptions* aOptions, + nsCOMArray* aResults) { + nsresult rv; + nsCOMPtr row = do_QueryInterface(statement, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore = false; + while (NS_SUCCEEDED(statement->ExecuteStep(&hasMore)) && hasMore) { + RefPtr result; + rv = RowToResult(row, aOptions, getter_AddRefs(result)); + NS_ENSURE_SUCCESS(rv, rv); + aResults->AppendElement(result.forget()); + } + return NS_OK; +} + +int64_t nsNavHistory::GetTagsFolder() { + // cache our tags folder + // note, we can't do this in nsNavHistory::Init(), + // as getting the bookmarks service would initialize it. + if (mTagsFolder == -1) { + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, -1); + + nsresult rv = bookmarks->GetTagsFolder(&mTagsFolder); + NS_ENSURE_SUCCESS(rv, -1); + } + return mTagsFolder; +} + +// nsNavHistory::FilterResultSet +// +// This does some post-query-execution filtering: +// - searching on title, url and tags +// - limit count +// +// Note: changes to filtering in FilterResultSet() +// may require changes to NeedToFilterResultSet() + +// static +nsresult nsNavHistory::FilterResultSet( + nsNavHistoryQueryResultNode* aQueryNode, + const nsCOMArray& aSet, + nsCOMArray* aFiltered, + const RefPtr& aQuery, + nsNavHistoryQueryOptions* aOptions) { + // parse the search terms + nsTArray terms; + ParseSearchTermsFromQuery(aQuery, &terms); + + bool excludeQueries = aOptions->ExcludeQueries(); + for (int32_t nodeIndex = 0; nodeIndex < aSet.Count(); nodeIndex++) { + if (excludeQueries && aSet[nodeIndex]->IsQuery()) { + continue; + } + + if (aSet[nodeIndex]->mItemId != -1 && aQueryNode && + aQueryNode->mItemId == aSet[nodeIndex]->mItemId) { + continue; + } + + // If there are search terms, we are already getting only uri nodes, + // thus we don't need to filter node types. Though, we must check for + // matching terms. + if (terms.Length()) { + // Filter based on search terms. + // Convert title and url for the current node to UTF16 strings. + NS_ConvertUTF8toUTF16 nodeTitle(aSet[nodeIndex]->mTitle); + // Unescape the URL for search terms matching. + nsAutoCString cNodeURL(aSet[nodeIndex]->mURI); + NS_ConvertUTF8toUTF16 nodeURL(NS_UnescapeURL(cNodeURL)); + + // Determine if every search term matches anywhere in the title, url or + // tag. + bool matchAllTerms = true; + for (int32_t termIndex = terms.Length() - 1; + termIndex >= 0 && matchAllTerms; termIndex--) { + nsString& term = terms.ElementAt(termIndex); + // True if any of them match; false makes us quit the loop + matchAllTerms = + CaseInsensitiveFindInReadable(term, nodeTitle) || + CaseInsensitiveFindInReadable(term, nodeURL) || + CaseInsensitiveFindInReadable(term, aSet[nodeIndex]->mTags); + } + // Skip the node if we don't match all terms in the title, url or tag + if (!matchAllTerms) { + continue; + } + } + + aFiltered->AppendObject(aSet[nodeIndex]); + + // Stop once we have reached max results. + if (aOptions->MaxResults() > 0 && + (uint32_t)aFiltered->Count() >= aOptions->MaxResults()) + break; + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::MakeGuid(nsACString& aGuid) { + if (NS_FAILED(GenerateGUID(aGuid))) { + MOZ_ASSERT(false, "Shouldn't fail to create a guid!"); + aGuid.SetIsVoid(true); + } + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::HashURL(const nsACString& aSpec, const nsACString& aMode, + uint64_t* _hash) { + return places::HashURL(aSpec, aMode, _hash); +} + +// nsNavHistory::CheckIsRecentEvent +// +// Sees if this URL happened "recently." +// +// It is always removed from our recent list no matter what. It only counts +// as "recent" if the event happened more recently than our event +// threshold ago. + +bool nsNavHistory::CheckIsRecentEvent(RecentEventHash* hashTable, + const nsACString& url) { + PRTime eventTime; + if (hashTable->Get(url, reinterpret_cast(&eventTime))) { + hashTable->Remove(url); + if (eventTime > GetNow() - RECENT_EVENT_THRESHOLD) return true; + return false; + } + return false; +} + +// nsNavHistory::ExpireNonrecentEvents +// +// This goes through our + +void nsNavHistory::ExpireNonrecentEvents(RecentEventHash* hashTable) { + int64_t threshold = GetNow() - RECENT_EVENT_THRESHOLD; + for (auto iter = hashTable->Iter(); !iter.Done(); iter.Next()) { + if (iter.Data() < threshold) { + iter.Remove(); + } + } +} + +// nsNavHistory::RowToResult +// +// Here, we just have a generic row. It could be a query, URL, visit, +// or full visit. + +nsresult nsNavHistory::RowToResult(mozIStorageValueArray* aRow, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult) { + NS_ASSERTION(aRow && aOptions && aResult, "Null pointer in RowToResult"); + + // URL + nsAutoCString url; + nsresult rv = aRow->GetUTF8String(kGetInfoIndex_URL, url); + NS_ENSURE_SUCCESS(rv, rv); + // In case of data corruption URL may be null, but our UI code prefers an + // empty string. + if (url.IsVoid()) { + MOZ_ASSERT(false, "Found a NULL url in moz_places"); + url.SetIsVoid(false); + } + + // title + nsAutoCString title; + bool isNull; + rv = aRow->GetIsNull(kGetInfoIndex_Title, &isNull); + NS_ENSURE_SUCCESS(rv, rv); + if (!isNull) { + rv = aRow->GetUTF8String(kGetInfoIndex_Title, title); + NS_ENSURE_SUCCESS(rv, rv); + } + + uint32_t accessCount = aRow->AsInt32(kGetInfoIndex_VisitCount); + PRTime time = aRow->AsInt64(kGetInfoIndex_VisitDate); + + // itemId + int64_t itemId = aRow->AsInt64(kGetInfoIndex_ItemId); + if (itemId == 0) { + // This is not a bookmark. For non-bookmarks we use a -1 itemId value. + // Notice ids in sqlite tables start from 1, so itemId cannot ever be 0. + itemId = -1; + } + + if (IsQueryURI(url)) { + // Special case "place:" URIs: turn them into containers. + if (itemId != -1) { + // We should never expose the history title for query nodes if the + // bookmark-item's title is set to null (the history title may be the + // query string without the place: prefix). Thus we call getItemTitle + // explicitly. Doing this in the SQL query would be less performant since + // it should be done for all results rather than only for queries. + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); + + rv = bookmarks->GetItemTitle(itemId, title); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsAutoCString guid; + if (itemId != -1) { + rv = aRow->GetUTF8String(nsNavBookmarks::kGetChildrenIndex_Guid, guid); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (aOptions->ResultType() == + nsNavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY || + aOptions->ResultType() == + nsNavHistoryQueryOptions::RESULTS_AS_LEFT_PANE_QUERY) { + rv = aRow->GetUTF8String(kGetInfoIndex_Guid, guid); + NS_ENSURE_SUCCESS(rv, rv); + } + + RefPtr resultNode; + rv = QueryRowToResult(itemId, guid, url, title, accessCount, time, + getter_AddRefs(resultNode)); + NS_ENSURE_SUCCESS(rv, rv); + + if (itemId != -1 || aOptions->ResultType() == + nsNavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT) { + // RESULTS_AS_TAGS_ROOT has date columns + resultNode->mDateAdded = aRow->AsInt64(kGetInfoIndex_ItemDateAdded); + resultNode->mLastModified = aRow->AsInt64(kGetInfoIndex_ItemLastModified); + if (resultNode->IsFolder()) { + // If it's a simple folder node (i.e. a shortcut to another folder), + // apply our options for it. However, if the parent type was tag query, + // we do not apply them, because it would not yield any results. + resultNode->GetAsContainer()->mOptions = aOptions; + } + } + + resultNode.forget(aResult); + return rv; + } else if (aOptions->ResultType() == + nsNavHistoryQueryOptions::RESULTS_AS_URI) { + RefPtr resultNode = + new nsNavHistoryResultNode(url, title, accessCount, time); + + if (itemId != -1) { + resultNode->mItemId = itemId; + resultNode->mDateAdded = aRow->AsInt64(kGetInfoIndex_ItemDateAdded); + resultNode->mLastModified = aRow->AsInt64(kGetInfoIndex_ItemLastModified); + + rv = aRow->GetUTF8String(nsNavBookmarks::kGetChildrenIndex_Guid, + resultNode->mBookmarkGuid); + NS_ENSURE_SUCCESS(rv, rv); + } + + resultNode->mFrecency = aRow->AsInt32(kGetInfoIndex_Frecency); + resultNode->mHidden = !!aRow->AsInt32(kGetInfoIndex_Hidden); + + nsAutoString tags; + rv = aRow->GetString(kGetInfoIndex_ItemTags, tags); + NS_ENSURE_SUCCESS(rv, rv); + if (!tags.IsVoid()) { + resultNode->mTags.Assign(tags); + } + + rv = aRow->GetUTF8String(kGetInfoIndex_Guid, resultNode->mPageGuid); + NS_ENSURE_SUCCESS(rv, rv); + + resultNode.forget(aResult); + return NS_OK; + } + + if (aOptions->ResultType() == nsNavHistoryQueryOptions::RESULTS_AS_VISIT) { + RefPtr resultNode = + new nsNavHistoryResultNode(url, title, accessCount, time); + + nsAutoString tags; + rv = aRow->GetString(kGetInfoIndex_ItemTags, tags); + if (!tags.IsVoid()) resultNode->mTags.Assign(tags); + + rv = aRow->GetUTF8String(kGetInfoIndex_Guid, resultNode->mPageGuid); + NS_ENSURE_SUCCESS(rv, rv); + + rv = aRow->GetInt64(kGetInfoIndex_VisitId, &resultNode->mVisitId); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t fromVisitId; + rv = aRow->GetInt64(kGetInfoIndex_FromVisitId, &fromVisitId); + NS_ENSURE_SUCCESS(rv, rv); + + if (fromVisitId > 0) { + resultNode->mFromVisitId = fromVisitId; + } + + resultNode->mTransitionType = aRow->AsInt32(kGetInfoIndex_VisitType); + + resultNode.forget(aResult); + return NS_OK; + } + + return NS_ERROR_FAILURE; +} + +// nsNavHistory::QueryRowToResult +// +// Called by RowToResult when the URI is a place: URI to generate the proper +// folder or query node. + +nsresult nsNavHistory::QueryRowToResult(int64_t itemId, + const nsACString& aBookmarkGuid, + const nsACString& aURI, + const nsACString& aTitle, + uint32_t aAccessCount, PRTime aTime, + nsNavHistoryResultNode** aNode) { + // Only assert if the itemId is set. In some cases (e.g. virtual queries), we + // have a guid, but not an itemId. + if (itemId != -1) { + MOZ_ASSERT(!aBookmarkGuid.IsEmpty()); + } + + nsCOMPtr query; + nsCOMPtr options; + nsresult rv = + QueryStringToQuery(aURI, getter_AddRefs(query), getter_AddRefs(options)); + RefPtr resultNode; + RefPtr queryObj = do_QueryObject(query); + NS_ENSURE_STATE(queryObj); + RefPtr optionsObj = do_QueryObject(options); + NS_ENSURE_STATE(optionsObj); + // If this failed the query does not parse correctly, let the error pass and + // handle it later. + if (NS_SUCCEEDED(rv)) { + // Check if this is a folder shortcut, so we can take a faster path. + nsCString targetFolderGuid = + GetSimpleBookmarksQueryParent(queryObj, optionsObj); + if (!targetFolderGuid.IsEmpty()) { + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); + + rv = bookmarks->ResultNodeForContainer(targetFolderGuid, optionsObj, + getter_AddRefs(resultNode)); + // If this failed the shortcut is pointing to nowhere, let the error pass + // and handle it later. + if (NS_SUCCEEDED(rv)) { + // At this point the node is set up like a regular folder node. Here + // we make the necessary change to make it a folder shortcut. + resultNode->mItemId = itemId; + resultNode->mBookmarkGuid = aBookmarkGuid; + resultNode->GetAsFolder()->mTargetFolderGuid = targetFolderGuid; + + // Use the query item title, unless it's empty (in that case use the + // concrete folder title). + if (!aTitle.IsEmpty()) { + resultNode->mTitle = aTitle; + } + } + } else { + // This is a regular query. + resultNode = new nsNavHistoryQueryResultNode(aTitle, aTime, aURI, + queryObj, optionsObj); + resultNode->mItemId = itemId; + resultNode->mBookmarkGuid = aBookmarkGuid; + } + } + + if (NS_FAILED(rv)) { + NS_WARNING("Generating a generic empty node for a broken query!"); + // This is a broken query, that either did not parse or points to not + // existing data. We don't want to return failure since that will kill the + // whole result. Instead make a generic empty query node. + resultNode = + new nsNavHistoryQueryResultNode(aTitle, 0, aURI, queryObj, optionsObj); + resultNode->mItemId = itemId; + resultNode->mBookmarkGuid = aBookmarkGuid; + // This is a perf hack to generate an empty query that skips filtering. + resultNode->GetAsQuery()->Options()->SetExcludeItems(true); + } + + resultNode.forget(aNode); + return NS_OK; +} + +// nsNavHistory::VisitIdToResultNode +// +// Used by the query results to create new nodes on the fly when +// notifications come in. This just creates a node for the given visit ID. + +nsresult nsNavHistory::VisitIdToResultNode(int64_t visitId, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult) { + MOZ_ASSERT(visitId > 0, "The passed-in visit id must be valid"); + nsAutoCString tagsFragment; + GetTagsSqlFragment(GetTagsFolder(), "h.id"_ns, true, tagsFragment); + + nsCOMPtr statement; + switch (aOptions->ResultType()) { + case nsNavHistoryQueryOptions::RESULTS_AS_VISIT: + // visit query - want exact visit time + // Should match kGetInfoIndex_* (see GetQueryResults) + statement = mDB->GetStatement( + nsLiteralCString( + "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " + "v.visit_date, null, null, null, null, null, ") + + tagsFragment + + nsLiteralCString(", h.frecency, h.hidden, h.guid, " + "v.id, v.from_visit, v.visit_type " + "FROM moz_places h " + "JOIN moz_historyvisits v ON h.id = v.place_id " + "WHERE v.id = :visit_id ")); + break; + + case nsNavHistoryQueryOptions::RESULTS_AS_URI: + // URL results - want last visit time + // Should match kGetInfoIndex_* (see GetQueryResults) + statement = mDB->GetStatement( + nsLiteralCString( + "SELECT h.id, h.url, h.title, h.rev_host, h.visit_count, " + "h.last_visit_date, null, null, null, null, null, ") + + tagsFragment + + nsLiteralCString(", h.frecency, h.hidden, h.guid, " + "null, null, null " + "FROM moz_places h " + "JOIN moz_historyvisits v ON h.id = v.place_id " + "WHERE v.id = :visit_id ")); + break; + + default: + // Query base types like RESULTS_AS_*_QUERY handle additions + // by registering their own observers when they are expanded. + return NS_OK; + } + NS_ENSURE_STATE(statement); + mozStorageStatementScoper scoper(statement); + + nsresult rv = statement->BindInt64ByName("visit_id"_ns, visitId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore = false; + rv = statement->ExecuteStep(&hasMore); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasMore) { + // Oops, we were passed an id that doesn't exist! It is indeed possible + // that between the insertion and the notification time, another enqueued + // task removed it. Since this can happen, we'll just issue a warning. + NS_WARNING( + "Cannot build a result node for a non existing visit id, was it " + "removed?"); + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr row = do_QueryInterface(statement, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return RowToResult(row, aOptions, aResult); +} + +nsresult nsNavHistory::BookmarkIdToResultNode( + int64_t aBookmarkId, nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult) { + MOZ_ASSERT(aBookmarkId > 0, "The passed-in bookmark id must be valid"); + nsAutoCString tagsFragment; + GetTagsSqlFragment(GetTagsFolder(), "h.id"_ns, true, tagsFragment); + // Should match kGetInfoIndex_* + nsCOMPtr stmt = mDB->GetStatement( + nsLiteralCString( + "SELECT b.fk, h.url, b.title, " + "h.rev_host, h.visit_count, h.last_visit_date, null, b.id, " + "b.dateAdded, b.lastModified, b.parent, ") + + tagsFragment + + nsLiteralCString(", h.frecency, h.hidden, h.guid, " + "null, null, null, b.guid, b.position, b.type, b.fk " + "FROM moz_bookmarks b " + "JOIN moz_places h ON b.fk = h.id " + "WHERE b.id = :item_id ")); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = stmt->BindInt64ByName("item_id"_ns, aBookmarkId); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore = false; + rv = stmt->ExecuteStep(&hasMore); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasMore) { + // Oops, we were passed an id that doesn't exist! It is indeed possible + // that between the insertion and the notification time, another enqueued + // task removed it. Since this can happen, we'll just issue a warning. + NS_WARNING( + "Cannot build a result node for a non existing bookmark id, was it " + "removed?"); + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr row = do_QueryInterface(stmt, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return RowToResult(row, aOptions, aResult); +} + +nsresult nsNavHistory::URIToResultNode(nsIURI* aURI, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult) { + MOZ_ASSERT(aURI, "The passed-in URI must be not-null"); + nsAutoCString tagsFragment; + GetTagsSqlFragment(GetTagsFolder(), "h.id"_ns, true, tagsFragment); + // Should match kGetInfoIndex_* + nsCOMPtr stmt = mDB->GetStatement( + nsLiteralCString("SELECT h.id, :page_url, COALESCE(b.title, h.title), " + "h.rev_host, h.visit_count, h.last_visit_date, null, " + "b.id, b.dateAdded, b.lastModified, b.parent, ") + + tagsFragment + + nsLiteralCString( + ", h.frecency, h.hidden, h.guid, " + "null, null, null, b.guid, b.position, b.type, b.fk " + "FROM moz_places h " + "LEFT JOIN moz_bookmarks b ON b.fk = h.id " + "WHERE h.url_hash = hash(:page_url) AND h.url = :page_url ")); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsresult rv = URIBinder::Bind(stmt, "page_url"_ns, aURI); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore = false; + rv = stmt->ExecuteStep(&hasMore); + NS_ENSURE_SUCCESS(rv, rv); + if (!hasMore) { + // Oops, we were passed an URL that doesn't exist! It is indeed possible + // that between the insertion and the notification time, another enqueued + // task removed it. Since this can happen, we'll just issue a warning. + NS_WARNING( + "Cannot build a result node for a non existing page id, was it " + "removed?"); + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr row = do_QueryInterface(stmt, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return RowToResult(row, aOptions, aResult); +} + +void nsNavHistory::GetAgeInDaysString(int32_t aInt, const char* aName, + nsACString& aResult) { + nsIStringBundle* bundle = GetBundle(); + if (bundle) { + AutoTArray strings; + strings.AppendElement()->AppendInt(aInt); + nsAutoString value; + nsresult rv = bundle->FormatStringFromName(aName, strings, value); + if (NS_SUCCEEDED(rv)) { + CopyUTF16toUTF8(value, aResult); + return; + } + } + aResult.Assign(aName); +} + +void nsNavHistory::GetStringFromName(const char* aName, nsACString& aResult) { + nsIStringBundle* bundle = GetBundle(); + if (bundle) { + nsAutoString value; + nsresult rv = bundle->GetStringFromName(aName, value); + if (NS_SUCCEEDED(rv)) { + CopyUTF16toUTF8(value, aResult); + return; + } + } + aResult.Assign(aName); +} + +// static +void nsNavHistory::GetMonthName(const PRExplodedTime& aTime, + nsACString& aResult) { + nsAutoString month; + + mozilla::intl::DateTimeFormat::ComponentsBag components; + components.month = Some(mozilla::intl::DateTimeFormat::Month::Long); + nsresult rv = + mozilla::intl::AppDateTimeFormat::Format(components, &aTime, month); + if (NS_FAILED(rv)) { + aResult = nsPrintfCString("[%d]", aTime.tm_month + 1); + return; + } + CopyUTF16toUTF8(month, aResult); +} + +// static +void nsNavHistory::GetMonthYear(const PRExplodedTime& aTime, + nsACString& aResult) { + nsAutoString monthYear; + mozilla::intl::DateTimeFormat::ComponentsBag components; + components.month = Some(mozilla::intl::DateTimeFormat::Month::Long); + components.year = Some(mozilla::intl::DateTimeFormat::Numeric::Numeric); + nsresult rv = + mozilla::intl::AppDateTimeFormat::Format(components, &aTime, monthYear); + if (NS_FAILED(rv)) { + aResult = nsPrintfCString("[%d-%d]", aTime.tm_month + 1, aTime.tm_year); + return; + } + CopyUTF16toUTF8(monthYear, aResult); +} + +namespace { + +// GetSimpleBookmarksQueryParent +// +// Determines if this is a simple bookmarks query for a +// folder with no other constraints. In these common cases, we can more +// efficiently compute the results. +// +// A simple bookmarks query will result in a hierarchical tree of +// bookmark items, folders and separators. +// +// Returns the folder ID if it is a simple folder query, 0 if not. +static nsCString GetSimpleBookmarksQueryParent( + const RefPtr& aQuery, + const RefPtr& aOptions) { + if (aQuery->Parents().Length() != 1) return ""_ns; + + bool hasIt; + if (NS_SUCCEEDED(aQuery->GetHasBeginTime(&hasIt)) && hasIt) return ""_ns; + if (NS_SUCCEEDED(aQuery->GetHasEndTime(&hasIt)) && hasIt) return ""_ns; + if (!aQuery->Domain().IsVoid()) return ""_ns; + if (aQuery->Uri()) return ""_ns; + if (!aQuery->SearchTerms().IsEmpty()) return ""_ns; + if (aQuery->Tags().Length() > 0) return ""_ns; + if (aOptions->MaxResults() > 0) return ""_ns; + + // Don't care about onlyBookmarked flag, since specifying a bookmark + // folder is inferring onlyBookmarked. + + return aQuery->Parents()[0]; +} + +// ParseSearchTermsFromQuery +// +// Construct an array of search terms from the given query. +// Within a query, all the terms are ANDed together. +// +// This just breaks the query up into words. We don't do anything fancy, +// not even quoting. We do, however, strip quotes, because people might +// try to input quotes expecting them to do something and get no results +// back. + +inline bool isQueryWhitespace(char16_t ch) { return ch == ' '; } + +void ParseSearchTermsFromQuery(const RefPtr& aQuery, + nsTArray* aTerms) { + int32_t lastBegin = -1; + if (!aQuery->SearchTerms().IsEmpty()) { + const nsString& searchTerms = aQuery->SearchTerms(); + for (uint32_t j = 0; j < searchTerms.Length(); j++) { + if (isQueryWhitespace(searchTerms[j]) || searchTerms[j] == '"') { + if (lastBegin >= 0) { + // found the end of a word + aTerms->AppendElement( + Substring(searchTerms, lastBegin, j - lastBegin)); + lastBegin = -1; + } + } else { + if (lastBegin < 0) { + // found the beginning of a word + lastBegin = j; + } + } + } + // last word + if (lastBegin >= 0) + aTerms->AppendElement(Substring(searchTerms, lastBegin)); + } +} + +} // namespace + +const mozilla::intl::Collator* nsNavHistory::GetCollator() { + if (mCollator) { + return mCollator.get(); + } + + auto result = mozilla::intl::LocaleService::TryCreateComponent< + mozilla::intl::Collator>(); + NS_ENSURE_TRUE(result.isOk(), nullptr); + auto collator = result.unwrap(); + + // Sort in a case-insensitive way, where "base" letters are considered + // equal, e.g: a = á, a = A, a ≠ b. + using mozilla::intl::Collator; + Collator::Options options{}; + options.sensitivity = Collator::Sensitivity::Base; + auto optResult = collator->SetOptions(options); + NS_ENSURE_TRUE(optResult.isOk(), nullptr); + + mCollator = UniquePtr(collator.release()); + + return mCollator.get(); +} + +nsIStringBundle* nsNavHistory::GetBundle() { + if (!mBundle) { + nsCOMPtr bundleService = + components::StringBundle::Service(); + NS_ENSURE_TRUE(bundleService, nullptr); + nsresult rv = bundleService->CreateBundle( + "chrome://places/locale/places.properties", getter_AddRefs(mBundle)); + NS_ENSURE_SUCCESS(rv, nullptr); + } + return mBundle; +} diff --git a/toolkit/components/places/nsNavHistory.h b/toolkit/components/places/nsNavHistory.h new file mode 100644 index 0000000000..2c291cfdbe --- /dev/null +++ b/toolkit/components/places/nsNavHistory.h @@ -0,0 +1,465 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsNavHistory_h_ +#define nsNavHistory_h_ + +#include "nsINavHistoryService.h" + +#include "nsIStringBundle.h" +#include "nsITimer.h" +#include "nsMaybeWeakPtr.h" +#include "nsCategoryCache.h" +#include "nsNetCID.h" +#include "nsToolkitCompsCID.h" +#include "nsURIHashKey.h" +#include "nsTHashtable.h" + +#include "nsNavHistoryResult.h" +#include "nsNavHistoryQuery.h" +#include "Database.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/intl/Collator.h" +#include "mozilla/UniquePtr.h" +#include "mozIStorageVacuumParticipant.h" + +#define QUERYUPDATE_TIME 0 +#define QUERYUPDATE_SIMPLE 1 +#define QUERYUPDATE_COMPLEX 2 +#define QUERYUPDATE_COMPLEX_WITH_BOOKMARKS 3 +#define QUERYUPDATE_HOST 4 +#define QUERYUPDATE_MOBILEPREF 5 +#define QUERYUPDATE_NONE 6 + +// Clamp title and URL to generously large, but not too large, length. +// See bug 319004 for details. +#define URI_LENGTH_MAX 65536 +#define TITLE_LENGTH_MAX 4096 + +// Microsecond timeout for "recent" events such as typed and bookmark following. +// If you typed it more than this time ago, it's not recent. +#define RECENT_EVENT_THRESHOLD PRTime((int64_t)15 * 60 * PR_USEC_PER_SEC) + +// The preference we watch to know when the mobile bookmarks folder is filled by +// sync. +#define MOBILE_BOOKMARKS_PREF "browser.bookmarks.showMobileBookmarks" + +// The guid of the mobile bookmarks virtual query. +#define MOBILE_BOOKMARKS_VIRTUAL_GUID "mobile_____v" + +#define ROOT_GUID "root________" +#define MENU_ROOT_GUID "menu________" +#define TOOLBAR_ROOT_GUID "toolbar_____" +#define UNFILED_ROOT_GUID "unfiled_____" +#define TAGS_ROOT_GUID "tags________" +#define MOBILE_ROOT_GUID "mobile______" + +class mozIStorageValueArray; +class nsIAutoCompleteController; +class nsIEffectiveTLDService; +class nsIIDNService; +class nsNavHistory; +class PlacesSQLQueryBuilder; + +// nsNavHistory + +class nsNavHistory final : public nsSupportsWeakReference, + public nsINavHistoryService, + public nsIObserver, + public mozIStorageVacuumParticipant { + friend class PlacesSQLQueryBuilder; + + public: + nsNavHistory(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSINAVHISTORYSERVICE + NS_DECL_NSIOBSERVER + NS_DECL_MOZISTORAGEVACUUMPARTICIPANT + + /** + * Obtains the nsNavHistory object. + */ + static already_AddRefed GetSingleton(); + + /** + * Initializes the nsNavHistory object. This should only be called once. + */ + nsresult Init(); + + /** + * Used by other components in the places directory such as the annotation + * service to get a reference to this history object. Returns a pointer to + * the service if it exists. Otherwise creates one. Returns nullptr on error. + */ + static nsNavHistory* GetHistoryService() { + if (!gHistoryService) { + nsCOMPtr serv = + do_GetService(NS_NAVHISTORYSERVICE_CONTRACTID); + NS_ENSURE_TRUE(serv, nullptr); + NS_ASSERTION(gHistoryService, "Should have static instance pointer now"); + } + return gHistoryService; + } + + /** + * Used by other components in the places directory to get a reference to a + * const version of this history object. + * + * @return A pointer to a const version of the service if it exists, + * nullptr otherwise. + */ + static const nsNavHistory* GetConstHistoryService() { + const nsNavHistory* const history = gHistoryService; + return history; + } + + /** + * Fetches the database id and the GUID associated to the given URI. + * + * @param aURI + * The page to look for. + * @param _pageId + * Will be set to the database id associated with the page. + * If the page doesn't exist, this will be zero. + * @param _GUID + * Will be set to the unique id associated with the page. + * If the page doesn't exist, this will be empty. + * @note This DOES NOT check for bad URLs other than that they're nonempty. + */ + nsresult GetIdForPage(nsIURI* aURI, int64_t* _pageId, nsCString& _GUID); + + /** + * Fetches the database id and the GUID associated to the given URI, creating + * a new database entry if one doesn't exist yet. + * + * @param aURI + * The page to look for or create. + * @param _pageId + * Will be set to the database id associated with the page. + * @param _GUID + * Will be set to the unique id associated with the page. + * @note This DOES NOT check for bad URLs other than that they're nonempty. + * @note This DOES NOT update frecency of the page. + */ + nsresult GetOrCreateIdForPage(nsIURI* aURI, int64_t* _pageId, + nsCString& _GUID); + + /** + * These functions return non-owning references to the locale-specific + * objects for places components. + */ + nsIStringBundle* GetBundle(); + const mozilla::intl::Collator* GetCollator(); + void GetStringFromName(const char* aName, nsACString& aResult); + void GetAgeInDaysString(int32_t aInt, const char* aName, nsACString& aResult); + static void GetMonthName(const PRExplodedTime& aTime, nsACString& aResult); + static void GetMonthYear(const PRExplodedTime& aTime, nsACString& aResult); + + // Returns whether history is enabled or not. + bool IsHistoryDisabled() { return !mHistoryEnabled; } + + // Returns whether or not diacritics must match in history text searches. + bool MatchDiacritics() const { return mMatchDiacritics; } + + // Constants for the columns returned by the above statement. + static const int32_t kGetInfoIndex_PageID; + static const int32_t kGetInfoIndex_URL; + static const int32_t kGetInfoIndex_Title; + static const int32_t kGetInfoIndex_RevHost; + static const int32_t kGetInfoIndex_VisitCount; + static const int32_t kGetInfoIndex_VisitDate; + static const int32_t kGetInfoIndex_FaviconURL; + static const int32_t kGetInfoIndex_ItemId; + static const int32_t kGetInfoIndex_ItemDateAdded; + static const int32_t kGetInfoIndex_ItemLastModified; + static const int32_t kGetInfoIndex_ItemParentId; + static const int32_t kGetInfoIndex_ItemTags; + static const int32_t kGetInfoIndex_Frecency; + static const int32_t kGetInfoIndex_Hidden; + static const int32_t kGetInfoIndex_Guid; + static const int32_t kGetInfoIndex_VisitId; + static const int32_t kGetInfoIndex_FromVisitId; + static const int32_t kGetInfoIndex_VisitType; + + int64_t GetTagsFolder(); + + // this actually executes a query and gives you results, it is used by + // nsNavHistoryQueryResultNode + nsresult GetQueryResults(nsNavHistoryQueryResultNode* aResultNode, + const RefPtr& aQuery, + const RefPtr& aOptions, + nsCOMArray* aResults); + + // Take a row of kGetInfoIndex_* columns and construct a ResultNode. + // The row must contain the full set of columns. + nsresult RowToResult(mozIStorageValueArray* aRow, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult); + nsresult QueryRowToResult(int64_t aItemId, const nsACString& aBookmarkGuid, + const nsACString& aURI, const nsACString& aTitle, + uint32_t aAccessCount, PRTime aTime, + nsNavHistoryResultNode** aNode); + + nsresult VisitIdToResultNode(int64_t visitId, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult); + + nsresult BookmarkIdToResultNode(int64_t aBookmarkId, + nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult); + nsresult URIToResultNode(nsIURI* aURI, nsNavHistoryQueryOptions* aOptions, + nsNavHistoryResultNode** aResult); + + /** + * Returns current number of days stored in history. + */ + int32_t GetDaysOfHistory(); + + void DomainNameFromURI(nsIURI* aURI, nsACString& aDomainName); + static PRTime NormalizeTime(uint32_t aRelative, PRTime aOffset); + + typedef nsTHashMap StringHash; + + enum RecentEventFlags { + RECENT_TYPED = 1 << 0, // User typed in URL recently + RECENT_ACTIVATED = 1 << 1, // User tapped URL link recently + RECENT_BOOKMARKED = 1 << 2 // User bookmarked URL recently + }; + + /** + * Returns any recent activity done with a URL. + * @return Any recent events associated with this URI. Each bit is set + * according to RecentEventFlags enum values. + */ + uint32_t GetRecentFlags(nsIURI* aURI); + + /** + * Whether there are visits. + * Note: This may cause synchronous I/O. + */ + bool hasHistoryEntries(); + + /** + * Returns whether the specified url has a embed visit. + * + * @param aURI + * URI of the page. + * @return whether the page has a embed visit. + */ + bool hasEmbedVisit(nsIURI* aURI); + + int32_t GetFrecencyAgedWeight(int32_t aAgeInDays) const { + if (aAgeInDays <= mFirstBucketCutoffInDays) { + return mFirstBucketWeight; + } + if (aAgeInDays <= mSecondBucketCutoffInDays) { + return mSecondBucketWeight; + } + if (aAgeInDays <= mThirdBucketCutoffInDays) { + return mThirdBucketWeight; + } + if (aAgeInDays <= mFourthBucketCutoffInDays) { + return mFourthBucketWeight; + } + return mDefaultWeight; + } + + int32_t GetFrecencyTransitionBonus(int32_t aTransitionType, bool aVisited, + bool aRedirect = false) const { + if (aRedirect) { + return mRedirectSourceVisitBonus; + } + + switch (aTransitionType) { + case nsINavHistoryService::TRANSITION_EMBED: + return mEmbedVisitBonus; + case nsINavHistoryService::TRANSITION_FRAMED_LINK: + return mFramedLinkVisitBonus; + case nsINavHistoryService::TRANSITION_LINK: + return mLinkVisitBonus; + case nsINavHistoryService::TRANSITION_TYPED: + return aVisited ? mTypedVisitBonus : mUnvisitedTypedBonus; + case nsINavHistoryService::TRANSITION_BOOKMARK: + return aVisited ? mBookmarkVisitBonus : mUnvisitedBookmarkBonus; + case nsINavHistoryService::TRANSITION_DOWNLOAD: + return mDownloadVisitBonus; + case nsINavHistoryService::TRANSITION_REDIRECT_PERMANENT: + return mPermRedirectVisitBonus; + case nsINavHistoryService::TRANSITION_REDIRECT_TEMPORARY: + return mTempRedirectVisitBonus; + case nsINavHistoryService::TRANSITION_RELOAD: + return mReloadVisitBonus; + default: + // 0 == undefined (see bug #375777 for details) + NS_WARNING_ASSERTION(!aTransitionType, + "new transition but no bonus for frecency"); + return mDefaultVisitBonus; + } + } + + int32_t GetNumVisitsForFrecency() const { return mNumVisitsForFrecency; } + + /** + * Updates and invalidates the mDaysOfHistory cache. Should be + * called whenever a visit is added. + */ + void UpdateDaysOfHistory(PRTime visitTime); + + /** + * Store last insterted id for a table. + */ + static mozilla::Atomic sLastInsertedPlaceId; + static mozilla::Atomic sLastInsertedVisitId; + + /** + * Tracks whether frecency is currently being decayed. + */ + static mozilla::Atomic sIsFrecencyDecaying; + /** + * Tracks whether there's frecency to be recalculated. + */ + static mozilla::Atomic sShouldStartFrecencyRecalculation; + + static void StoreLastInsertedId(const nsACString& aTable, + const int64_t aLastInsertedId); + + static nsresult FilterResultSet( + nsNavHistoryQueryResultNode* aParentNode, + const nsCOMArray& aSet, + nsCOMArray* aFiltered, + const RefPtr& aQuery, + nsNavHistoryQueryOptions* aOptions); + + static void InvalidateDaysOfHistory(); + + private: + ~nsNavHistory(); + + // used by GetHistoryService + static nsNavHistory* gHistoryService; + + static mozilla::Atomic sDaysOfHistory; + + protected: + // Database handle. + RefPtr mDB; + + /** + * Loads all of the preferences that we use into member variables. + * + * @note If mPrefBranch is nullptr, this does nothing. + */ + void LoadPrefs(); + + /** + * Calculates and returns value for mCachedNow. + * This is an hack to avoid calling PR_Now() too often, as is the case when + * we're asked the ageindays of many history entries in a row. A timer is + * set which will clear our valid flag after a short timeout. + */ + PRTime GetNow(); + PRTime mCachedNow; + nsCOMPtr mExpireNowTimer; + /** + * Called when the cached now value is expired and needs renewal. + */ + static void expireNowTimerCallback(nsITimer* aTimer, void* aClosure); + + nsresult ConstructQueryString( + const RefPtr& aQuery, + const RefPtr& aOptions, nsCString& queryString, + bool& aParamsPresent, StringHash& aAddParams); + + nsresult QueryToSelectClause(const RefPtr& aQuery, + const RefPtr& aOptions, + nsCString* aClause); + nsresult BindQueryClauseParameters( + mozIStorageBaseStatement* statement, + const RefPtr& aQuery, + const RefPtr& aOptions); + + nsresult ResultsAsList(mozIStorageStatement* statement, + nsNavHistoryQueryOptions* aOptions, + nsCOMArray* aResults); + + // effective tld service + nsCOMPtr mTLDService; + nsCOMPtr mIDNService; + + // localization + nsCOMPtr mBundle; + mozilla::UniquePtr mCollator; + + // recent events + typedef nsTHashMap RecentEventHash; + RecentEventHash mRecentTyped; + RecentEventHash mRecentLink; + RecentEventHash mRecentBookmark; + + bool CheckIsRecentEvent(RecentEventHash* hashTable, const nsACString& url); + void ExpireNonrecentEvents(RecentEventHash* hashTable); + + // Whether history is enabled or not. + // Will mimic value of the places.history.enabled preference. + bool mHistoryEnabled; + + // Whether or not diacritics must match in history text searches. + // Will mimic value of the places.search.matchDiacritics preference. + bool mMatchDiacritics; + + // Frecency preferences. + int32_t mNumVisitsForFrecency; + int32_t mFirstBucketCutoffInDays; + int32_t mSecondBucketCutoffInDays; + int32_t mThirdBucketCutoffInDays; + int32_t mFourthBucketCutoffInDays; + int32_t mFirstBucketWeight; + int32_t mSecondBucketWeight; + int32_t mThirdBucketWeight; + int32_t mFourthBucketWeight; + int32_t mDefaultWeight; + int32_t mEmbedVisitBonus; + int32_t mFramedLinkVisitBonus; + int32_t mLinkVisitBonus; + int32_t mTypedVisitBonus; + int32_t mBookmarkVisitBonus; + int32_t mDownloadVisitBonus; + int32_t mPermRedirectVisitBonus; + int32_t mTempRedirectVisitBonus; + int32_t mRedirectSourceVisitBonus; + int32_t mDefaultVisitBonus; + int32_t mUnvisitedBookmarkBonus; + int32_t mUnvisitedTypedBonus; + int32_t mReloadVisitBonus; + + nsresult RecalculateOriginFrecencyStatsInternal(); + + // in nsNavHistoryQuery.cpp + nsresult TokensToQuery( + const nsTArray& aTokens, + nsNavHistoryQuery* aQuery, nsNavHistoryQueryOptions* aOptions); + + int64_t mTagsFolder; + + int64_t mLastCachedStartOfDay; + int64_t mLastCachedEndOfDay; +}; + +#define PLACES_URI_PREFIX "place:" + +/* Returns true if the given URI represents a history query. */ +inline bool IsQueryURI(const nsCString& uri) { + return StringBeginsWith(uri, nsLiteralCString(PLACES_URI_PREFIX)); +} + +/* Extracts the query string from a query URI. */ +inline const nsDependentCSubstring QueryURIToQuery(const nsCString& uri) { + NS_ASSERTION(IsQueryURI(uri), "should only be called for query URIs"); + return Substring(uri, nsLiteralCString(PLACES_URI_PREFIX).Length()); +} + +#endif // nsNavHistory_h_ diff --git a/toolkit/components/places/nsNavHistoryQuery.cpp b/toolkit/components/places/nsNavHistoryQuery.cpp new file mode 100644 index 0000000000..7118e86b28 --- /dev/null +++ b/toolkit/components/places/nsNavHistoryQuery.cpp @@ -0,0 +1,1241 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains the definitions of nsNavHistoryQuery, + * nsNavHistoryQueryOptions, and those functions in nsINavHistory that directly + * support queries (specifically QueryStringToQuery and QueryToQueryString). + */ + +#include "mozilla/DebugOnly.h" + +#include "nsNavHistory.h" +#include "nsNavBookmarks.h" +#include "nsEscape.h" +#include "nsCOMArray.h" +#include "nsNetUtil.h" +#include "nsTArray.h" +#include "nsQueryObject.h" +#include "prprf.h" +#include "nsVariant.h" + +using namespace mozilla; +using namespace mozilla::places; + +static nsresult ParseQueryBooleanString(const nsCString& aString, bool* aValue); + +// query getters +typedef decltype(&nsINavHistoryQuery::GetOnlyBookmarked) BoolQueryGetter; +typedef decltype(&nsINavHistoryQuery::GetBeginTimeReference) Uint32QueryGetter; +typedef decltype(&nsINavHistoryQuery::GetBeginTime) Int64QueryGetter; +static void AppendBoolKeyValueIfTrue(nsACString& aString, + const nsCString& aName, + nsINavHistoryQuery* aQuery, + BoolQueryGetter getter); +static void AppendUint32KeyValueIfNonzero(nsACString& aString, + const nsCString& aName, + nsINavHistoryQuery* aQuery, + Uint32QueryGetter getter); +static void AppendInt64KeyValueIfNonzero(nsACString& aString, + const nsCString& aName, + nsINavHistoryQuery* aQuery, + Int64QueryGetter getter); + +// query setters +typedef decltype(&nsINavHistoryQuery::SetOnlyBookmarked) BoolQuerySetter; +typedef decltype(&nsINavHistoryQuery::SetBeginTimeReference) Uint32QuerySetter; +typedef decltype(&nsINavHistoryQuery::SetBeginTime) Int64QuerySetter; +static void SetQueryKeyBool(const nsCString& aValue, nsINavHistoryQuery* aQuery, + BoolQuerySetter setter); +static void SetQueryKeyUint32(const nsCString& aValue, + nsINavHistoryQuery* aQuery, + Uint32QuerySetter setter); +static void SetQueryKeyInt64(const nsCString& aValue, + nsINavHistoryQuery* aQuery, + Int64QuerySetter setter); + +// options setters +typedef decltype(&nsINavHistoryQueryOptions::SetExpandQueries) + BoolOptionsSetter; +typedef decltype(&nsINavHistoryQueryOptions::SetMaxResults) Uint32OptionsSetter; +typedef decltype(&nsINavHistoryQueryOptions::SetResultType) Uint16OptionsSetter; +static void SetOptionsKeyBool(const nsCString& aValue, + nsINavHistoryQueryOptions* aOptions, + BoolOptionsSetter setter); +static void SetOptionsKeyUint16(const nsCString& aValue, + nsINavHistoryQueryOptions* aOptions, + Uint16OptionsSetter setter); +static void SetOptionsKeyUint32(const nsCString& aValue, + nsINavHistoryQueryOptions* aOptions, + Uint32OptionsSetter setter); + +// Components of a query string. +// Note that query strings are also generated in nsNavBookmarks::GetFolderURI +// for performance reasons, so if you change these values, change that, too. +#define QUERYKEY_BEGIN_TIME "beginTime" +#define QUERYKEY_BEGIN_TIME_REFERENCE "beginTimeRef" +#define QUERYKEY_END_TIME "endTime" +#define QUERYKEY_END_TIME_REFERENCE "endTimeRef" +#define QUERYKEY_SEARCH_TERMS "terms" +#define QUERYKEY_MIN_VISITS "minVisits" +#define QUERYKEY_MAX_VISITS "maxVisits" +#define QUERYKEY_ONLY_BOOKMARKED "onlyBookmarked" +#define QUERYKEY_DOMAIN_IS_HOST "domainIsHost" +#define QUERYKEY_DOMAIN "domain" +#define QUERYKEY_PARENT "parent" +#define QUERYKEY_NOTANNOTATION "!annotation" +#define QUERYKEY_ANNOTATION "annotation" +#define QUERYKEY_URI "uri" +#define QUERYKEY_GROUP "group" +#define QUERYKEY_SORT "sort" +#define QUERYKEY_RESULT_TYPE "type" +#define QUERYKEY_EXCLUDE_ITEMS "excludeItems" +#define QUERYKEY_EXCLUDE_QUERIES "excludeQueries" +#define QUERYKEY_EXPAND_QUERIES "expandQueries" +#define QUERYKEY_FORCE_ORIGINAL_TITLE "originalTitle" +#define QUERYKEY_INCLUDE_HIDDEN "includeHidden" +#define QUERYKEY_MAX_RESULTS "maxResults" +#define QUERYKEY_QUERY_TYPE "queryType" +#define QUERYKEY_TAG "tag" +#define QUERYKEY_NOTTAGS "!tags" +#define QUERYKEY_ASYNC_ENABLED "asyncEnabled" +#define QUERYKEY_TRANSITION "transition" + +inline void AppendAmpersandIfNonempty(nsACString& aString) { + if (!aString.IsEmpty()) aString.Append('&'); +} +inline void AppendInt16(nsACString& str, int16_t i) { + nsAutoCString tmp; + tmp.AppendInt(i); + str.Append(tmp); +} +inline void AppendInt32(nsACString& str, int32_t i) { + nsAutoCString tmp; + tmp.AppendInt(i); + str.Append(tmp); +} +inline void AppendInt64(nsACString& str, int64_t i) { + nsCString tmp; + tmp.AppendInt(i); + str.Append(tmp); +} + +NS_IMETHODIMP +nsNavHistory::QueryStringToQuery(const nsACString& aQueryString, + nsINavHistoryQuery** _query, + nsINavHistoryQueryOptions** _options) { + NS_ENSURE_ARG_POINTER(_query); + NS_ENSURE_ARG_POINTER(_options); + + nsTArray tokens; + nsresult rv = TokenizeQueryString(aQueryString, &tokens); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr options = new nsNavHistoryQueryOptions(); + RefPtr query = new nsNavHistoryQuery(); + rv = TokensToQuery(tokens, query, options); + MOZ_ASSERT(NS_SUCCEEDED(rv), "The query string should be valid"); + if (NS_FAILED(rv)) { + NS_WARNING("Unable to parse the query string: "); + NS_WARNING(PromiseFlatCString(aQueryString).get()); + } + + options.forget(_options); + query.forget(_query); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistory::QueryToQueryString(nsINavHistoryQuery* aQuery, + nsINavHistoryQueryOptions* aOptions, + nsACString& aQueryString) { + NS_ENSURE_ARG(aQuery); + NS_ENSURE_ARG(aOptions); + + RefPtr query = do_QueryObject(aQuery); + NS_ENSURE_STATE(query); + RefPtr options = do_QueryObject(aOptions); + NS_ENSURE_STATE(options); + + nsAutoCString queryString; + bool hasIt; + + // begin time + query->GetHasBeginTime(&hasIt); + if (hasIt) { + AppendInt64KeyValueIfNonzero(queryString, + nsLiteralCString(QUERYKEY_BEGIN_TIME), query, + &nsINavHistoryQuery::GetBeginTime); + AppendUint32KeyValueIfNonzero( + queryString, nsLiteralCString(QUERYKEY_BEGIN_TIME_REFERENCE), query, + &nsINavHistoryQuery::GetBeginTimeReference); + } + + // end time + query->GetHasEndTime(&hasIt); + if (hasIt) { + AppendInt64KeyValueIfNonzero(queryString, + nsLiteralCString(QUERYKEY_END_TIME), query, + &nsINavHistoryQuery::GetEndTime); + AppendUint32KeyValueIfNonzero( + queryString, nsLiteralCString(QUERYKEY_END_TIME_REFERENCE), query, + &nsINavHistoryQuery::GetEndTimeReference); + } + + // search terms + if (!query->SearchTerms().IsEmpty()) { + const nsString& searchTerms = query->SearchTerms(); + nsCString escapedTerms; + if (!NS_Escape(NS_ConvertUTF16toUTF8(searchTerms), escapedTerms, + url_XAlphas)) + return NS_ERROR_OUT_OF_MEMORY; + + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_SEARCH_TERMS "="); + queryString += escapedTerms; + } + + // min and max visits + int32_t minVisits; + if (NS_SUCCEEDED(query->GetMinVisits(&minVisits)) && minVisits >= 0) { + AppendAmpersandIfNonempty(queryString); + queryString.AppendLiteral(QUERYKEY_MIN_VISITS "="); + AppendInt32(queryString, minVisits); + } + + int32_t maxVisits; + if (NS_SUCCEEDED(query->GetMaxVisits(&maxVisits)) && maxVisits >= 0) { + AppendAmpersandIfNonempty(queryString); + queryString.AppendLiteral(QUERYKEY_MAX_VISITS "="); + AppendInt32(queryString, maxVisits); + } + + // only bookmarked + AppendBoolKeyValueIfTrue(queryString, + nsLiteralCString(QUERYKEY_ONLY_BOOKMARKED), query, + &nsINavHistoryQuery::GetOnlyBookmarked); + + // domain (+ is host), only call if hasDomain, which means non-IsVoid + // this means we may get an empty string for the domain in the result, + // which is valid + if (!query->Domain().IsVoid()) { + AppendBoolKeyValueIfTrue(queryString, + nsLiteralCString(QUERYKEY_DOMAIN_IS_HOST), query, + &nsINavHistoryQuery::GetDomainIsHost); + const nsCString& domain = query->Domain(); + nsCString escapedDomain; + bool success = NS_Escape(domain, escapedDomain, url_XAlphas); + NS_ENSURE_TRUE(success, NS_ERROR_OUT_OF_MEMORY); + + AppendAmpersandIfNonempty(queryString); + queryString.AppendLiteral(QUERYKEY_DOMAIN "="); + queryString.Append(escapedDomain); + } + + // uri + if (query->Uri()) { + nsCOMPtr uri = query->Uri(); + nsAutoCString uriSpec; + nsresult rv = uri->GetSpec(uriSpec); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString escaped; + bool success = NS_Escape(uriSpec, escaped, url_XAlphas); + NS_ENSURE_TRUE(success, NS_ERROR_OUT_OF_MEMORY); + + AppendAmpersandIfNonempty(queryString); + queryString.AppendLiteral(QUERYKEY_URI "="); + queryString.Append(escaped); + } + + // annotation + if (!query->Annotation().IsEmpty()) { + AppendAmpersandIfNonempty(queryString); + if (query->AnnotationIsNot()) { + queryString.AppendLiteral(QUERYKEY_NOTANNOTATION "="); + } else { + queryString.AppendLiteral(QUERYKEY_ANNOTATION "="); + } + const nsCString& annot = query->Annotation(); + nsAutoCString escaped; + bool success = NS_Escape(annot, escaped, url_XAlphas); + NS_ENSURE_TRUE(success, NS_ERROR_OUT_OF_MEMORY); + queryString.Append(escaped); + } + + // parents + const nsTArray& parents = query->Parents(); + for (uint32_t i = 0; i < parents.Length(); ++i) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_PARENT "="); + queryString += parents[i]; + } + + // tags + const nsTArray& tags = query->Tags(); + for (uint32_t i = 0; i < tags.Length(); ++i) { + nsAutoCString escapedTag; + if (!NS_Escape(NS_ConvertUTF16toUTF8(tags[i]), escapedTag, url_XAlphas)) + return NS_ERROR_OUT_OF_MEMORY; + + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_TAG "="); + queryString += escapedTag; + } + AppendBoolKeyValueIfTrue(queryString, nsLiteralCString(QUERYKEY_NOTTAGS), + query, &nsINavHistoryQuery::GetTagsAreNot); + + // transitions + const nsTArray& transitions = query->Transitions(); + for (uint32_t i = 0; i < transitions.Length(); ++i) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_TRANSITION "="); + AppendInt64(queryString, transitions[i]); + } + + // sorting + if (options->SortingMode() != nsINavHistoryQueryOptions::SORT_BY_NONE) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_SORT "="); + AppendInt16(queryString, options->SortingMode()); + } + + // result type + if (options->ResultType() != nsINavHistoryQueryOptions::RESULTS_AS_URI) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_RESULT_TYPE "="); + AppendInt16(queryString, options->ResultType()); + } + + // exclude items + if (options->ExcludeItems()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_EXCLUDE_ITEMS "=1"); + } + + // exclude queries + if (options->ExcludeQueries()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_EXCLUDE_QUERIES "=1"); + } + + // expand queries + if (!options->ExpandQueries()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_EXPAND_QUERIES "=0"); + } + + // include hidden + if (options->IncludeHidden()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_INCLUDE_HIDDEN "=1"); + } + + // max results + if (options->MaxResults()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_MAX_RESULTS "="); + AppendInt32(queryString, options->MaxResults()); + } + + // queryType + if (options->QueryType() != nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_QUERY_TYPE "="); + AppendInt16(queryString, options->QueryType()); + } + + // async enabled + if (options->AsyncEnabled()) { + AppendAmpersandIfNonempty(queryString); + queryString += nsLiteralCString(QUERYKEY_ASYNC_ENABLED "=1"); + } + + aQueryString.AssignLiteral("place:"); + aQueryString.Append(queryString); + return NS_OK; +} + +nsresult nsNavHistory::TokensToQuery(const nsTArray& aTokens, + nsNavHistoryQuery* aQuery, + nsNavHistoryQueryOptions* aOptions) { + nsresult rv; + + if (aTokens.Length() == 0) return NS_OK; + + nsTArray parents; + nsTArray tags; + nsTArray transitions; + for (uint32_t i = 0; i < aTokens.Length(); i++) { + const QueryKeyValuePair& kvp = aTokens[i]; + + // begin time + if (kvp.key.EqualsLiteral(QUERYKEY_BEGIN_TIME)) { + SetQueryKeyInt64(kvp.value, aQuery, &nsINavHistoryQuery::SetBeginTime); + + // begin time reference + } else if (kvp.key.EqualsLiteral(QUERYKEY_BEGIN_TIME_REFERENCE)) { + SetQueryKeyUint32(kvp.value, aQuery, + &nsINavHistoryQuery::SetBeginTimeReference); + + // end time + } else if (kvp.key.EqualsLiteral(QUERYKEY_END_TIME)) { + SetQueryKeyInt64(kvp.value, aQuery, &nsINavHistoryQuery::SetEndTime); + + // end time reference + } else if (kvp.key.EqualsLiteral(QUERYKEY_END_TIME_REFERENCE)) { + SetQueryKeyUint32(kvp.value, aQuery, + &nsINavHistoryQuery::SetEndTimeReference); + + // search terms + } else if (kvp.key.EqualsLiteral(QUERYKEY_SEARCH_TERMS)) { + nsCString unescapedTerms = kvp.value; + NS_UnescapeURL(unescapedTerms); // modifies input + rv = aQuery->SetSearchTerms(NS_ConvertUTF8toUTF16(unescapedTerms)); + NS_ENSURE_SUCCESS(rv, rv); + + // min visits + } else if (kvp.key.EqualsLiteral(QUERYKEY_MIN_VISITS)) { + int32_t visits = kvp.value.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) + aQuery->SetMinVisits(visits); + else + NS_WARNING("Bad number for minVisits in query"); + + // max visits + } else if (kvp.key.EqualsLiteral(QUERYKEY_MAX_VISITS)) { + int32_t visits = kvp.value.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) + aQuery->SetMaxVisits(visits); + else + NS_WARNING("Bad number for maxVisits in query"); + + // onlyBookmarked flag + } else if (kvp.key.EqualsLiteral(QUERYKEY_ONLY_BOOKMARKED)) { + SetQueryKeyBool(kvp.value, aQuery, + &nsINavHistoryQuery::SetOnlyBookmarked); + + // domainIsHost flag + } else if (kvp.key.EqualsLiteral(QUERYKEY_DOMAIN_IS_HOST)) { + SetQueryKeyBool(kvp.value, aQuery, &nsINavHistoryQuery::SetDomainIsHost); + + // domain string + } else if (kvp.key.EqualsLiteral(QUERYKEY_DOMAIN)) { + nsAutoCString unescapedDomain(kvp.value); + NS_UnescapeURL(unescapedDomain); // modifies input + rv = aQuery->SetDomain(unescapedDomain); + NS_ENSURE_SUCCESS(rv, rv); + + // parent folders (guids) + } else if (kvp.key.EqualsLiteral(QUERYKEY_PARENT)) { + parents.AppendElement(kvp.value); + + // uri + } else if (kvp.key.EqualsLiteral(QUERYKEY_URI)) { + nsAutoCString unescapedUri(kvp.value); + NS_UnescapeURL(unescapedUri); // modifies input + nsCOMPtr uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), unescapedUri); + if (NS_FAILED(rv)) { + NS_WARNING("Unable to parse URI"); + } + rv = aQuery->SetUri(uri); + NS_ENSURE_SUCCESS(rv, rv); + + // not annotation + } else if (kvp.key.EqualsLiteral(QUERYKEY_NOTANNOTATION)) { + nsAutoCString unescaped(kvp.value); + NS_UnescapeURL(unescaped); // modifies input + aQuery->SetAnnotationIsNot(true); + aQuery->SetAnnotation(unescaped); + + // annotation + } else if (kvp.key.EqualsLiteral(QUERYKEY_ANNOTATION)) { + nsAutoCString unescaped(kvp.value); + NS_UnescapeURL(unescaped); // modifies input + aQuery->SetAnnotationIsNot(false); + aQuery->SetAnnotation(unescaped); + + // tag + } else if (kvp.key.EqualsLiteral(QUERYKEY_TAG)) { + nsAutoCString unescaped(kvp.value); + NS_UnescapeURL(unescaped); // modifies input + NS_ConvertUTF8toUTF16 tag(unescaped); + if (!tags.Contains(tag)) { + tags.AppendElement(tag); + } + + // not tags + } else if (kvp.key.EqualsLiteral(QUERYKEY_NOTTAGS)) { + SetQueryKeyBool(kvp.value, aQuery, &nsINavHistoryQuery::SetTagsAreNot); + + // transition + } else if (kvp.key.EqualsLiteral(QUERYKEY_TRANSITION)) { + uint32_t transition = kvp.value.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + if (!transitions.Contains(transition)) + transitions.AppendElement(transition); + } else { + NS_WARNING("Invalid Int32 transition value."); + } + + // sorting mode + } else if (kvp.key.EqualsLiteral(QUERYKEY_SORT)) { + SetOptionsKeyUint16(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetSortingMode); + // result type + } else if (kvp.key.EqualsLiteral(QUERYKEY_RESULT_TYPE)) { + SetOptionsKeyUint16(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetResultType); + + // exclude items + } else if (kvp.key.EqualsLiteral(QUERYKEY_EXCLUDE_ITEMS)) { + SetOptionsKeyBool(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetExcludeItems); + + // exclude queries + } else if (kvp.key.EqualsLiteral(QUERYKEY_EXCLUDE_QUERIES)) { + SetOptionsKeyBool(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetExcludeQueries); + + // expand queries + } else if (kvp.key.EqualsLiteral(QUERYKEY_EXPAND_QUERIES)) { + SetOptionsKeyBool(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetExpandQueries); + // include hidden + } else if (kvp.key.EqualsLiteral(QUERYKEY_INCLUDE_HIDDEN)) { + SetOptionsKeyBool(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetIncludeHidden); + // max results + } else if (kvp.key.EqualsLiteral(QUERYKEY_MAX_RESULTS)) { + SetOptionsKeyUint32(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetMaxResults); + // query type + } else if (kvp.key.EqualsLiteral(QUERYKEY_QUERY_TYPE)) { + SetOptionsKeyUint16(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetQueryType); + // async enabled + } else if (kvp.key.EqualsLiteral(QUERYKEY_ASYNC_ENABLED)) { + SetOptionsKeyBool(kvp.value, aOptions, + &nsINavHistoryQueryOptions::SetAsyncEnabled); + // unknown key + } else { + NS_WARNING("TokensToQueries(), ignoring unknown key: "); + NS_WARNING(kvp.key.get()); + } + } + + if (parents.Length() != 0) { + rv = aQuery->SetParents(parents); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (tags.Length() > 0) { + aQuery->SetTags(std::move(tags)); + } + + if (transitions.Length() > 0) { + rv = aQuery->SetTransitions(transitions); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +// ParseQueryBooleanString +// +// Converts a 0/1 or true/false string into a bool + +nsresult ParseQueryBooleanString(const nsCString& aString, bool* aValue) { + if (aString.EqualsLiteral("1") || aString.EqualsLiteral("true")) { + *aValue = true; + return NS_OK; + } + if (aString.EqualsLiteral("0") || aString.EqualsLiteral("false")) { + *aValue = false; + return NS_OK; + } + return NS_ERROR_INVALID_ARG; +} + +// nsINavHistoryQuery ********************************************************** + +NS_IMPL_ISUPPORTS(nsNavHistoryQuery, nsNavHistoryQuery, nsINavHistoryQuery) + +// nsINavHistoryQuery::nsNavHistoryQuery +// +// This must initialize the object such that the default values will cause +// all history to be returned if this query is used. Then the caller can +// just set the things it's interested in. + +nsNavHistoryQuery::nsNavHistoryQuery() + : mMinVisits(-1), + mMaxVisits(-1), + mBeginTime(0), + mBeginTimeReference(TIME_RELATIVE_EPOCH), + mEndTime(0), + mEndTimeReference(TIME_RELATIVE_EPOCH), + mOnlyBookmarked(false), + mDomainIsHost(false), + mAnnotationIsNot(false), + mTagsAreNot(false) { + // differentiate not set (IsVoid) from empty string (local files) + mDomain.SetIsVoid(true); +} + +nsNavHistoryQuery::nsNavHistoryQuery(const nsNavHistoryQuery& aOther) + : mMinVisits(aOther.mMinVisits), + mMaxVisits(aOther.mMaxVisits), + mBeginTime(aOther.mBeginTime), + mBeginTimeReference(aOther.mBeginTimeReference), + mEndTime(aOther.mEndTime), + mEndTimeReference(aOther.mEndTimeReference), + mSearchTerms(aOther.mSearchTerms), + mOnlyBookmarked(aOther.mOnlyBookmarked), + mDomainIsHost(aOther.mDomainIsHost), + mDomain(aOther.mDomain), + mUri(aOther.mUri), + mAnnotationIsNot(aOther.mAnnotationIsNot), + mAnnotation(aOther.mAnnotation), + mParents(aOther.mParents.Clone()), + mTags(aOther.mTags.Clone()), + mTagsAreNot(aOther.mTagsAreNot), + mTransitions(aOther.mTransitions.Clone()) {} + +NS_IMETHODIMP nsNavHistoryQuery::GetBeginTime(PRTime* aBeginTime) { + *aBeginTime = mBeginTime; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetBeginTime(PRTime aBeginTime) { + mBeginTime = aBeginTime; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetBeginTimeReference(uint32_t* _retval) { + *_retval = mBeginTimeReference; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetBeginTimeReference(uint32_t aReference) { + if (aReference > TIME_RELATIVE_NOW) return NS_ERROR_INVALID_ARG; + mBeginTimeReference = aReference; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetHasBeginTime(bool* _retval) { + *_retval = !(mBeginTimeReference == TIME_RELATIVE_EPOCH && mBeginTime == 0); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetAbsoluteBeginTime(PRTime* _retval) { + *_retval = nsNavHistory::NormalizeTime(mBeginTimeReference, mBeginTime); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetEndTime(PRTime* aEndTime) { + *aEndTime = mEndTime; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetEndTime(PRTime aEndTime) { + mEndTime = aEndTime; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetEndTimeReference(uint32_t* _retval) { + *_retval = mEndTimeReference; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetEndTimeReference(uint32_t aReference) { + if (aReference > TIME_RELATIVE_NOW) return NS_ERROR_INVALID_ARG; + mEndTimeReference = aReference; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetHasEndTime(bool* _retval) { + *_retval = !(mEndTimeReference == TIME_RELATIVE_EPOCH && mEndTime == 0); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetAbsoluteEndTime(PRTime* _retval) { + *_retval = nsNavHistory::NormalizeTime(mEndTimeReference, mEndTime); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetSearchTerms(nsAString& aSearchTerms) { + aSearchTerms = mSearchTerms; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetSearchTerms(const nsAString& aSearchTerms) { + mSearchTerms = aSearchTerms; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::GetHasSearchTerms(bool* _retval) { + *_retval = (!mSearchTerms.IsEmpty()); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetMinVisits(int32_t* _retval) { + NS_ENSURE_ARG_POINTER(_retval); + *_retval = mMinVisits; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetMinVisits(int32_t aVisits) { + mMinVisits = aVisits; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetMaxVisits(int32_t* _retval) { + NS_ENSURE_ARG_POINTER(_retval); + *_retval = mMaxVisits; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetMaxVisits(int32_t aVisits) { + mMaxVisits = aVisits; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetOnlyBookmarked(bool* aOnlyBookmarked) { + *aOnlyBookmarked = mOnlyBookmarked; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetOnlyBookmarked(bool aOnlyBookmarked) { + mOnlyBookmarked = aOnlyBookmarked; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetDomainIsHost(bool* aDomainIsHost) { + *aDomainIsHost = mDomainIsHost; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetDomainIsHost(bool aDomainIsHost) { + mDomainIsHost = aDomainIsHost; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetDomain(nsACString& aDomain) { + aDomain = mDomain; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetDomain(const nsACString& aDomain) { + mDomain = aDomain; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::GetHasDomain(bool* _retval) { + // note that empty but not void is still a valid query (local files) + *_retval = (!mDomain.IsVoid()); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetUri(nsIURI** aUri) { + NS_IF_ADDREF(*aUri = mUri); + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetUri(nsIURI* aUri) { + mUri = aUri; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::GetHasUri(bool* aHasUri) { + *aHasUri = (mUri != nullptr); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetAnnotationIsNot(bool* aIsNot) { + *aIsNot = mAnnotationIsNot; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetAnnotationIsNot(bool aIsNot) { + mAnnotationIsNot = aIsNot; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetAnnotation(nsACString& aAnnotation) { + aAnnotation = mAnnotation; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::SetAnnotation(const nsACString& aAnnotation) { + mAnnotation = aAnnotation; + return NS_OK; +} +NS_IMETHODIMP nsNavHistoryQuery::GetHasAnnotation(bool* aHasIt) { + *aHasIt = !mAnnotation.IsEmpty(); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetTags(nsIVariant** aTags) { + NS_ENSURE_ARG_POINTER(aTags); + + RefPtr out = new nsVariant(); + + uint32_t arrayLen = mTags.Length(); + + nsresult rv; + if (arrayLen == 0) + rv = out->SetAsEmptyArray(); + else { + // Note: The resulting nsIVariant dupes both the array and its elements. + const char16_t** array = reinterpret_cast( + moz_xmalloc(arrayLen * sizeof(char16_t*))); + for (uint32_t i = 0; i < arrayLen; ++i) { + array[i] = mTags[i].get(); + } + + rv = out->SetAsArray(nsIDataType::VTYPE_WCHAR_STR, nullptr, arrayLen, + reinterpret_cast(array)); + free(array); + } + NS_ENSURE_SUCCESS(rv, rv); + + out.forget(aTags); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::SetTags(nsIVariant* aTags) { + NS_ENSURE_ARG(aTags); + + uint16_t dataType = aTags->GetDataType(); + + // Caller passed in empty array. Easy -- clear our mTags array and return. + if (dataType == nsIDataType::VTYPE_EMPTY_ARRAY) { + mTags.Clear(); + return NS_OK; + } + + // Before we go any further, make sure caller passed in an array. + NS_ENSURE_TRUE(dataType == nsIDataType::VTYPE_ARRAY, NS_ERROR_ILLEGAL_VALUE); + + uint16_t eltType; + nsIID eltIID; + uint32_t arrayLen; + void* array; + + // Convert the nsIVariant to an array. We own the resulting buffer and its + // elements. + nsresult rv = aTags->GetAsArray(&eltType, &eltIID, &arrayLen, &array); + NS_ENSURE_SUCCESS(rv, rv); + + // If element type is not wstring, thanks a lot. Your memory die now. + if (eltType != nsIDataType::VTYPE_WCHAR_STR) { + switch (eltType) { + case nsIDataType::VTYPE_ID: + case nsIDataType::VTYPE_CHAR_STR: { + char** charArray = reinterpret_cast(array); + for (uint32_t i = 0; i < arrayLen; ++i) { + if (charArray[i]) free(charArray[i]); + } + } break; + case nsIDataType::VTYPE_INTERFACE: + case nsIDataType::VTYPE_INTERFACE_IS: { + nsISupports** supportsArray = reinterpret_cast(array); + for (uint32_t i = 0; i < arrayLen; ++i) { + NS_IF_RELEASE(supportsArray[i]); + } + } break; + // The other types are primitives that do not need to be freed. + } + free(array); + return NS_ERROR_ILLEGAL_VALUE; + } + + char16_t** tags = reinterpret_cast(array); + mTags.Clear(); + + // Finally, add each passed-in tag to our mTags array and then sort it. + for (uint32_t i = 0; i < arrayLen; ++i) { + // Don't allow nulls. + if (!tags[i]) { + free(tags); + return NS_ERROR_ILLEGAL_VALUE; + } + + nsDependentString tag(tags[i]); + + // Don't store duplicate tags. This isn't just to save memory or to be + // fancy; the SQL that's built from the tags relies on no dupes. + if (!mTags.Contains(tag)) { + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + mTags.AppendElement(tag); + } + free(tags[i]); + } + free(tags); + + mTags.Sort(); + + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetTagsAreNot(bool* aTagsAreNot) { + NS_ENSURE_ARG_POINTER(aTagsAreNot); + *aTagsAreNot = mTagsAreNot; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::SetTagsAreNot(bool aTagsAreNot) { + mTagsAreNot = aTagsAreNot; + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetParents(nsTArray& aGuids) { + aGuids = mParents.Clone(); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetParentCount(uint32_t* aGuidCount) { + *aGuidCount = mParents.Length(); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::SetParents(const nsTArray& aGuids) { + mParents.Clear(); + if (!mParents.Assign(aGuids, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetTransitions( + nsTArray& aTransitions) { + aTransitions = mTransitions.Clone(); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::GetTransitionCount(uint32_t* aCount) { + *aCount = mTransitions.Length(); + return NS_OK; +} + +NS_IMETHODIMP nsNavHistoryQuery::SetTransitions( + const nsTArray& aTransitions) { + if (!mTransitions.Assign(aTransitions, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQuery::Clone(nsINavHistoryQuery** _clone) { + nsNavHistoryQuery* clone = nullptr; + Unused << Clone(&clone); + *_clone = clone; + return NS_OK; +} + +nsresult nsNavHistoryQuery::Clone(nsNavHistoryQuery** _clone) { + *_clone = nullptr; + RefPtr clone = new nsNavHistoryQuery(*this); + clone.forget(_clone); + return NS_OK; +} + +// nsNavHistoryQueryOptions +NS_IMPL_ISUPPORTS(nsNavHistoryQueryOptions, nsNavHistoryQueryOptions, + nsINavHistoryQueryOptions) + +nsNavHistoryQueryOptions::nsNavHistoryQueryOptions() + : mSort(0), + mResultType(0), + mExcludeItems(false), + mExcludeQueries(false), + mExpandQueries(true), + mIncludeHidden(false), + mMaxResults(0), + mQueryType(nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY), + mAsyncEnabled(false) {} + +nsNavHistoryQueryOptions::nsNavHistoryQueryOptions( + const nsNavHistoryQueryOptions& other) + : mSort(other.mSort), + mResultType(other.mResultType), + mExcludeItems(other.mExcludeItems), + mExcludeQueries(other.mExcludeQueries), + mExpandQueries(other.mExpandQueries), + mIncludeHidden(other.mIncludeHidden), + mMaxResults(other.mMaxResults), + mQueryType(other.mQueryType), + mAsyncEnabled(other.mAsyncEnabled) {} + +// sortingMode +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetSortingMode(uint16_t* aMode) { + *aMode = mSort; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetSortingMode(uint16_t aMode) { + if (aMode > SORT_BY_FRECENCY_DESCENDING) return NS_ERROR_INVALID_ARG; + mSort = aMode; + return NS_OK; +} + +// resultType +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetResultType(uint16_t* aType) { + *aType = mResultType; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetResultType(uint16_t aType) { + if (aType > RESULTS_AS_LEFT_PANE_QUERY) return NS_ERROR_INVALID_ARG; + // Tag queries, containers and the roots query are bookmarks related, so we + // set the QueryType accordingly. + if (aType == RESULTS_AS_TAGS_ROOT || aType == RESULTS_AS_ROOTS_QUERY || + aType == RESULTS_AS_LEFT_PANE_QUERY) { + mQueryType = QUERY_TYPE_BOOKMARKS; + } + mResultType = aType; + return NS_OK; +} + +// excludeItems +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetExcludeItems(bool* aExclude) { + *aExclude = mExcludeItems; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetExcludeItems(bool aExclude) { + mExcludeItems = aExclude; + return NS_OK; +} + +// excludeQueries +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetExcludeQueries(bool* aExclude) { + *aExclude = mExcludeQueries; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetExcludeQueries(bool aExclude) { + mExcludeQueries = aExclude; + return NS_OK; +} +// expandQueries +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetExpandQueries(bool* aExpand) { + *aExpand = mExpandQueries; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetExpandQueries(bool aExpand) { + mExpandQueries = aExpand; + return NS_OK; +} + +// includeHidden +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetIncludeHidden(bool* aIncludeHidden) { + *aIncludeHidden = mIncludeHidden; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetIncludeHidden(bool aIncludeHidden) { + mIncludeHidden = aIncludeHidden; + return NS_OK; +} + +// maxResults +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetMaxResults(uint32_t* aMaxResults) { + *aMaxResults = mMaxResults; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetMaxResults(uint32_t aMaxResults) { + mMaxResults = aMaxResults; + return NS_OK; +} + +// queryType +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetQueryType(uint16_t* _retval) { + *_retval = mQueryType; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetQueryType(uint16_t aQueryType) { + // Tag query and containers are forced to QUERY_TYPE_BOOKMARKS when the + // resultType is set. + if (mResultType == RESULTS_AS_TAGS_ROOT || + mResultType == RESULTS_AS_LEFT_PANE_QUERY || + mResultType == RESULTS_AS_ROOTS_QUERY) + return NS_OK; + mQueryType = aQueryType; + return NS_OK; +} + +// asyncEnabled +NS_IMETHODIMP +nsNavHistoryQueryOptions::GetAsyncEnabled(bool* _asyncEnabled) { + *_asyncEnabled = mAsyncEnabled; + return NS_OK; +} +NS_IMETHODIMP +nsNavHistoryQueryOptions::SetAsyncEnabled(bool aAsyncEnabled) { + mAsyncEnabled = aAsyncEnabled; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQueryOptions::Clone(nsINavHistoryQueryOptions** _clone) { + nsNavHistoryQueryOptions* clone = nullptr; + Unused << Clone(&clone); + *_clone = clone; + return NS_OK; +} + +nsresult nsNavHistoryQueryOptions::Clone(nsNavHistoryQueryOptions** _clone) { + *_clone = nullptr; + RefPtr clone = new nsNavHistoryQueryOptions(*this); + clone.forget(_clone); + return NS_OK; +} + +// AppendBoolKeyValueIfTrue + +void // static +AppendBoolKeyValueIfTrue(nsACString& aString, const nsCString& aName, + nsINavHistoryQuery* aQuery, BoolQueryGetter getter) { + bool value; + DebugOnly rv = (aQuery->*getter)(&value); + NS_ASSERTION(NS_SUCCEEDED(rv), "Failure getting boolean value"); + if (value) { + AppendAmpersandIfNonempty(aString); + aString += aName; + aString.AppendLiteral("=1"); + } +} + +// AppendUint32KeyValueIfNonzero + +void // static +AppendUint32KeyValueIfNonzero(nsACString& aString, const nsCString& aName, + nsINavHistoryQuery* aQuery, + Uint32QueryGetter getter) { + uint32_t value; + DebugOnly rv = (aQuery->*getter)(&value); + NS_ASSERTION(NS_SUCCEEDED(rv), "Failure getting value"); + if (value) { + AppendAmpersandIfNonempty(aString); + aString += aName; + + // AppendInt requires a concrete string + nsAutoCString appendMe("="); + appendMe.AppendInt(value); + aString.Append(appendMe); + } +} + +// AppendInt64KeyValueIfNonzero + +void // static +AppendInt64KeyValueIfNonzero(nsACString& aString, const nsCString& aName, + nsINavHistoryQuery* aQuery, + Int64QueryGetter getter) { + PRTime value; + DebugOnly rv = (aQuery->*getter)(&value); + NS_ASSERTION(NS_SUCCEEDED(rv), "Failure getting value"); + if (value) { + AppendAmpersandIfNonempty(aString); + aString += aName; + nsAutoCString appendMe("="); + appendMe.AppendInt(static_cast(value)); + aString.Append(appendMe); + } +} + +// SetQuery/OptionsKeyBool + +void // static +SetQueryKeyBool(const nsCString& aValue, nsINavHistoryQuery* aQuery, + BoolQuerySetter setter) { + bool value; + nsresult rv = ParseQueryBooleanString(aValue, &value); + if (NS_SUCCEEDED(rv)) { + rv = (aQuery->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting boolean key value"); + } + } else { + NS_WARNING("Invalid boolean key value in query string."); + } +} +void // static +SetOptionsKeyBool(const nsCString& aValue, nsINavHistoryQueryOptions* aOptions, + BoolOptionsSetter setter) { + bool value = false; + nsresult rv = ParseQueryBooleanString(aValue, &value); + if (NS_SUCCEEDED(rv)) { + rv = (aOptions->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting boolean key value"); + } + } else { + NS_WARNING("Invalid boolean key value in query string."); + } +} + +// SetQuery/OptionsKeyUint32 + +void // static +SetQueryKeyUint32(const nsCString& aValue, nsINavHistoryQuery* aQuery, + Uint32QuerySetter setter) { + nsresult rv; + uint32_t value = aValue.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + rv = (aQuery->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting Int32 key value"); + } + } else { + NS_WARNING("Invalid Int32 key value in query string."); + } +} +void // static +SetOptionsKeyUint32(const nsCString& aValue, + nsINavHistoryQueryOptions* aOptions, + Uint32OptionsSetter setter) { + nsresult rv; + uint32_t value = aValue.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + rv = (aOptions->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting Int32 key value"); + } + } else { + NS_WARNING("Invalid Int32 key value in query string."); + } +} + +void // static +SetOptionsKeyUint16(const nsCString& aValue, + nsINavHistoryQueryOptions* aOptions, + Uint16OptionsSetter setter) { + nsresult rv; + uint16_t value = static_cast(aValue.ToInteger(&rv)); + if (NS_SUCCEEDED(rv)) { + rv = (aOptions->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting Int16 key value"); + } + } else { + NS_WARNING("Invalid Int16 key value in query string."); + } +} + +// SetQueryKeyInt64 + +void SetQueryKeyInt64(const nsCString& aValue, nsINavHistoryQuery* aQuery, + Int64QuerySetter setter) { + nsresult rv; + int64_t value; + if (PR_sscanf(aValue.get(), "%lld", &value) == 1) { + rv = (aQuery->*setter)(value); + if (NS_FAILED(rv)) { + NS_WARNING("Error setting Int64 key value"); + } + } else { + NS_WARNING("Invalid Int64 value in query string."); + } +} diff --git a/toolkit/components/places/nsNavHistoryQuery.h b/toolkit/components/places/nsNavHistoryQuery.h new file mode 100644 index 0000000000..1537ac8d20 --- /dev/null +++ b/toolkit/components/places/nsNavHistoryQuery.h @@ -0,0 +1,143 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The definitions of nsNavHistoryQuery and nsNavHistoryQueryOptions. This + * header file should only be included from nsNavHistory.h, include that if + * you want these classes. + */ + +#ifndef nsNavHistoryQuery_h_ +#define nsNavHistoryQuery_h_ + +// nsNavHistoryQuery +// +// This class encapsulates the parameters for basic history queries for +// building UI, trees, lists, etc. + +#include "mozilla/Attributes.h" + +#define NS_NAVHISTORYQUERY_IID \ + { \ + 0xb10185e0, 0x86eb, 0x4612, { \ + 0x95, 0x7c, 0x09, 0x34, 0xf2, 0xb1, 0xce, 0xd7 \ + } \ + } + +class nsNavHistoryQuery final : public nsINavHistoryQuery { + public: + nsNavHistoryQuery(); + nsNavHistoryQuery(const nsNavHistoryQuery& aOther); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_NAVHISTORYQUERY_IID) + NS_DECL_ISUPPORTS + NS_DECL_NSINAVHISTORYQUERY + + int32_t MinVisits() { return mMinVisits; } + int32_t MaxVisits() { return mMaxVisits; } + PRTime BeginTime() { return mBeginTime; } + uint32_t BeginTimeReference() { return mBeginTimeReference; } + PRTime EndTime() { return mEndTime; } + uint32_t EndTimeReference() { return mEndTimeReference; } + const nsString& SearchTerms() { return mSearchTerms; } + bool OnlyBookmarked() { return mOnlyBookmarked; } + bool DomainIsHost() { return mDomainIsHost; } + const nsCString& Domain() { return mDomain; } + nsIURI* Uri() { return mUri; } // NOT AddRef-ed! + bool AnnotationIsNot() { return mAnnotationIsNot; } + const nsCString& Annotation() { return mAnnotation; } + const nsTArray& Parents() const { return mParents; } + + const nsTArray& Tags() const { return mTags; } + void SetTags(nsTArray aTags) { mTags = std::move(aTags); } + bool TagsAreNot() { return mTagsAreNot; } + + const nsTArray& Transitions() const { return mTransitions; } + + nsresult Clone(nsNavHistoryQuery** _clone); + + private: + ~nsNavHistoryQuery() = default; + + protected: + // IF YOU ADD MORE ITEMS: + // * Add to the copy constructor + int32_t mMinVisits; + int32_t mMaxVisits; + PRTime mBeginTime; + uint32_t mBeginTimeReference; + PRTime mEndTime; + uint32_t mEndTimeReference; + nsString mSearchTerms; + bool mOnlyBookmarked; + bool mDomainIsHost; + nsCString mDomain; // Default is IsVoid, empty string is valid query + nsCOMPtr mUri; + bool mAnnotationIsNot; + nsCString mAnnotation; + nsTArray mParents; + nsTArray mTags; + bool mTagsAreNot; + nsTArray mTransitions; +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsNavHistoryQuery, NS_NAVHISTORYQUERY_IID) + +// nsNavHistoryQueryOptions + +#define NS_NAVHISTORYQUERYOPTIONS_IID \ + { \ + 0x95f8ba3b, 0xd681, 0x4d89, { \ + 0xab, 0xd1, 0xfd, 0xae, 0xf2, 0xa3, 0xde, 0x18 \ + } \ + } + +class nsNavHistoryQueryOptions final : public nsINavHistoryQueryOptions { + public: + nsNavHistoryQueryOptions(); + nsNavHistoryQueryOptions(const nsNavHistoryQueryOptions& other); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_NAVHISTORYQUERYOPTIONS_IID) + + NS_DECL_ISUPPORTS + NS_DECL_NSINAVHISTORYQUERYOPTIONS + + uint16_t SortingMode() const { return mSort; } + uint16_t ResultType() const { return mResultType; } + bool ExcludeItems() const { return mExcludeItems; } + bool ExcludeQueries() const { return mExcludeQueries; } + bool ExpandQueries() const { return mExpandQueries; } + bool IncludeHidden() const { return mIncludeHidden; } + uint32_t MaxResults() const { return mMaxResults; } + uint16_t QueryType() const { return mQueryType; } + bool AsyncEnabled() const { return mAsyncEnabled; } + + nsresult Clone(nsNavHistoryQueryOptions** _clone); + + private: + ~nsNavHistoryQueryOptions() = default; + + // IF YOU ADD MORE ITEMS: + // * Add to the copy constructor + // * Add a new getter for C++ above if it makes sense + // * Add to the serialization code (see nsNavHistory::QueriesToQueryString()) + // * Add to the deserialization code (see nsNavHistory::QueryStringToQueries) + // * Add to the nsNavHistory.cpp::GetSimpleBookmarksQueryFolder function if + // applicable + uint16_t mSort; + uint16_t mResultType; + bool mExcludeItems; + bool mExcludeQueries; + bool mExpandQueries; + bool mIncludeHidden; + uint32_t mMaxResults; + uint16_t mQueryType; + bool mAsyncEnabled; +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsNavHistoryQueryOptions, + NS_NAVHISTORYQUERYOPTIONS_IID) + +#endif // nsNavHistoryQuery_h_ diff --git a/toolkit/components/places/nsNavHistoryResult.cpp b/toolkit/components/places/nsNavHistoryResult.cpp new file mode 100644 index 0000000000..e3e9e48ec3 --- /dev/null +++ b/toolkit/components/places/nsNavHistoryResult.cpp @@ -0,0 +1,4409 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include "nsNavHistory.h" +#include "nsNavBookmarks.h" +#include "nsFaviconService.h" +#include "Helpers.h" +#include "mozilla/DebugOnly.h" +#include "nsDebug.h" +#include "nsNetUtil.h" +#include "nsString.h" +#include "nsReadableUtils.h" +#include "nsUnicharUtils.h" +#include "prtime.h" +#include "mozIStorageRow.h" +#include "mozIStorageResultSet.h" +#include "nsQueryObject.h" +#include "mozilla/dom/PlacesObservers.h" +#include "mozilla/dom/PlacesVisit.h" +#include "mozilla/dom/PlacesVisitRemoved.h" +#include "mozilla/dom/PlacesVisitTitle.h" +#include "mozilla/dom/PlacesBookmarkAddition.h" +#include "mozilla/dom/PlacesBookmarkRemoved.h" +#include "mozilla/dom/PlacesBookmarkMoved.h" +#include "mozilla/dom/PlacesBookmarkKeyword.h" +#include "mozilla/dom/PlacesBookmarkTags.h" +#include "mozilla/dom/PlacesBookmarkTime.h" +#include "mozilla/dom/PlacesBookmarkTitle.h" +#include "mozilla/dom/PlacesBookmarkUrl.h" +#include "mozilla/dom/PlacesFavicon.h" + +#include "nsCycleCollectionParticipant.h" + +// Thanks, Windows.h :( +#undef CompareString + +#define TO_ICONTAINER(_node) \ + static_cast(_node) + +#define TO_CONTAINER(_node) static_cast(_node) + +#define NOTIFY_RESULT_OBSERVERS_RET(_result, _method, _ret) \ + PR_BEGIN_MACRO \ + NS_ENSURE_TRUE(_result, _ret); \ + if (!_result->mSuppressNotifications) { \ + ENUMERATE_WEAKARRAY(_result->mObservers, nsINavHistoryResultObserver, \ + _method) \ + } \ + PR_END_MACRO + +#define NOTIFY_RESULT_OBSERVERS(_result, _method) \ + NOTIFY_RESULT_OBSERVERS_RET(_result, _method, NS_ERROR_UNEXPECTED) + +// What we want is: NS_INTERFACE_MAP_ENTRY(self) for static IID accessors, +// but some of our classes (like nsNavHistoryResult) have an ambiguous base +// class of nsISupports which prevents this from working (the default macro +// converts it to nsISupports, then addrefs it, then returns it). Therefore, we +// expand the macro here and change it so that it works. Yuck. +#define NS_INTERFACE_MAP_STATIC_AMBIGUOUS(_class) \ + if (aIID.Equals(NS_GET_IID(_class))) { \ + NS_ADDREF(this); \ + *aInstancePtr = this; \ + return NS_OK; \ + } else + +// Number of changes to handle separately in a batch. If more changes are +// requested the node will switch to full refresh mode. +#define MAX_BATCH_CHANGES_BEFORE_REFRESH 5 + +// Number of Page_removed events to handle by simply refreshing all containers, +// because doing so is much faster than incremental updates. +#define MAX_PAGE_REMOVES_BEFORE_REFRESH 10 + +using namespace mozilla; +using namespace mozilla::dom; +using namespace mozilla::places; + +namespace { + +/** + * Returns conditions for query update. + * QUERYUPDATE_TIME: + * This query is only limited by an inclusive time range on the first + * query object. The caller can quickly evaluate the time itself if it + * chooses. This is even simpler than "simple" below. + * QUERYUPDATE_SIMPLE: + * This query is evaluatable using evaluateQueryForNode to do live + * updating. + * QUERYUPDATE_COMPLEX: + * This query is not evaluatable using evaluateQueryForNode. When something + * happens that this query updates, you will need to re-run the query. + * QUERYUPDATE_COMPLEX_WITH_BOOKMARKS: + * A complex query that additionally has dependence on bookmarks. All + * bookmark-dependent queries fall under this category. + * QUERYUPDATE_MOBILEPREF: + * A complex query but only updates when the mobile preference changes. + * QUERYUPDATE_NONE: + * A query that never updates, e.g. the left-pane root query. + * + * aHasSearchTerms will be set to true if the query has any dependence on + * keywords. When there is no dependence on keywords, we can handle title + * change operations as simple instead of complex. + */ +uint32_t getUpdateRequirements(const RefPtr& aQuery, + const RefPtr& aOptions, + bool* aHasSearchTerms) { + // first check if there are search terms + bool hasSearchTerms = *aHasSearchTerms = !aQuery->SearchTerms().IsEmpty(); + + bool nonTimeBasedItems = false; + bool domainBasedItems = false; + + if (aQuery->Parents().Length() > 0 || aQuery->OnlyBookmarked() || + aQuery->Tags().Length() > 0 || + (aOptions->QueryType() == + nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS && + hasSearchTerms)) { + return QUERYUPDATE_COMPLEX_WITH_BOOKMARKS; + } + + // Note: we don't currently have any complex non-bookmarked items, but these + // are expected to be added. Put detection of these items here. + if (hasSearchTerms || !aQuery->Domain().IsVoid() || aQuery->Uri() != nullptr) + nonTimeBasedItems = true; + + if (!aQuery->Domain().IsVoid()) domainBasedItems = true; + + if (aOptions->ResultType() == nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT) + return QUERYUPDATE_COMPLEX_WITH_BOOKMARKS; + + if (aOptions->ResultType() == + nsINavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY) + return QUERYUPDATE_MOBILEPREF; + + if (aOptions->ResultType() == + nsINavHistoryQueryOptions::RESULTS_AS_LEFT_PANE_QUERY) + return QUERYUPDATE_NONE; + + // Whenever there is a maximum number of results, + // and we are not a bookmark query we must requery. This + // is because we can't generally know if any given addition/change causes + // the item to be in the top N items in the database. + uint16_t sortingMode = aOptions->SortingMode(); + if (aOptions->MaxResults() > 0 && + sortingMode != nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING && + sortingMode != nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING) + return QUERYUPDATE_COMPLEX; + + if (domainBasedItems) return QUERYUPDATE_HOST; + if (!nonTimeBasedItems) return QUERYUPDATE_TIME; + + return QUERYUPDATE_SIMPLE; +} + +/** + * We might have interesting encodings and different case in the host name. + * This will convert that host name into an ASCII host name by sending it + * through the URI canonicalization. The result can be used for comparison + * with other ASCII host name strings. + */ +nsresult asciiHostNameFromHostString(const nsACString& aHostName, + nsACString& aAscii) { + aAscii.Truncate(); + if (aHostName.IsEmpty()) { + return NS_OK; + } + // To properly generate a uri we must provide a protocol. + nsAutoCString fakeURL("http://"); + fakeURL.Append(aHostName); + nsCOMPtr uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), fakeURL); + NS_ENSURE_SUCCESS(rv, rv); + rv = uri->GetAsciiHost(aAscii); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +/** + * This runs the node through the given query to see if satisfies the + * query conditions. Not every query parameters are handled by this code, + * but we handle the most common ones so that performance is better. + * We assume that the time on the node is the time that we want to compare. + * This is not necessarily true because URL nodes have the last access time, + * which is not necessarily the same. However, since this is being called + * to update the list, we assume that the last access time is the current + * access time that we are being asked to compare so it works out. + * Returns true if node matches the query, false if not. + */ +bool evaluateQueryForNode(const RefPtr& aQuery, + const RefPtr& aOptions, + const RefPtr& aNode) { + // Hidden + if (aNode->mHidden && !aOptions->IncludeHidden()) return false; + + bool hasIt; + // Begin time + aQuery->GetHasBeginTime(&hasIt); + if (hasIt) { + PRTime beginTime = nsNavHistory::NormalizeTime(aQuery->BeginTimeReference(), + aQuery->BeginTime()); + if (aNode->mTime < beginTime) return false; + } + + // End time + aQuery->GetHasEndTime(&hasIt); + if (hasIt) { + PRTime endTime = nsNavHistory::NormalizeTime(aQuery->EndTimeReference(), + aQuery->EndTime()); + if (aNode->mTime > endTime) return false; + } + + // Search terms + if (!aQuery->SearchTerms().IsEmpty()) { + // we can use the existing filtering code, just give it our one object in + // an array. + nsCOMArray inputSet; + inputSet.AppendObject(aNode); + nsCOMArray filteredSet; + nsresult rv = nsNavHistory::FilterResultSet(nullptr, inputSet, &filteredSet, + aQuery, aOptions); + if (NS_FAILED(rv)) return false; + if (!filteredSet.Count()) return false; + } + + // Domain/host + if (!aQuery->Domain().IsVoid()) { + nsCOMPtr nodeUri; + if (NS_FAILED(NS_NewURI(getter_AddRefs(nodeUri), aNode->mURI))) + return false; + nsAutoCString asciiRequest; + if (NS_FAILED(asciiHostNameFromHostString(aQuery->Domain(), asciiRequest))) + return false; + if (aQuery->DomainIsHost()) { + nsAutoCString host; + if (NS_FAILED(nodeUri->GetAsciiHost(host))) return false; + + if (!asciiRequest.Equals(host)) return false; + } + // check domain names. + nsNavHistory* history = nsNavHistory::GetHistoryService(); + if (!history) return false; + nsAutoCString domain; + history->DomainNameFromURI(nodeUri, domain); + if (!asciiRequest.Equals(domain)) return false; + } + + // URI + if (aQuery->Uri()) { + nsCOMPtr nodeUri; + if (NS_FAILED(NS_NewURI(getter_AddRefs(nodeUri), aNode->mURI))) + return false; + bool equals; + nsresult rv = aQuery->Uri()->Equals(nodeUri, &equals); + NS_ENSURE_SUCCESS(rv, false); + if (!equals) return false; + } + + // Transitions + const nsTArray& transitions = aQuery->Transitions(); + if (aNode->mTransitionType > 0 && transitions.Length() && + !transitions.Contains(aNode->mTransitionType)) { + return false; + } + + // If we ever make it to the bottom, that means it passed all the tests for + // the given query. + return true; +} + +// Emulate string comparison (used for sorting) for PRTime and int. +inline int32_t ComparePRTime(PRTime a, PRTime b) { + if (a < b) + return -1; + else if (a > b) + return 1; + return 0; +} +inline int32_t CompareIntegers(uint32_t a, uint32_t b) { return a - b; } + +} // anonymous namespace + +NS_IMPL_CYCLE_COLLECTION(nsNavHistoryResultNode, mParent) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsNavHistoryResultNode) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsINavHistoryResultNode) + NS_INTERFACE_MAP_ENTRY(nsINavHistoryResultNode) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(nsNavHistoryResultNode) +NS_IMPL_CYCLE_COLLECTING_RELEASE(nsNavHistoryResultNode) + +nsNavHistoryResultNode::nsNavHistoryResultNode(const nsACString& aURI, + const nsACString& aTitle, + uint32_t aAccessCount, + PRTime aTime) + : mParent(nullptr), + mURI(aURI), + mTitle(aTitle), + mAreTagsSorted(false), + mAccessCount(aAccessCount), + mTime(aTime), + mBookmarkIndex(-1), + mItemId(-1), + mVisitId(-1), + mFromVisitId(-1), + mDateAdded(0), + mLastModified(0), + mIndentLevel(-1), + mFrecency(0), + mHidden(false), + mTransitionType(0) { + mTags.SetIsVoid(true); +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetIcon(nsACString& aIcon) { + if (this->IsContainer() || mURI.IsEmpty()) { + return NS_OK; + } + + aIcon.AppendLiteral("page-icon:"); + aIcon.Append(mURI); + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetParent(nsINavHistoryContainerResultNode** aParent) { + NS_IF_ADDREF(*aParent = mParent); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetParentResult(nsINavHistoryResult** aResult) { + *aResult = nullptr; + if (IsContainer()) + NS_IF_ADDREF(*aResult = GetAsContainer()->mResult); + else if (mParent) + NS_IF_ADDREF(*aResult = mParent->mResult); + + NS_ENSURE_STATE(*aResult); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetTags(nsAString& aTags) { + // Only URI-nodes may be associated with tags + if (!IsURI()) { + aTags.Truncate(); + return NS_OK; + } + + // Initially, the tags string is set to a void string (see constructor). We + // then build it the first time this method called is called (and by that, + // implicitly unset the void flag). Result observers may re-set the void flag + // in order to force rebuilding of the tags string. + if (!mTags.IsVoid()) { + // If mTags is assigned by a history query it is unsorted for performance + // reasons, it must be sorted by name on first read access. + if (!mAreTagsSorted) { + nsTArray tags; + ParseString(NS_ConvertUTF16toUTF8(mTags), ',', tags); + tags.Sort(); + mTags.SetIsVoid(true); + for (nsTArray::index_type i = 0; i < tags.Length(); ++i) { + AppendUTF8toUTF16(tags[i], mTags); + if (i < tags.Length() - 1) mTags.AppendLiteral(", "); + } + mAreTagsSorted = true; + } + aTags.Assign(mTags); + return NS_OK; + } + + // Fetch the tags + RefPtr DB = Database::GetDatabase(); + NS_ENSURE_STATE(DB); + nsCOMPtr stmt = DB->GetStatement( + "/* do not warn (bug 487594) */ " + "SELECT GROUP_CONCAT(tag_title, ', ') " + "FROM ( " + "SELECT t.title AS tag_title " + "FROM moz_bookmarks b " + "JOIN moz_bookmarks t ON t.id = +b.parent " + "WHERE b.fk = (SELECT id FROM moz_places WHERE url_hash = " + "hash(:page_url) AND url = :page_url) " + "AND t.parent = :tags_folder " + "ORDER BY t.title COLLATE NOCASE ASC " + ") "); + NS_ENSURE_STATE(stmt); + mozStorageStatementScoper scoper(stmt); + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_STATE(history); + nsresult rv = + stmt->BindInt64ByName("tags_folder"_ns, history->GetTagsFolder()); + NS_ENSURE_SUCCESS(rv, rv); + rv = URIBinder::Bind(stmt, "page_url"_ns, mURI); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasTags = false; + if (NS_SUCCEEDED(stmt->ExecuteStep(&hasTags)) && hasTags) { + rv = stmt->GetString(0, mTags); + NS_ENSURE_SUCCESS(rv, rv); + aTags.Assign(mTags); + mAreTagsSorted = true; + } + + // If this node is a child of a history query, we need to make sure changes + // to tags are properly live-updated. + if (mParent && mParent->IsQuery() && + mParent->mOptions->QueryType() == + nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY) { + nsNavHistoryQueryResultNode* query = mParent->GetAsQuery(); + nsNavHistoryResult* result = query->GetResult(); + NS_ENSURE_STATE(result); + result->AddAllBookmarksObserver(query); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetPageGuid(nsACString& aPageGuid) { + aPageGuid = mPageGuid; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetBookmarkGuid(nsACString& aBookmarkGuid) { + aBookmarkGuid = mBookmarkGuid; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetVisitId(int64_t* aVisitId) { + *aVisitId = mVisitId; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetFromVisitId(int64_t* aFromVisitId) { + *aFromVisitId = mFromVisitId; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResultNode::GetVisitType(uint32_t* aVisitType) { + *aVisitType = mTransitionType; + return NS_OK; +} + +void nsNavHistoryResultNode::OnRemoving() { mParent = nullptr; } + +/** + * This will find the result for this node. We can ask the nearest container + * for this value (either ourselves or our parents should be a container, + * and all containers have result pointers). + * + * @note The result may be null, if the container is detached from the result + * who owns it. + */ +nsNavHistoryResult* nsNavHistoryResultNode::GetResult() { + nsNavHistoryResultNode* node = this; + do { + if (node->IsContainer()) { + nsNavHistoryContainerResultNode* container = TO_CONTAINER(node); + return container->mResult; + } + node = node->mParent; + } while (node); + MOZ_ASSERT(false, "No container node found in hierarchy!"); + return nullptr; +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(nsNavHistoryContainerResultNode, + nsNavHistoryResultNode, mResult, mChildren) + +NS_IMPL_ADDREF_INHERITED(nsNavHistoryContainerResultNode, + nsNavHistoryResultNode) +NS_IMPL_RELEASE_INHERITED(nsNavHistoryContainerResultNode, + nsNavHistoryResultNode) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsNavHistoryContainerResultNode) + NS_INTERFACE_MAP_STATIC_AMBIGUOUS(nsNavHistoryContainerResultNode) + NS_INTERFACE_MAP_ENTRY(nsINavHistoryContainerResultNode) +NS_INTERFACE_MAP_END_INHERITING(nsNavHistoryResultNode) + +nsNavHistoryContainerResultNode::nsNavHistoryContainerResultNode( + const nsACString& aURI, const nsACString& aTitle, PRTime aTime, + uint32_t aContainerType, nsNavHistoryQueryOptions* aOptions) + : nsNavHistoryResultNode(aURI, aTitle, 0, aTime), + mResult(nullptr), + mContainerType(aContainerType), + mExpanded(false), + mOptions(aOptions), + mAsyncCanceledState(NOT_CANCELED) { + MOZ_ASSERT(mOptions); + MOZ_ALWAYS_SUCCEEDS(mOptions->Clone(getter_AddRefs(mOriginalOptions))); +} + +nsNavHistoryContainerResultNode::~nsNavHistoryContainerResultNode() { + // Explicitly clean up array of children of this container. We must ensure + // all references are gone and all of their destructors are called. + mChildren.Clear(); +} + +/** + * Containers should notify their children that they are being removed when the + * container is being removed. + */ +void nsNavHistoryContainerResultNode::OnRemoving() { + nsNavHistoryResultNode::OnRemoving(); + for (int32_t i = 0; i < mChildren.Count(); ++i) mChildren[i]->OnRemoving(); + mChildren.Clear(); + mResult = nullptr; +} + +bool nsNavHistoryContainerResultNode::AreChildrenVisible() { + nsNavHistoryResult* result = GetResult(); + if (!result) { + MOZ_ASSERT_UNREACHABLE("Invalid result"); + return false; + } + + if (!mExpanded) return false; + + // Now check if any ancestor is closed. + nsNavHistoryContainerResultNode* ancestor = mParent; + while (ancestor) { + if (!ancestor->mExpanded) return false; + + ancestor = ancestor->mParent; + } + + return true; +} + +nsresult nsNavHistoryContainerResultNode::OnVisitsRemoved(nsIURI* aURI) { + if (!AreChildrenVisible()) { + return NS_OK; + } + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->CanSkipHistoryDetailsNotifications()) { + return NS_OK; + } + + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMArray nodes; + FindChildrenByURI(spec, &nodes); + for (int32_t i = 0; i < nodes.Count(); i++) { + nodes[i]->OnVisitsRemoved(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetContainerOpen(bool* aContainerOpen) { + *aContainerOpen = mExpanded; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryContainerResultNode::SetContainerOpen(bool aContainerOpen) { + if (aContainerOpen) { + if (!mExpanded) { + if (mOptions->AsyncEnabled()) + OpenContainerAsync(); + else + OpenContainer(); + } + } else { + if (mExpanded) + CloseContainer(); + else if (mAsyncPendingStmt) + CancelAsyncOpen(false); + } + + return NS_OK; +} + +/** + * Notifies the result's observers of a change in the container's state. The + * notification includes both the old and new states: The old is aOldState, and + * the new is the container's current state. + * + * @param aOldState + * The state being transitioned out of. + */ +nsresult nsNavHistoryContainerResultNode::NotifyOnStateChange( + uint16_t aOldState) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + + nsresult rv; + uint16_t currState; + rv = GetState(&currState); + NS_ENSURE_SUCCESS(rv, rv); + + // Notify via the new ContainerStateChanged observer method. + NOTIFY_RESULT_OBSERVERS(result, + ContainerStateChanged(this, aOldState, currState)); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetState(uint16_t* _state) { + NS_ENSURE_ARG_POINTER(_state); + + *_state = mExpanded ? (uint16_t)STATE_OPENED + : mAsyncPendingStmt ? (uint16_t)STATE_LOADING + : (uint16_t)STATE_CLOSED; + + return NS_OK; +} + +/** + * This handles the generic container case. Other container types should + * override this to do their own handling. + */ +nsresult nsNavHistoryContainerResultNode::OpenContainer() { + NS_ASSERTION(!mExpanded, "Container must not be expanded to open it"); + mExpanded = true; + + nsresult rv = NotifyOnStateChange(STATE_CLOSED); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/** + * Unset aSuppressNotifications to notify observers on this change. That is + * the normal operation. This is set to false for the recursive calls since the + * root container that is being closed will handle recomputation of the visible + * elements for its entire subtree. + */ +nsresult nsNavHistoryContainerResultNode::CloseContainer( + bool aSuppressNotifications) { + NS_ASSERTION( + (mExpanded && !mAsyncPendingStmt) || (!mExpanded && mAsyncPendingStmt), + "Container must be expanded or loading to close it"); + + nsresult rv; + uint16_t oldState; + rv = GetState(&oldState); + NS_ENSURE_SUCCESS(rv, rv); + + if (mExpanded) { + // Recursively close all child containers. + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->IsContainer() && + mChildren[i]->GetAsContainer()->mExpanded) + mChildren[i]->GetAsContainer()->CloseContainer(true); + } + + mExpanded = false; + } + + // Be sure to set this to null before notifying observers. It signifies that + // the container is no longer loading (if it was in the first place). + mAsyncPendingStmt = nullptr; + + if (!aSuppressNotifications) { + rv = NotifyOnStateChange(oldState); + NS_ENSURE_SUCCESS(rv, rv); + } + + // If this is the root container of a result, we can tell the result to stop + // observing changes, otherwise the result will stay in memory and updates + // itself till it is cycle collected. + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->mRootNode == this) { + result->StopObserving(); + // When reopening this node its result will be out of sync. + // We must clear our children to ensure we will call FillChildren + // again in such a case. + if (this->IsQuery()) + this->GetAsQuery()->ClearChildren(true); + else if (this->IsFolder()) + this->GetAsFolder()->ClearChildren(true); + } + + return NS_OK; +} + +/** + * The async version of OpenContainer. + */ +nsresult nsNavHistoryContainerResultNode::OpenContainerAsync() { + return NS_ERROR_NOT_IMPLEMENTED; +} + +/** + * Cancels the pending asynchronous Storage execution triggered by + * FillChildrenAsync, if it exists. This method doesn't do much, because after + * cancelation Storage will call this node's HandleCompletion callback, where + * the real work is done. + * + * @param aRestart + * If true, async execution will be restarted by HandleCompletion. + */ +void nsNavHistoryContainerResultNode::CancelAsyncOpen(bool aRestart) { + NS_ASSERTION(mAsyncPendingStmt, "Async execution canceled but not pending"); + + mAsyncCanceledState = aRestart ? CANCELED_RESTART_NEEDED : CANCELED; + + // Cancel will fail if the pending statement has already been canceled. + // That's OK since this method may be called multiple times, and multiple + // cancels don't harm anything. + (void)mAsyncPendingStmt->Cancel(); +} + +/** + * This builds up tree statistics from the bottom up. Call with a container + * and the indent level of that container. To init the full tree, call with + * the root container. The default indent level is -1, which is appropriate + * for the root level. + * + * CALL THIS AFTER FILLING ANY CONTAINER to update the parent and result node + * pointers, even if you don't care about visit counts and last visit dates. + */ +void nsNavHistoryContainerResultNode::FillStats() { + uint32_t accessCount = 0; + PRTime newTime = 0; + + for (int32_t i = 0; i < mChildren.Count(); ++i) { + nsNavHistoryResultNode* node = mChildren[i]; + SetAsParentOfNode(node); + accessCount += node->mAccessCount; + // this is how container nodes get sorted by date + // The container gets the most recent time of the child nodes. + if (node->mTime > newTime) newTime = node->mTime; + } + + if (mExpanded) { + mAccessCount = accessCount; + if (!IsQuery() || newTime > mTime) mTime = newTime; + } +} + +void nsNavHistoryContainerResultNode::SetAsParentOfNode( + nsNavHistoryResultNode* aNode) { + aNode->mParent = this; + aNode->mIndentLevel = mIndentLevel + 1; + if (aNode->IsContainer()) { + nsNavHistoryContainerResultNode* container = aNode->GetAsContainer(); + // Propagate some of the parent's options to this container. + if (mOptions->ExcludeItems()) { + container->mOptions->SetExcludeItems(true); + } + if (mOptions->ExcludeQueries()) { + container->mOptions->SetExcludeQueries(true); + } + if (aNode->IsFolder() && mOptions->AsyncEnabled()) { + container->mOptions->SetAsyncEnabled(true); + } + if (!mOptions->ExpandQueries()) { + container->mOptions->SetExpandQueries(false); + } + container->mResult = mResult; + container->FillStats(); + } +} + +/** + * This is used when one container changes to do a minimal update of the tree + * structure. When something changes, you want to call FillStats if necessary + * and update this container completely. Then call this function which will + * walk up the tree and fill in the previous containers. + * + * Note that you have to tell us by how much our access count changed. Our + * access count should already be set to the new value; this is used tochange + * the parents without having to re-count all their children. + * + * This does NOT update the last visit date downward. Therefore, if you are + * deleting a node that has the most recent last visit date, the parents will + * not get their last visit dates downshifted accordingly. This is a rather + * unusual case: we don't often delete things, and we usually don't even show + * the last visit date for folders. Updating would be slower because we would + * have to recompute it from scratch. + */ +nsresult nsNavHistoryContainerResultNode::ReverseUpdateStats( + int32_t aAccessCountChange) { + if (mParent) { + nsNavHistoryResult* result = GetResult(); + bool shouldNotify = + result && mParent->mParent && mParent->mParent->AreChildrenVisible(); + + uint32_t oldAccessCount = mParent->mAccessCount; + PRTime oldTime = mParent->mTime; + + mParent->mAccessCount += aAccessCountChange; + bool timeChanged = false; + if (mTime > mParent->mTime) { + timeChanged = true; + mParent->mTime = mTime; + } + + if (shouldNotify && !result->CanSkipHistoryDetailsNotifications()) { + NOTIFY_RESULT_OBSERVERS( + result, NodeHistoryDetailsChanged(TO_ICONTAINER(mParent), oldTime, + oldAccessCount)); + } + + // check sorting, the stats may have caused this node to move if the + // sorting depended on something we are changing. + uint16_t sortMode = mParent->GetSortType(); + bool sortingByVisitCount = + sortMode == nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING || + sortMode == nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING; + bool sortingByTime = + sortMode == nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING || + sortMode == nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING; + + if ((sortingByVisitCount && aAccessCountChange != 0) || + (sortingByTime && timeChanged)) { + int32_t ourIndex = mParent->FindChild(this); + NS_ASSERTION(ourIndex >= 0, "Could not find self in parent"); + if (ourIndex >= 0) EnsureItemPosition(static_cast(ourIndex)); + } + + nsresult rv = mParent->ReverseUpdateStats(aAccessCountChange); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +/** + * This walks up the tree until we find a query result node or the root to get + * the sorting type. + */ +uint16_t nsNavHistoryContainerResultNode::GetSortType() { + if (mParent) return mParent->GetSortType(); + if (mResult) return mResult->mSortingMode; + + // This is a detached container, just use natural order. + return nsINavHistoryQueryOptions::SORT_BY_NONE; +} + +nsresult nsNavHistoryContainerResultNode::Refresh() { + NS_WARNING( + "Refresh() is supported by queries or folders, not generic containers."); + return NS_OK; +} + +/** + * @return the sorting comparator function for the give sort type, or null if + * there is no comparator. + */ +nsNavHistoryContainerResultNode::SortComparator +nsNavHistoryContainerResultNode::GetSortingComparator(uint16_t aSortType) { + switch (aSortType) { + case nsINavHistoryQueryOptions::SORT_BY_NONE: + return &SortComparison_Bookmark; + case nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING: + return &SortComparison_TitleLess; + case nsINavHistoryQueryOptions::SORT_BY_TITLE_DESCENDING: + return &SortComparison_TitleGreater; + case nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING: + return &SortComparison_DateLess; + case nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING: + return &SortComparison_DateGreater; + case nsINavHistoryQueryOptions::SORT_BY_URI_ASCENDING: + return &SortComparison_URILess; + case nsINavHistoryQueryOptions::SORT_BY_URI_DESCENDING: + return &SortComparison_URIGreater; + case nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING: + return &SortComparison_VisitCountLess; + case nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING: + return &SortComparison_VisitCountGreater; + case nsINavHistoryQueryOptions::SORT_BY_DATEADDED_ASCENDING: + return &SortComparison_DateAddedLess; + case nsINavHistoryQueryOptions::SORT_BY_DATEADDED_DESCENDING: + return &SortComparison_DateAddedGreater; + case nsINavHistoryQueryOptions::SORT_BY_LASTMODIFIED_ASCENDING: + return &SortComparison_LastModifiedLess; + case nsINavHistoryQueryOptions::SORT_BY_LASTMODIFIED_DESCENDING: + return &SortComparison_LastModifiedGreater; + case nsINavHistoryQueryOptions::SORT_BY_TAGS_ASCENDING: + return &SortComparison_TagsLess; + case nsINavHistoryQueryOptions::SORT_BY_TAGS_DESCENDING: + return &SortComparison_TagsGreater; + case nsINavHistoryQueryOptions::SORT_BY_FRECENCY_ASCENDING: + return &SortComparison_FrecencyLess; + case nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING: + return &SortComparison_FrecencyGreater; + default: + MOZ_ASSERT_UNREACHABLE("Bad sorting type"); + return nullptr; + } +} + +/** + * This is used by Result::SetSortingMode and QueryResultNode::FillChildren to + * sort the child list. + * + * This does NOT update any visibility or tree information. The caller will + * have to completely rebuild the visible list after this. + */ +void nsNavHistoryContainerResultNode::RecursiveSort( + SortComparator aComparator) { + mChildren.Sort(aComparator, nullptr); + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->IsContainer()) + mChildren[i]->GetAsContainer()->RecursiveSort(aComparator); + } +} + +/** + * @return the index that the given item would fall on if it were to be + * inserted using the given sorting. + */ +uint32_t nsNavHistoryContainerResultNode::FindInsertionPoint( + nsNavHistoryResultNode* aNode, SortComparator aComparator, + bool* aItemExists) { + if (aItemExists) (*aItemExists) = false; + + if (mChildren.Count() == 0) return 0; + + // The common case is the beginning or the end because this is used to insert + // new items that are added to history, which is usually sorted by date. + int32_t res; + res = aComparator(aNode, mChildren[0], nullptr); + if (res <= 0) { + if (aItemExists && res == 0) (*aItemExists) = true; + return 0; + } + res = aComparator(aNode, mChildren[mChildren.Count() - 1], nullptr); + if (res >= 0) { + if (aItemExists && res == 0) (*aItemExists) = true; + return mChildren.Count(); + } + + uint32_t beginRange = 0; // inclusive + uint32_t endRange = mChildren.Count(); // exclusive + while (1) { + if (beginRange == endRange) return endRange; + uint32_t center = beginRange + (endRange - beginRange) / 2; + int32_t res = aComparator(aNode, mChildren[center], nullptr); + if (res <= 0) { + endRange = center; // left side + if (aItemExists && res == 0) (*aItemExists) = true; + } else { + beginRange = center + 1; // right site + } + } +} + +/** + * This checks the child node at the given index to see if its sorting is + * correct. This is called when nodes are updated and we need to see whether + * we need to move it. + * + * @returns true if not and it should be resorted. + */ +bool nsNavHistoryContainerResultNode::DoesChildNeedResorting( + uint32_t aIndex, SortComparator aComparator) { + NS_ASSERTION(aIndex < uint32_t(mChildren.Count()), + "Input index out of range"); + if (mChildren.Count() == 1) return false; + + if (aIndex > 0) { + // compare to previous item + if (aComparator(mChildren[aIndex - 1], mChildren[aIndex], nullptr) > 0) + return true; + } + if (aIndex < uint32_t(mChildren.Count()) - 1) { + // compare to next item + if (aComparator(mChildren[aIndex], mChildren[aIndex + 1], nullptr) > 0) + return true; + } + return false; +} + +/* static */ +int32_t nsNavHistoryContainerResultNode::SortComparison_StringLess( + const nsAString& a, const nsAString& b) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, 0); + const mozilla::intl::Collator* collator = history->GetCollator(); + NS_ENSURE_TRUE(collator, 0); + + int32_t res = collator->CompareStrings(a, b); + return res; +} + +/** + * When there are bookmark indices, we should never have ties, so we don't + * need to worry about tiebreaking. When there are no bookmark indices, + * everything will be -1 and we don't worry about sorting. + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_Bookmark( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return a->mBookmarkIndex - b->mBookmarkIndex; +} + +/** + * These are a little more complicated because they do a localization + * conversion. If this is too slow, we can compute the sort keys once in + * advance, sort that array, and then reorder the real array accordingly. + * This would save some key generations. + * + * The collation object must be allocated before sorting on title! + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_TitleLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + uint32_t aType; + a->GetType(&aType); + + int32_t value = SortComparison_StringLess(NS_ConvertUTF8toUTF16(a->mTitle), + NS_ConvertUTF8toUTF16(b->mTitle)); + if (value == 0) { + // resolve by URI + if (a->IsURI()) { + value = Compare(a->mURI, b->mURI); + } + if (value == 0) { + // resolve by date + value = ComparePRTime(a->mTime, b->mTime); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark( + a, b, closure); + } + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_TitleGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -SortComparison_TitleLess(a, b, closure); +} + +/** + * Equal times will be very unusual, but it is important that there is some + * deterministic ordering of the results so they don't move around. + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_DateLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = ComparePRTime(a->mTime, b->mTime); + if (value == 0) { + value = SortComparison_StringLess(NS_ConvertUTF8toUTF16(a->mTitle), + NS_ConvertUTF8toUTF16(b->mTitle)); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_DateGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -nsNavHistoryContainerResultNode::SortComparison_DateLess(a, b, + closure); +} + +int32_t nsNavHistoryContainerResultNode::SortComparison_DateAddedLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = ComparePRTime(a->mDateAdded, b->mDateAdded); + if (value == 0) { + value = SortComparison_StringLess(NS_ConvertUTF8toUTF16(a->mTitle), + NS_ConvertUTF8toUTF16(b->mTitle)); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_DateAddedGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -nsNavHistoryContainerResultNode::SortComparison_DateAddedLess( + a, b, closure); +} + +int32_t nsNavHistoryContainerResultNode::SortComparison_LastModifiedLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = ComparePRTime(a->mLastModified, b->mLastModified); + if (value == 0) { + value = SortComparison_StringLess(NS_ConvertUTF8toUTF16(a->mTitle), + NS_ConvertUTF8toUTF16(b->mTitle)); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_LastModifiedGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -nsNavHistoryContainerResultNode::SortComparison_LastModifiedLess( + a, b, closure); +} + +/** + * Certain types of parent nodes are treated specially because URIs are not + * valid (like days or hosts). + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_URILess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value; + if (a->IsURI() && b->IsURI()) { + // normal URI or visit + value = Compare(a->mURI, b->mURI); + } else if (a->IsContainer() && !b->IsContainer()) { + // Containers appear before entries with a uri. + return -1; + } else if (b->IsContainer() && !a->IsContainer()) { + return 1; + } else { + // For everything else, use title sorting. + value = SortComparison_StringLess(NS_ConvertUTF8toUTF16(a->mTitle), + NS_ConvertUTF8toUTF16(b->mTitle)); + } + + if (value == 0) { + value = ComparePRTime(a->mTime, b->mTime); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_URIGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -SortComparison_URILess(a, b, closure); +} + +/** + * Fall back on dates for conflict resolution + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_VisitCountLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = CompareIntegers(a->mAccessCount, b->mAccessCount); + if (value == 0) { + value = ComparePRTime(a->mTime, b->mTime); + if (value == 0) + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_VisitCountGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -nsNavHistoryContainerResultNode::SortComparison_VisitCountLess( + a, b, closure); +} + +int32_t nsNavHistoryContainerResultNode::SortComparison_TagsLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = 0; + nsAutoString aTags, bTags; + + nsresult rv = a->GetTags(aTags); + NS_ENSURE_SUCCESS(rv, 0); + + rv = b->GetTags(bTags); + NS_ENSURE_SUCCESS(rv, 0); + + value = SortComparison_StringLess(aTags, bTags); + + // fall back to title sorting + if (value == 0) value = SortComparison_TitleLess(a, b, closure); + + return value; +} + +int32_t nsNavHistoryContainerResultNode::SortComparison_TagsGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -SortComparison_TagsLess(a, b, closure); +} + +/** + * Fall back on date and bookmarked status, for conflict resolution. + */ +int32_t nsNavHistoryContainerResultNode::SortComparison_FrecencyLess( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + int32_t value = CompareIntegers(a->mFrecency, b->mFrecency); + if (value == 0) { + value = ComparePRTime(a->mTime, b->mTime); + if (value == 0) { + value = nsNavHistoryContainerResultNode::SortComparison_Bookmark(a, b, + closure); + } + } + return value; +} +int32_t nsNavHistoryContainerResultNode::SortComparison_FrecencyGreater( + nsNavHistoryResultNode* a, nsNavHistoryResultNode* b, void* closure) { + return -nsNavHistoryContainerResultNode::SortComparison_FrecencyLess(a, b, + closure); +} + +/** + * Searches this folder for a node with the given URI. Returns null if not + * found. + * + * @note Does not addref the node! + */ +nsNavHistoryResultNode* nsNavHistoryContainerResultNode::FindChildByURI( + const nsACString& aSpec, uint32_t* aNodeIndex) { + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->IsURI()) { + if (aSpec.Equals(mChildren[i]->mURI)) { + *aNodeIndex = i; + return mChildren[i]; + } + } + } + return nullptr; +} + +/** + * Searches for matches for the given URI. + */ +void nsNavHistoryContainerResultNode::FindChildrenByURI( + const nsCString& aSpec, nsCOMArray* aMatches) { + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->IsURI()) { + if (aSpec.Equals(mChildren[i]->mURI)) { + aMatches->AppendObject(mChildren[i]); + } + } + } +} + +/** + * Searches this folder for a node with the given guid/target-folder-guid. + * + * @return the node if found, null otherwise. + * @note Does not addref the node! + */ +nsNavHistoryResultNode* nsNavHistoryContainerResultNode::FindChildByGuid( + const nsACString& guid, int32_t* nodeIndex) { + *nodeIndex = -1; + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->mBookmarkGuid == guid || + mChildren[i]->mPageGuid == guid || + (mChildren[i]->IsFolder() && + mChildren[i]->GetAsFolder()->mTargetFolderGuid == guid)) { + *nodeIndex = i; + return mChildren[i]; + } + } + return nullptr; +} + +/** + * Searches this folder for a node with the given id/target-folder-id. + * + * @return the node if found, null otherwise. + * @note Does not addref the node! + */ +nsNavHistoryResultNode* nsNavHistoryContainerResultNode::FindChildById( + int64_t aItemId, uint32_t* aNodeIndex) { + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->mItemId == aItemId || + (mChildren[i]->IsFolder() && + mChildren[i]->GetAsFolder()->mTargetFolderItemId == aItemId)) { + *aNodeIndex = i; + return mChildren[i]; + } + } + return nullptr; +} + +/** + * This does the work of adding a child to the container. The child can be + * either a container or or a single item that may even be collapsed with the + * adjacent ones. + */ +nsresult nsNavHistoryContainerResultNode::InsertChildAt( + nsNavHistoryResultNode* aNode, int32_t aIndex) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + + SetAsParentOfNode(aNode); + + if (!mChildren.InsertObjectAt(aNode, aIndex)) return NS_ERROR_OUT_OF_MEMORY; + + // Update our stats and notify the result's observers. + uint32_t oldAccessCount = mAccessCount; + PRTime oldTime = mTime; + + mAccessCount += aNode->mAccessCount; + if (mTime < aNode->mTime) mTime = aNode->mTime; + if ((!mParent || mParent->AreChildrenVisible()) && + !result->CanSkipHistoryDetailsNotifications()) { + NOTIFY_RESULT_OBSERVERS( + result, NodeHistoryDetailsChanged(TO_ICONTAINER(this), oldTime, + oldAccessCount)); + } + + nsresult rv = ReverseUpdateStats(aNode->mAccessCount); + NS_ENSURE_SUCCESS(rv, rv); + + // Update tree if we are visible. Note that we could be here and not + // expanded, like when there is a bookmark folder being updated because its + // parent is visible. + if (AreChildrenVisible()) + NOTIFY_RESULT_OBSERVERS(result, NodeInserted(this, aNode, aIndex)); + + return NS_OK; +} + +/** + * This locates the proper place for insertion according to the current sort + * and calls InsertChildAt + */ +nsresult nsNavHistoryContainerResultNode::InsertSortedChild( + nsNavHistoryResultNode* aNode, bool aIgnoreDuplicates) { + if (mChildren.Count() == 0) return InsertChildAt(aNode, 0); + + SortComparator comparator = GetSortingComparator(GetSortType()); + if (comparator) { + // When inserting a new node, it must have proper statistics because we use + // them to find the correct insertion point. The insert function will then + // recompute these statistics and fill in the proper parents and hierarchy + // level. Doing this twice shouldn't be a large performance penalty because + // when we are inserting new containers, they typically contain only one + // item (because we've browsed a new page). + if (aNode->IsContainer()) { + // need to update all the new item's children + nsNavHistoryContainerResultNode* container = aNode->GetAsContainer(); + container->mResult = mResult; + container->FillStats(); + } + + bool itemExists; + uint32_t position = FindInsertionPoint(aNode, comparator, &itemExists); + if (aIgnoreDuplicates && itemExists) return NS_OK; + + return InsertChildAt(aNode, position); + } + return InsertChildAt(aNode, mChildren.Count()); +} + +/** + * This checks if the item at aIndex is located correctly given the sorting + * move. If it's not, the item is moved, and the result's observers are + * notified. + * + * @return true if the item position has been changed, false otherwise. + */ +bool nsNavHistoryContainerResultNode::EnsureItemPosition(uint32_t aIndex) { + NS_ASSERTION(aIndex < (uint32_t)mChildren.Count(), "Invalid index"); + if (aIndex >= (uint32_t)mChildren.Count()) return false; + + SortComparator comparator = GetSortingComparator(GetSortType()); + if (!comparator) return false; + + if (!DoesChildNeedResorting(aIndex, comparator)) return false; + + RefPtr node(mChildren[aIndex]); + mChildren.RemoveObjectAt(aIndex); + + uint32_t newIndex = FindInsertionPoint(node, comparator, nullptr); + mChildren.InsertObjectAt(node.get(), newIndex); + + if (AreChildrenVisible()) { + nsNavHistoryResult* result = GetResult(); + NOTIFY_RESULT_OBSERVERS_RET( + result, NodeMoved(node, this, aIndex, this, newIndex), false); + } + + return true; +} + +/** + * This does all the work of removing a child from this container, including + * updating the tree if necessary. Note that we do not need to be open for + * this to work. + */ +nsresult nsNavHistoryContainerResultNode::RemoveChildAt(int32_t aIndex) { + NS_ASSERTION(aIndex >= 0 && aIndex < mChildren.Count(), "Invalid index"); + + // Hold an owning reference to keep from expiring while we work with it. + RefPtr oldNode = mChildren[aIndex]; + + // Update stats. + // XXX This assertion does not reliably pass -- investigate!! (bug 1049797) + // MOZ_ASSERT(mAccessCount >= mChildren[aIndex]->mAccessCount, + // "Invalid access count while updating!"); + uint32_t oldAccessCount = mAccessCount; + mAccessCount -= mChildren[aIndex]->mAccessCount; + + // Remove it from our list and notify the result's observers. + mChildren.RemoveObjectAt(aIndex); + if (AreChildrenVisible()) { + nsNavHistoryResult* result = GetResult(); + NOTIFY_RESULT_OBSERVERS(result, NodeRemoved(this, oldNode, aIndex)); + } + + nsresult rv = ReverseUpdateStats(mAccessCount - oldAccessCount); + NS_ENSURE_SUCCESS(rv, rv); + oldNode->OnRemoving(); + return NS_OK; +} + +/** + * Searches for matches for the given URI. If aOnlyOne is set, it will + * terminate as soon as it finds a single match. This would be used when there + * are URI results so there will only ever be one copy of any URI. + * + * When aOnlyOne is false, it will check all elements. This is for non-history + * or visit style results that may have multiple copies of any given URI. + */ +void nsNavHistoryContainerResultNode::RecursiveFindURIs( + bool aOnlyOne, nsNavHistoryContainerResultNode* aContainer, + const nsCString& aSpec, nsCOMArray* aMatches) { + for (int32_t i = 0; i < aContainer->mChildren.Count(); ++i) { + auto* node = aContainer->mChildren[i]; + if (node->IsURI()) { + if (node->mURI.Equals(aSpec)) { + aMatches->AppendObject(node); + if (aOnlyOne) { + return; + } + } + } else if (node->IsContainer() && node->GetAsContainer()->mExpanded) { + RecursiveFindURIs(aOnlyOne, node->GetAsContainer(), aSpec, aMatches); + } + } +} + +/** + * If aUpdateSort is true, we will also update the sorting of this item. + * Normally you want this to be true, but it can be false if the thing you are + * changing can not affect sorting (like favicons). + * + * You should NOT change any child lists as part of the callback function. + */ +bool nsNavHistoryContainerResultNode::UpdateURIs( + bool aRecursive, bool aOnlyOne, bool aUpdateSort, const nsCString& aSpec, + nsresult (*aCallback)(nsNavHistoryResultNode*, const void*, + const nsNavHistoryResult*), + const void* aClosure) { + const nsNavHistoryResult* result = GetResult(); + if (!result) { + MOZ_ASSERT(false, "Should have a result"); + return false; + } + + // this needs to be owning since sometimes we remove and re-insert nodes + // in their parents and we don't want them to go away. + nsCOMArray matches; + + if (aRecursive) { + RecursiveFindURIs(aOnlyOne, this, aSpec, &matches); + } else if (aOnlyOne) { + uint32_t nodeIndex; + nsNavHistoryResultNode* node = FindChildByURI(aSpec, &nodeIndex); + if (node) matches.AppendObject(node); + } else { + MOZ_ASSERT( + false, + "UpdateURIs does not handle nonrecursive updates of multiple items."); + // this case easy to add if you need it, just find all the matching URIs + // at this level. However, this isn't currently used. History uses + // recursive, Bookmarks uses one level and knows that the match is unique. + return false; + } + + if (matches.Count() == 0) return false; + + // PERFORMANCE: This updates each container for each child in it that + // changes. In some cases, many elements have changed inside the same + // container. It would be better to compose a list of containers, and + // update each one only once for all the items that have changed in it. + for (int32_t i = 0; i < matches.Count(); ++i) { + nsNavHistoryResultNode* node = matches[i]; + nsNavHistoryContainerResultNode* parent = node->mParent; + if (!parent) { + MOZ_ASSERT(false, "All URI nodes being updated must have parents"); + continue; + } + + uint32_t oldAccessCount = node->mAccessCount; + PRTime oldTime = node->mTime; + uint32_t parentOldAccessCount = parent->mAccessCount; + PRTime parentOldTime = parent->mTime; + + aCallback(node, aClosure, result); + + if (oldAccessCount != node->mAccessCount || oldTime != node->mTime) { + parent->mAccessCount += node->mAccessCount - oldAccessCount; + if (node->mTime > parent->mTime) parent->mTime = node->mTime; + if (parent->AreChildrenVisible() && + !result->CanSkipHistoryDetailsNotifications()) { + NOTIFY_RESULT_OBSERVERS_RET( + result, + NodeHistoryDetailsChanged(TO_ICONTAINER(parent), parentOldTime, + parentOldAccessCount), + true); + } + DebugOnly rv = + parent->ReverseUpdateStats(node->mAccessCount - oldAccessCount); + MOZ_ASSERT(NS_SUCCEEDED(rv), "should be able to ReverseUpdateStats"); + } + + if (aUpdateSort) { + int32_t childIndex = parent->FindChild(node); + MOZ_ASSERT(childIndex >= 0, + "Could not find child we just got a reference to"); + if (childIndex >= 0) parent->EnsureItemPosition(childIndex); + } + } + + return true; +} + +/** + * This is used to update the titles in the tree. This is called from both + * query and bookmark folder containers to update the tree. Bookmark folders + * should be sure to set recursive to false, since child folders will have + * their own callbacks registered. + */ +static nsresult setTitleCallback(nsNavHistoryResultNode* aNode, + const void* aClosure, + const nsNavHistoryResult* aResult) { + const nsACString* newTitle = static_cast(aClosure); + aNode->mTitle = *newTitle; + + if (aResult && (!aNode->mParent || aNode->mParent->AreChildrenVisible())) + NOTIFY_RESULT_OBSERVERS(aResult, NodeTitleChanged(aNode, *newTitle)); + + return NS_OK; +} +nsresult nsNavHistoryContainerResultNode::ChangeTitles( + nsIURI* aURI, const nsACString& aNewTitle, bool aRecursive, bool aOnlyOne) { + // uri string + nsAutoCString uriString; + nsresult rv = aURI->GetSpec(uriString); + NS_ENSURE_SUCCESS(rv, rv); + + // The recursive function will update the result's tree nodes, but only if we + // give it a non-null pointer. So if there isn't a tree, just pass nullptr + // so it doesn't bother trying to call the result. + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + + uint16_t sortType = GetSortType(); + bool updateSorting = + (sortType == nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_TITLE_DESCENDING); + + UpdateURIs(aRecursive, aOnlyOne, updateSorting, uriString, setTitleCallback, + static_cast(&aNewTitle)); + + return NS_OK; +} + +/** + * Complex containers (folders and queries) will override this. Here, we + * handle the case of simple containers (like host groups) where the children + * are always stored. + */ +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetHasChildren(bool* aHasChildren) { + *aHasChildren = (mChildren.Count() > 0); + return NS_OK; +} + +/** + * @throws if this node is closed. + */ +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetChildCount(uint32_t* aChildCount) { + if (!mExpanded) return NS_ERROR_NOT_AVAILABLE; + *aChildCount = mChildren.Count(); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetChild(uint32_t aIndex, + nsINavHistoryResultNode** _child) { + if (!mExpanded) return NS_ERROR_NOT_AVAILABLE; + if (aIndex >= uint32_t(mChildren.Count())) return NS_ERROR_INVALID_ARG; + nsCOMPtr child = mChildren[aIndex]; + child.forget(_child); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryContainerResultNode::GetChildIndex(nsINavHistoryResultNode* aNode, + uint32_t* _retval) { + if (!mExpanded) return NS_ERROR_NOT_AVAILABLE; + + int32_t nodeIndex = FindChild(static_cast(aNode)); + if (nodeIndex == -1) return NS_ERROR_INVALID_ARG; + + *_retval = nodeIndex; + return NS_OK; +} + +/** + * HOW QUERY UPDATING WORKS + * + * Queries are different than bookmark folders in that we can not always do + * dynamic updates (easily) and updates are more expensive. Therefore, we do + * NOT query if we are not open and want to see if we have any children (for + * drawing a twisty) and always assume we will. + * + * When the container is opened, we execute the query and register the + * listeners. Like bookmark folders, we stay registered even when closed, and + * clear ourselves as soon as a message comes in. This lets us respond quickly + * if the user closes and reopens the container. + * + * We try to handle the most common notifications for the most common query + * types dynamically, that is, figuring out what should happen in response to + * a message without doing a requery. For complex changes or complex queries, + * we give up and requery. + */ +NS_IMPL_ISUPPORTS_INHERITED(nsNavHistoryQueryResultNode, + nsNavHistoryContainerResultNode, + nsINavHistoryQueryResultNode) + +nsNavHistoryQueryResultNode::nsNavHistoryQueryResultNode( + const nsACString& aTitle, PRTime aTime, const nsACString& aQueryURI, + const RefPtr& aQuery, + const RefPtr& aOptions) + : nsNavHistoryContainerResultNode(aQueryURI, aTitle, aTime, + nsNavHistoryResultNode::RESULT_TYPE_QUERY, + aOptions), + mQuery(aQuery), + mLiveUpdate(getUpdateRequirements(aQuery, aOptions, &mHasSearchTerms)), + mContentsValid(false), + mBatchChanges(0), + mTransitions(aQuery->Transitions().Clone()) {} + +nsNavHistoryQueryResultNode::~nsNavHistoryQueryResultNode() { + // Remove this node from result's observers. We don't need to be notified + // anymore. + if (mResult && mResult->mAllBookmarksObservers.Contains(this)) + mResult->RemoveAllBookmarksObserver(this); + if (mResult && mResult->mHistoryObservers.Contains(this)) + mResult->RemoveHistoryObserver(this); + if (mResult && mResult->mMobilePrefObservers.Contains(this)) + mResult->RemoveMobilePrefsObserver(this); +} + +/** + * Whoever made us may want non-expanding queries. However, we always expand + * when we are the root node, or else asking for non-expanding queries would be + * useless. A query node is not expandable if excludeItems is set or if + * expandQueries is unset. + */ +bool nsNavHistoryQueryResultNode::CanExpand() { + // The root node and containersQueries can always expand; + if ((mResult && mResult->mRootNode == this) || IsContainersQuery()) { + return true; + } + + if (mOptions->ExcludeItems()) { + return false; + } + if (mOptions->ExpandQueries()) { + return true; + } + + return false; +} + +/** + * Some query with a particular result type can contain other queries. They + * must be always expandable + */ +bool nsNavHistoryQueryResultNode::IsContainersQuery() { + uint16_t resultType = Options()->ResultType(); + return resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_SITE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_LEFT_PANE_QUERY; +} + +/** + * Here we do not want to call ContainerResultNode::OnRemoving since our own + * ClearChildren will do the same thing and more (unregister the observers). + * The base ResultNode::OnRemoving will clear some regular node stats, so it + * is OK. + */ +void nsNavHistoryQueryResultNode::OnRemoving() { + nsNavHistoryResultNode::OnRemoving(); + ClearChildren(true); + mResult = nullptr; +} + +/** + * Marks the container as open, rebuilding results if they are invalid. We + * may still have valid results if the container was previously open and + * nothing happened since closing it. + * + * We do not handle CloseContainer specially. The default one just marks the + * container as closed, but doesn't actually mark the results as invalid. + * The results will be invalidated by the next history or bookmark + * notification that comes in. This means if you open and close the item + * without anything happening in between, it will be fast (this actually + * happens when results are used as menus). + */ +nsresult nsNavHistoryQueryResultNode::OpenContainer() { + NS_ASSERTION(!mExpanded, "Container must be closed to open it"); + mExpanded = true; + + nsresult rv; + + if (!CanExpand()) return NS_OK; + if (!mContentsValid) { + rv = FillChildren(); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = NotifyOnStateChange(STATE_CLOSED); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/** + * When we have valid results we can always give an exact answer. When we + * don't we just assume we'll have results, since actually doing the query + * might be hard. This is used to draw twisties on the tree, so precise results + * don't matter. + */ +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetHasChildren(bool* aHasChildren) { + *aHasChildren = false; + + if (!CanExpand()) { + return NS_OK; + } + + uint16_t resultType = mOptions->ResultType(); + + // Tags are always populated, otherwise they are removed. + if (mQuery->Tags().Length() == 1 && mParent && + mParent->mOptions->ResultType() == + nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT) { + *aHasChildren = true; + return NS_OK; + } + + // AllBookmarks and the left pane folder also always have children. + if (resultType == nsINavHistoryQueryOptions::RESULTS_AS_ROOTS_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_LEFT_PANE_QUERY) { + *aHasChildren = true; + return NS_OK; + } + + // For history containers query we must check if we have any history + if (resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_SITE_QUERY) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + *aHasChildren = history->hasHistoryEntries(); + return NS_OK; + } + + // TODO (Bug 1477934): We don't have a good synchronous way to fetch whether + // we have tags or not, to properly reply to the hasChildren request on the + // tags root. Potentially we could pass this information when we create the + // container. + + // If the container is open and populated, this is trivial. + if (mContentsValid) { + *aHasChildren = (mChildren.Count() > 0); + return NS_OK; + } + + // Fallback to assume we have children. + *aHasChildren = true; + return NS_OK; +} + +/** + * This doesn't just return mURI because in the case of queries that may + * be lazily constructed from the query objects. + */ +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetUri(nsACString& aURI) { + aURI = mURI; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetFolderItemId(int64_t* aItemId) { + *aItemId = -1; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetTargetFolderGuid(nsACString& aGuid) { + aGuid.Truncate(); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetQuery(nsINavHistoryQuery** _query) { + RefPtr query = mQuery; + query.forget(_query); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryQueryResultNode::GetQueryOptions( + nsINavHistoryQueryOptions** _options) { + MOZ_ASSERT(mOptions, "Options should be valid"); + RefPtr options = mOptions; + options.forget(_options); + return NS_OK; +} + +/** + * Safe options getter, ensures query is parsed first. + */ +nsNavHistoryQueryOptions* nsNavHistoryQueryResultNode::Options() { + MOZ_ASSERT(mOptions, "Options invalid, cannot generate from URI"); + return mOptions; +} + +nsresult nsNavHistoryQueryResultNode::FillChildren() { + MOZ_ASSERT(!mContentsValid, + "Don't call FillChildren when contents are valid"); + MOZ_ASSERT(mChildren.Count() == 0, + "We are trying to fill children when there already are some"); + NS_ENSURE_STATE(mQuery && mOptions); + + // get the results from the history service + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + nsresult rv = history->GetQueryResults(this, mQuery, mOptions, &mChildren); + NS_ENSURE_SUCCESS(rv, rv); + + // it is important to call FillStats to fill in the parents on all + // nodes and the result node pointers on the containers + FillStats(); + + uint16_t sortType = GetSortType(); + + if (mResult && mResult->mNeedsToApplySortingMode) { + // We should repopulate container and then apply sortingMode. To avoid + // sorting 2 times we simply do that here. + mResult->SetSortingMode(mResult->mSortingMode); + } else if (mOptions->QueryType() != + nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY || + sortType != nsINavHistoryQueryOptions::SORT_BY_NONE) { + // The default SORT_BY_NONE sorts by the bookmark index (position), + // which we do not have for history queries. + // Once we've computed all tree stats, we can sort, because containers will + // then have proper visit counts and dates. + SortComparator comparator = GetSortingComparator(GetSortType()); + if (comparator) { + // Usually containers queries results comes already sorted from the + // database, but some locales could have special rules to sort by title. + // RecursiveSort won't apply these rules to containers in containers + // queries because when setting sortingMode on the result we want to sort + // contained items (bug 473157). + // Base container RecursiveSort will sort both our children and all + // descendants, and is used in this case because we have to do manual + // title sorting. + // Query RecursiveSort will instead only sort descendants if we are a + // constinaersQuery, e.g. a grouped query that will return other queries. + // For other type of queries it will act as the base one. + if (IsContainersQuery() && sortType == mOptions->SortingMode() && + (sortType == nsINavHistoryQueryOptions::SORT_BY_TITLE_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_TITLE_DESCENDING)) { + nsNavHistoryContainerResultNode::RecursiveSort(comparator); + } else { + RecursiveSort(comparator); + } + } + } + + // if we are limiting our results remove items from the end of the + // mChildren array after sorting. This is done for root node only. + // note, if count < max results, we won't do anything. + if (!mParent && mOptions->MaxResults()) { + while ((uint32_t)mChildren.Count() > mOptions->MaxResults()) + mChildren.RemoveObjectAt(mChildren.Count() - 1); + } + + // If we're not updating the query, we don't need to add listeners, so bail + // out early. + if (mLiveUpdate == QUERYUPDATE_NONE) { + mContentsValid = true; + return NS_OK; + } + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + + // Ensure to add history observer before bookmarks observer, because the + // latter wants to know if an history observer was added. + + if (mOptions->QueryType() == nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY || + mOptions->QueryType() == nsINavHistoryQueryOptions::QUERY_TYPE_UNIFIED) { + // Date containers that contain site containers have no reason to observe + // history, if the inside site container is expanded it will update, + // otherwise we are going to refresh the parent query. + if (!mParent || mParent->mOptions->ResultType() != + nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY) { + // register with the result for history updates + result->AddHistoryObserver(this); + } + } + + if (mOptions->QueryType() == + nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS || + mOptions->QueryType() == nsINavHistoryQueryOptions::QUERY_TYPE_UNIFIED || + mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS || mHasSearchTerms) { + // register with the result for bookmark updates + result->AddAllBookmarksObserver(this); + } + + if (mLiveUpdate == QUERYUPDATE_MOBILEPREF) { + result->AddMobilePrefsObserver(this); + } + + mContentsValid = true; + return NS_OK; +} + +/** + * Call with unregister = false when we are going to update the children (for + * example, when the container is open). This will clear the list and notify + * all the children that they are going away. + * + * When the results are becoming invalid and we are not going to refresh them, + * set unregister = true, which will unregister the listener from the + * result if any. We use unregister = false when we are refreshing the list + * immediately so want to stay a notifier. + */ +void nsNavHistoryQueryResultNode::ClearChildren(bool aUnregister) { + for (int32_t i = 0; i < mChildren.Count(); ++i) mChildren[i]->OnRemoving(); + mChildren.Clear(); + + if (aUnregister && mContentsValid) { + nsNavHistoryResult* result = GetResult(); + if (result) { + result->RemoveHistoryObserver(this); + result->RemoveAllBookmarksObserver(this); + result->RemoveMobilePrefsObserver(this); + } + } + mContentsValid = false; +} + +/** + * This is called to update the result when something has changed that we + * can not incrementally update. + */ +nsresult nsNavHistoryQueryResultNode::Refresh() { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->IsBatching()) { + result->requestRefresh(this); + return NS_OK; + } + + // This is not a root node but it does not have a parent - this means that + // the node has already been cleared and it is now called, because it was + // left in a local copy of the observers array. + if (mIndentLevel > -1 && !mParent) return NS_OK; + + // Do not refresh if we are not expanded or if we are child of a query + // containing other queries. In this case calling Refresh for each child + // query could cause a major slowdown. We should not refresh nested + // queries, since we will already refresh the parent one. + // The only exception to this, is if the parent query is of QUERYUPDATE_NONE, + // this can be the case for the RESULTS_AS_TAGS_ROOT + // under RESULTS_AS_LEFT_PANE_QUERY. + if (!mExpanded) { + ClearChildren(true); + return NS_OK; + } + + if (mParent && mParent->IsQuery()) { + nsNavHistoryQueryResultNode* parent = mParent->GetAsQuery(); + if (parent->IsContainersQuery() && + parent->mLiveUpdate != QUERYUPDATE_NONE) { + // Don't update, just invalidate and unhook + ClearChildren(true); + return NS_OK; // no updates in tree state + } + } + + if (mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS) + ClearChildren(true); + else + ClearChildren(false); + + // Ignore errors from FillChildren, since we will still want to refresh + // the tree (there just might not be anything in it on error). + (void)FillChildren(); + + NOTIFY_RESULT_OBSERVERS(result, InvalidateContainer(TO_CONTAINER(this))); + return NS_OK; +} + +/** + * Here, we override GetSortType to return the current sorting for this + * query. GetSortType is used when dynamically inserting query results so we + * can see which comparator we should use to find the proper insertion point + * (it shouldn't be called from folder containers which maintain their own + * sorting). + * + * Normally, the container just forwards it up the chain. This is what we want + * for host groups, for example. For queries, we often want to use the query's + * sorting mode. + * + * However, we only use this query node's sorting when it is not the root. + * When it is the root, we use the result's sorting mode. This is because + * there are two cases: + * - You are looking at a bookmark hierarchy that contains an embedded + * result. We should always use the query's sort ordering since the result + * node's headers have nothing to do with us (and are disabled). + * - You are looking at a query in the tree. In this case, we want the + * result sorting to override ours (it should be initialized to the same + * sorting mode). + */ +uint16_t nsNavHistoryQueryResultNode::GetSortType() { + if (mParent) return mOptions->SortingMode(); + if (mResult) return mResult->mSortingMode; + + // This is a detached container, just use natural order. + return nsINavHistoryQueryOptions::SORT_BY_NONE; +} + +void nsNavHistoryQueryResultNode::RecursiveSort(SortComparator aComparator) { + if (!IsContainersQuery()) mChildren.Sort(aComparator, nullptr); + + for (int32_t i = 0; i < mChildren.Count(); ++i) { + if (mChildren[i]->IsContainer()) + mChildren[i]->GetAsContainer()->RecursiveSort(aComparator); + } +} + +nsresult nsNavHistoryQueryResultNode::OnBeginUpdateBatch() { return NS_OK; } + +nsresult nsNavHistoryQueryResultNode::OnEndUpdateBatch() { + // If the query has no children it's possible it's not yet listening to + // bookmarks changes, in such a case it's safer to force a refresh to gather + // eventual new nodes matching query options. + if (mChildren.Count() == 0) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + } + + mBatchChanges = 0; + return NS_OK; +} + +static nsresult setHistoryDetailsCallback(nsNavHistoryResultNode* aNode, + const void* aClosure, + const nsNavHistoryResult* aResult) { + const nsNavHistoryResultNode* updatedNode = + static_cast(aClosure); + + aNode->mAccessCount = updatedNode->mAccessCount; + aNode->mTime = updatedNode->mTime; + aNode->mFrecency = updatedNode->mFrecency; + aNode->mHidden = updatedNode->mHidden; + + return NS_OK; +} + +/** + * Here we need to update all copies of the URI we have with the new visit + * count, and potentially add a new entry in our query. This is the most + * common update operation and it is important that it be as efficient as + * possible. + */ +nsresult nsNavHistoryQueryResultNode::OnVisit(nsIURI* aURI, int64_t aVisitId, + PRTime aTime, + uint32_t aTransitionType, + bool aHidden, uint32_t* aAdded) { + if (aHidden && !mOptions->IncludeHidden()) return NS_OK; + // Skip the notification if the query is filtered by specific transition types + // and this visit has a different one. + if (mTransitions.Length() > 0 && !mTransitions.Contains(aTransitionType)) + return NS_OK; + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->IsBatching() && + ++mBatchChanges > MAX_BATCH_CHANGES_BEFORE_REFRESH) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; + } + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + + switch (mLiveUpdate) { + case QUERYUPDATE_MOBILEPREF: { + return NS_OK; + } + + case QUERYUPDATE_HOST: { + // For these simple yet common cases we can check the host ourselves + // before doing the overhead of creating a new result node. + if (mQuery->Domain().IsVoid()) return NS_OK; + + nsAutoCString host; + if (NS_FAILED(aURI->GetAsciiHost(host))) return NS_OK; + + if (!mQuery->Domain().Equals(host)) return NS_OK; + + // Fall through to check the time, if the time is not present it will + // still match. + [[fallthrough]]; + } + + case QUERYUPDATE_TIME: { + // For these simple yet common cases we can check the time ourselves + // before doing the overhead of creating a new result node. + bool hasIt; + mQuery->GetHasBeginTime(&hasIt); + if (hasIt) { + PRTime beginTime = history->NormalizeTime(mQuery->BeginTimeReference(), + mQuery->BeginTime()); + if (aTime < beginTime) return NS_OK; // before our time range + } + mQuery->GetHasEndTime(&hasIt); + if (hasIt) { + PRTime endTime = history->NormalizeTime(mQuery->EndTimeReference(), + mQuery->EndTime()); + if (aTime > endTime) return NS_OK; // after our time range + } + // Now we know that our visit satisfies the time range, fall through to + // the QUERYUPDATE_SIMPLE case below. + [[fallthrough]]; + } + + case QUERYUPDATE_SIMPLE: { + // The history service can tell us whether the new item should appear + // in the result. We first have to construct a node for it to check. + RefPtr addition; + nsresult rv = history->VisitIdToResultNode(aVisitId, mOptions, + getter_AddRefs(addition)); + NS_ENSURE_SUCCESS(rv, rv); + if (!addition) { + // Certain result types manage the nodes by themselves. + return NS_OK; + } + addition->mTransitionType = aTransitionType; + if (!evaluateQueryForNode(mQuery, mOptions, addition)) + return NS_OK; // don't need to include in our query + + // Optimization for a common case: if the query has maxResults and is + // sorted by date, get the current boundaries and check if the added visit + // would fit. + // Later, we may have to remove the last child to respect maxResults. + if (mOptions->MaxResults() && + static_cast(mChildren.Count()) >= mOptions->MaxResults()) { + uint16_t sortType = GetSortType(); + if (sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING && + aTime > std::max(mChildren[0]->mTime, + mChildren[mChildren.Count() - 1]->mTime)) { + return NS_OK; + } + if (sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING && + aTime < std::min(mChildren[0]->mTime, + mChildren[mChildren.Count() - 1]->mTime)) { + return NS_OK; + } + } + + if (mOptions->ResultType() == + nsNavHistoryQueryOptions::RESULTS_AS_VISIT) { + // If this is a visit type query, just insert the new visit. We never + // update visits, only add or remove them. + rv = InsertSortedChild(addition); + NS_ENSURE_SUCCESS(rv, rv); + } else { + uint16_t sortType = GetSortType(); + bool updateSorting = + sortType == + nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING || + sortType == + nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_FRECENCY_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING; + + if (!UpdateURIs( + false, true, updateSorting, addition->mURI, + setHistoryDetailsCallback, + const_cast(static_cast(addition.get())))) { + // Couldn't find a node to update. + rv = InsertSortedChild(addition); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + // Trim the children if necessary. + if (mOptions->MaxResults() && + static_cast(mChildren.Count()) > mOptions->MaxResults()) { + mChildren.RemoveObjectAt(mChildren.Count() - 1); + } + + if (aAdded) ++(*aAdded); + + break; + } + + case QUERYUPDATE_COMPLEX: + case QUERYUPDATE_COMPLEX_WITH_BOOKMARKS: + // need to requery in complex cases + return Refresh(); + + default: + MOZ_ASSERT(false, "Invalid value for mLiveUpdate"); + return Refresh(); + } + + return NS_OK; +} + +/** + * Find every node that matches this URI and rename it. We try to do + * incremental updates here, even when we are closed, because changing titles + * is easier than requerying if we are invalid. + * + * This actually gets called a lot. Typically, we will get an AddURI message + * when the user visits the page, and then the title will be set asynchronously + * when the title element of the page is parsed. + */ +nsresult nsNavHistoryQueryResultNode::OnTitleChanged( + nsIURI* aURI, const nsAString& aPageTitle, const nsACString& aGUID) { + if (!mExpanded) { + // When we are not expanded, we don't update, just invalidate and unhook. + // It would still be pretty easy to traverse the results and update the + // titles, but when a title changes, its unlikely that it will be the only + // thing. Therefore, we just give up. + ClearChildren(true); + return NS_OK; // no updates in tree state + } + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->IsBatching() && + ++mBatchChanges > MAX_BATCH_CHANGES_BEFORE_REFRESH) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; + } + + // compute what the new title should be + NS_ConvertUTF16toUTF8 newTitle(aPageTitle); + + bool onlyOneEntry = + mOptions->ResultType() == nsINavHistoryQueryOptions::RESULTS_AS_URI; + + // See if our query has any search term matching. + if (mHasSearchTerms) { + // Find all matching URI nodes. + nsCOMArray matches; + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + RecursiveFindURIs(onlyOneEntry, this, spec, &matches); + if (matches.Count() == 0) { + // This could be a new node matching the query, thus we could need + // to add it to the result. + RefPtr node; + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + rv = history->URIToResultNode(aURI, mOptions, getter_AddRefs(node)); + NS_ENSURE_SUCCESS(rv, rv); + if (evaluateQueryForNode(mQuery, mOptions, node)) { + rv = InsertSortedChild(node); + NS_ENSURE_SUCCESS(rv, rv); + } + } + for (int32_t i = 0; i < matches.Count(); ++i) { + // For each matched node we check if it passes the query filter, if not + // we remove the node from the result, otherwise we'll update the title + // later. + nsNavHistoryResultNode* node = matches[i]; + // We must check the node with the new title. + node->mTitle = newTitle; + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + if (!evaluateQueryForNode(mQuery, mOptions, node)) { + nsNavHistoryContainerResultNode* parent = node->mParent; + // URI nodes should always have parents + NS_ENSURE_TRUE(parent, NS_ERROR_UNEXPECTED); + int32_t childIndex = parent->FindChild(node); + NS_ASSERTION(childIndex >= 0, "Child not found in parent"); + parent->RemoveChildAt(childIndex); + } + } + } + + return ChangeTitles(aURI, newTitle, true, onlyOneEntry); +} + +/** + * Here, we can always live update by just deleting all occurrences of + * the given URI. + */ +nsresult nsNavHistoryQueryResultNode::OnPageRemovedFromStore( + nsIURI* aURI, const nsACString& aGUID, uint16_t aReason) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->IsBatching() && + ++mBatchChanges > MAX_BATCH_CHANGES_BEFORE_REFRESH) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; + } + + if (IsContainersQuery()) { + // Incremental updates of query returning queries are pretty much + // complicated. In this case it's possible one of the child queries has + // no more children and it should be removed. Unfortunately there is no + // way to know that without executing the child query and counting results. + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; + } + + bool onlyOneEntry = + mOptions->ResultType() == nsINavHistoryQueryOptions::RESULTS_AS_URI; + + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMArray matches; + RecursiveFindURIs(onlyOneEntry, this, spec, &matches); + for (int32_t i = 0; i < matches.Count(); ++i) { + nsNavHistoryResultNode* node = matches[i]; + nsNavHistoryContainerResultNode* parent = node->mParent; + // URI nodes should always have parents + NS_ENSURE_TRUE(parent, NS_ERROR_UNEXPECTED); + + int32_t childIndex = parent->FindChild(node); + NS_ASSERTION(childIndex >= 0, "Child not found in parent"); + parent->RemoveChildAt(childIndex); + if (parent->mChildren.Count() == 0 && parent->IsQuery() && + parent->mIndentLevel > -1) { + // When query subcontainers (like hosts) get empty we should remove them + // as well. If the parent is not the root node, append it to our list + // and it will get evaluated later in the loop. + matches.AppendObject(parent); + } + } + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnClearHistory() { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +static nsresult setFaviconCallback(nsNavHistoryResultNode* aNode, + const void* aClosure, + const nsNavHistoryResult* aResult) { + if (aResult && (!aNode->mParent || aNode->mParent->AreChildrenVisible())) + NOTIFY_RESULT_OBSERVERS(aResult, NodeIconChanged(aNode)); + + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnPageRemovedVisits( + nsIURI* aURI, bool aPartialRemoval, const nsACString& aGUID, + uint16_t aReason, uint32_t aTransitionType) { + MOZ_ASSERT( + mOptions->QueryType() == nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY, + "Bookmarks queries should not get a OnDeleteVisits notification"); + + if (!aPartialRemoval) { + // All visits for this uri have been removed, but the uri won't be removed + // from the databse, most likely because it's a bookmark. For a history + // query this is equivalent to a OnPageRemovedFromStore notification. + nsresult rv = OnPageRemovedFromStore(aURI, aGUID, aReason); + NS_ENSURE_SUCCESS(rv, rv); + } + if (aTransitionType > 0) { + // All visits for aTransitionType have been removed, if the query is + // filtering on such transition type, this is equivalent to an + // OnPageRemovedFromStore notification. + if (mTransitions.Length() > 0 && mTransitions.Contains(aTransitionType)) { + nsresult rv = OnPageRemovedFromStore(aURI, aGUID, aReason); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::NotifyIfTagsChanged(nsIURI* aURI) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + bool onlyOneEntry = + mOptions->ResultType() == nsINavHistoryQueryOptions::RESULTS_AS_URI; + + // Find matching URI nodes. + RefPtr node; + nsNavHistory* history = nsNavHistory::GetHistoryService(); + + nsCOMArray matches; + RecursiveFindURIs(onlyOneEntry, this, spec, &matches); + + if (matches.Count() == 0 && mHasSearchTerms) { + // A new tag has been added, it's possible it matches our query. + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + rv = history->URIToResultNode(aURI, mOptions, getter_AddRefs(node)); + NS_ENSURE_SUCCESS(rv, rv); + if (evaluateQueryForNode(mQuery, mOptions, node)) { + rv = InsertSortedChild(node); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + for (int32_t i = 0; i < matches.Count(); ++i) { + nsNavHistoryResultNode* node = matches[i]; + // Force a tags update before checking the node. + node->mTags.SetIsVoid(true); + nsAutoString tags; + rv = node->GetTags(tags); + NS_ENSURE_SUCCESS(rv, rv); + // It's possible now this node does not respect anymore the conditions. + // In such a case it should be removed. + if (mHasSearchTerms && !evaluateQueryForNode(mQuery, mOptions, node)) { + nsNavHistoryContainerResultNode* parent = node->mParent; + // URI nodes should always have parents + NS_ENSURE_TRUE(parent, NS_ERROR_UNEXPECTED); + int32_t childIndex = parent->FindChild(node); + NS_ASSERTION(childIndex >= 0, "Child not found in parent"); + parent->RemoveChildAt(childIndex); + } else { + NOTIFY_RESULT_OBSERVERS(result, NodeTagsChanged(node)); + } + } + + return NS_OK; +} + +/** + * These are the bookmark observer functions for query nodes. They listen + * for bookmark events and refresh the results if we have any dependence on + * the bookmark system. + */ +nsresult nsNavHistoryQueryResultNode::OnItemAdded( + int64_t aItemId, int64_t aParentId, int32_t aIndex, uint16_t aItemType, + nsIURI* aURI, PRTime aDateAdded, const nsACString& aGUID, + const nsACString& aParentGUID, uint16_t aSource) { + if (aItemType == nsINavBookmarksService::TYPE_BOOKMARK && + mLiveUpdate != QUERYUPDATE_SIMPLE && mLiveUpdate != QUERYUPDATE_TIME && + mLiveUpdate != QUERYUPDATE_MOBILEPREF) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemRemoved( + int64_t aItemId, int64_t aParentFolder, int32_t aIndex, uint16_t aItemType, + nsIURI* aURI, const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource) { + if ((aItemType == nsINavBookmarksService::TYPE_BOOKMARK || + (aItemType == nsINavBookmarksService::TYPE_FOLDER && + mOptions->ResultType() == + nsINavHistoryQueryOptions::RESULTS_AS_TAGS_ROOT && + aParentGUID.EqualsLiteral(TAGS_ROOT_GUID))) && + mLiveUpdate != QUERYUPDATE_SIMPLE && mLiveUpdate != QUERYUPDATE_TIME && + mLiveUpdate != QUERYUPDATE_MOBILEPREF) { + nsresult rv = Refresh(); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemMoved( + int64_t aFolder, int32_t aOldIndex, int32_t aNewIndex, uint16_t aItemType, + const nsACString& aGUID, const nsACString& aOldParentGUID, + const nsACString& aNewParentGUID, uint16_t aSource, + const nsACString& aURI) { + // 1. The query cannot be affected by the item's position + // 2. For the time being, we cannot optimize this not to update + // queries which are not restricted to some folders, due to way + // sub-queries are updated (see Refresh) + if (mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS && + aItemType != nsINavBookmarksService::TYPE_SEPARATOR && + !aNewParentGUID.Equals(aOldParentGUID)) { + return Refresh(); + } + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemTagsChanged(int64_t aItemId, + const nsAString& aURL) { + nsresult rv = nsNavHistoryResultNode::OnItemTagsChanged(aItemId, aURL); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr uri; + rv = NS_NewURI(getter_AddRefs(uri), aURL); + NS_ENSURE_SUCCESS(rv, rv); + rv = NotifyIfTagsChanged(uri); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemTimeChanged(int64_t aItemId, + const nsACString& aGUID, + PRTime aDateAdded, + PRTime aLastModified) { + // Update ourselves first. + nsresult rv = nsNavHistoryResultNode::OnItemTimeChanged( + aItemId, aGUID, aDateAdded, aLastModified); + NS_ENSURE_SUCCESS(rv, rv); + + if (mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS) { + return Refresh(); + } + + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemTitleChanged( + int64_t aItemId, const nsACString& aGUID, const nsACString& aTitle, + PRTime aLastModified) { + // Update ourselves first. + nsresult rv = nsNavHistoryResultNode::OnItemTitleChanged( + aItemId, aGUID, aTitle, aLastModified); + NS_ENSURE_SUCCESS(rv, rv); + + if (mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS) { + return Refresh(); + } + + return NS_OK; +} + +nsresult nsNavHistoryQueryResultNode::OnItemUrlChanged(int64_t aItemId, + const nsACString& aGUID, + const nsACString& aURL, + PRTime aLastModified) { + if (aItemId == mItemId) { + nsresult rv = nsNavHistoryResultNode::OnItemUrlChanged(aItemId, aGUID, aURL, + aLastModified); + NS_ENSURE_SUCCESS(rv, rv); + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + nsCOMPtr query; + nsCOMPtr options; + rv = history->QueryStringToQuery(mURI, getter_AddRefs(query), + getter_AddRefs(options)); + NS_ENSURE_SUCCESS(rv, rv); + mQuery = do_QueryObject(query); + NS_ENSURE_STATE(mQuery); + mOptions = do_QueryObject(options); + NS_ENSURE_STATE(mOptions); + rv = mOptions->Clone(getter_AddRefs(mOriginalOptions)); + NS_ENSURE_SUCCESS(rv, rv); + + return Refresh(); + } + + if (mLiveUpdate == QUERYUPDATE_COMPLEX_WITH_BOOKMARKS) { + return Refresh(); + } + + uint32_t index; + nsNavHistoryResultNode* node = FindChildById(aItemId, &index); + if (node) { + return node->OnItemUrlChanged(aItemId, aGUID, aURL, aLastModified); + } + + return NS_OK; +} + +/** + * HOW DYNAMIC FOLDER UPDATING WORKS + * + * When you create a result, it will automatically keep itself in sync with + * stuff that happens in the system. For folder nodes, this means changes to + * bookmarks. + * + * A folder will fill its children "when necessary." This means it is being + * opened or whether we need to see if it is empty for twisty drawing. It will + * then register its ID with the main result object that owns it. This result + * object will listen for all bookmark notifications and pass those + * notifications to folder nodes that have registered for that specific folder + * ID. + * + * When a bookmark folder is closed, it will not clear its children. Instead, + * it will keep them and also stay registered as a listener. This means that + * you can more quickly re-open the same folder without doing any work. This + * happens a lot for menus, and bookmarks don't change very often. + * + * When a message comes in and the folder is open, we will do the correct + * operations to keep ourselves in sync with the bookmark service. If the + * folder is closed, we just clear our list to mark it as invalid and + * unregister as a listener. This means we do not have to keep maintaining + * an up-to-date list for the entire bookmark menu structure in every place + * it is used. + */ +NS_IMPL_ISUPPORTS_INHERITED(nsNavHistoryFolderResultNode, + nsNavHistoryContainerResultNode, + nsINavHistoryQueryResultNode, + mozIStorageStatementCallback) + +nsNavHistoryFolderResultNode::nsNavHistoryFolderResultNode( + const nsACString& aTitle, nsNavHistoryQueryOptions* aOptions, + int64_t aFolderId) + : nsNavHistoryContainerResultNode( + ""_ns, aTitle, 0, nsNavHistoryResultNode::RESULT_TYPE_FOLDER, + aOptions), + mContentsValid(false), + mTargetFolderItemId(aFolderId), + mIsRegisteredFolderObserver(false) { + mItemId = aFolderId; +} + +nsNavHistoryFolderResultNode::~nsNavHistoryFolderResultNode() { + if (mIsRegisteredFolderObserver && mResult) + mResult->RemoveBookmarkFolderObserver(this, mTargetFolderGuid); +} + +/** + * Here we do not want to call ContainerResultNode::OnRemoving since our own + * ClearChildren will do the same thing and more (unregister the observers). + * The base ResultNode::OnRemoving will clear some regular node stats, so it is + * OK. + */ +void nsNavHistoryFolderResultNode::OnRemoving() { + nsNavHistoryResultNode::OnRemoving(); + ClearChildren(true); + mResult = nullptr; +} + +nsresult nsNavHistoryFolderResultNode::OpenContainer() { + NS_ASSERTION(!mExpanded, "Container must be expanded to close it"); + nsresult rv; + + if (!mContentsValid) { + rv = FillChildren(); + NS_ENSURE_SUCCESS(rv, rv); + } + mExpanded = true; + + rv = NotifyOnStateChange(STATE_CLOSED); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/** + * The async version of OpenContainer. + */ +nsresult nsNavHistoryFolderResultNode::OpenContainerAsync() { + NS_ASSERTION(!mExpanded, "Container already expanded when opening it"); + + // If the children are valid, open the container synchronously. This will be + // the case when the container has already been opened and any other time + // FillChildren or FillChildrenAsync has previously been called. + if (mContentsValid) return OpenContainer(); + + nsresult rv = FillChildrenAsync(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = NotifyOnStateChange(STATE_CLOSED); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/** + * @see nsNavHistoryQueryResultNode::HasChildren. The semantics here are a + * little different. Querying the contents of a bookmark folder is relatively + * fast and it is common to have empty folders. Therefore, we always want to + * return the correct result so that twisties are drawn properly. + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetHasChildren(bool* aHasChildren) { + if (!mContentsValid) { + nsresult rv = FillChildren(); + NS_ENSURE_SUCCESS(rv, rv); + } + *aHasChildren = (mChildren.Count() > 0); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetFolderItemId(int64_t* aItemId) { + *aItemId = mTargetFolderItemId; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetTargetFolderGuid(nsACString& aGuid) { + aGuid = mTargetFolderGuid; + return NS_OK; +} + +/** + * Lazily computes the URI for this specific folder query with the current + * options. + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetUri(nsACString& aURI) { + if (!mURI.IsEmpty()) { + aURI = mURI; + return NS_OK; + } + + nsCOMPtr query; + nsresult rv = GetQuery(getter_AddRefs(query)); + NS_ENSURE_SUCCESS(rv, rv); + + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + rv = history->QueryToQueryString(query, mOriginalOptions, mURI); + NS_ENSURE_SUCCESS(rv, rv); + aURI = mURI; + return NS_OK; +} + +/** + * @return the queries that give you this bookmarks folder + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetQuery(nsINavHistoryQuery** _query) { + // get the query object + RefPtr query = new nsNavHistoryQuery(); + + nsTArray parents; + // query just has the folder ID set and nothing else + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier, or change the return type to void. + parents.AppendElement(mTargetFolderGuid); + nsresult rv = query->SetParents(parents); + NS_ENSURE_SUCCESS(rv, rv); + + query.forget(_query); + return NS_OK; +} + +/** + * Options for the query that gives you this bookmarks folder. This is just + * the options for the folder with the current folder ID set. + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::GetQueryOptions( + nsINavHistoryQueryOptions** _options) { + MOZ_ASSERT(mOptions, "Options should be valid"); + RefPtr options = mOptions; + options.forget(_options); + return NS_OK; +} + +nsresult nsNavHistoryFolderResultNode::FillChildren() { + NS_ASSERTION(!mContentsValid, + "Don't call FillChildren when contents are valid"); + NS_ASSERTION(mChildren.Count() == 0, + "We are trying to fill children when there already are some"); + + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); + + // Actually get the folder children from the bookmark service. + nsresult rv = + bookmarks->QueryFolderChildren(mTargetFolderItemId, mOptions, &mChildren); + NS_ENSURE_SUCCESS(rv, rv); + + // PERFORMANCE: it may be better to also fill any child folders at this point + // so that we can draw tree twisties without doing a separate query later. + // If we don't end up drawing twisties a lot, it doesn't matter. If we do + // this, we should wrap everything in a transaction here on the bookmark + // service's connection. + + return OnChildrenFilled(); +} + +/** + * Performs some tasks after all the children of the container have been added. + * The container's contents are not valid until this method has been called. + */ +nsresult nsNavHistoryFolderResultNode::OnChildrenFilled() { + // It is important to call FillStats to fill in the parents on all + // nodes and the result node pointers on the containers. + FillStats(); + + if (mResult && mResult->mNeedsToApplySortingMode) { + // We should repopulate container and then apply sortingMode. To avoid + // sorting 2 times we simply do that here. + mResult->SetSortingMode(mResult->mSortingMode); + } else { + // Once we've computed all tree stats, we can sort, because containers will + // then have proper visit counts and dates. + SortComparator comparator = GetSortingComparator(GetSortType()); + if (comparator) { + RecursiveSort(comparator); + } + } + + // If we are limiting our results remove items from the end of the + // mChildren array after sorting. This is done for root node only. + // Note, if count < max results, we won't do anything. + if (!mParent && mOptions->MaxResults()) { + while ((uint32_t)mChildren.Count() > mOptions->MaxResults()) + mChildren.RemoveObjectAt(mChildren.Count() - 1); + } + + // Register with the result for updates. + EnsureRegisteredAsFolderObserver(); + + mContentsValid = true; + return NS_OK; +} + +/** + * Registers the node with its result as a folder observer if it is not already + * registered. + */ +void nsNavHistoryFolderResultNode::EnsureRegisteredAsFolderObserver() { + if (!mIsRegisteredFolderObserver && mResult) { + mResult->AddBookmarkFolderObserver(this, mTargetFolderGuid); + mIsRegisteredFolderObserver = true; + } +} + +/** + * The async version of FillChildren. This begins asynchronous execution by + * calling nsNavBookmarks::QueryFolderChildrenAsync. During execution, this + * node's async Storage callbacks, HandleResult and HandleCompletion, will be + * called. + */ +nsresult nsNavHistoryFolderResultNode::FillChildrenAsync() { + NS_ASSERTION(!mContentsValid, "FillChildrenAsync when contents are valid"); + NS_ASSERTION(mChildren.Count() == 0, "FillChildrenAsync when children exist"); + + // ProcessFolderNodeChild, called in HandleResult, increments this for every + // result row it processes. Initialize it here as we begin async execution. + mAsyncBookmarkIndex = -1; + + nsNavBookmarks* bmSvc = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bmSvc, NS_ERROR_OUT_OF_MEMORY); + nsresult rv = + bmSvc->QueryFolderChildrenAsync(this, getter_AddRefs(mAsyncPendingStmt)); + NS_ENSURE_SUCCESS(rv, rv); + + // Register with the result for updates. All updates during async execution + // will cause it to be restarted. + EnsureRegisteredAsFolderObserver(); + + return NS_OK; +} + +/** + * A mozIStorageStatementCallback method. Called during the async execution + * begun by FillChildrenAsync. + * + * @param aResultSet + * The result set containing the data from the database. + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::HandleResult(mozIStorageResultSet* aResultSet) { + NS_ENSURE_ARG_POINTER(aResultSet); + + nsNavBookmarks* bmSvc = nsNavBookmarks::GetBookmarksService(); + if (!bmSvc) { + CancelAsyncOpen(false); + return NS_ERROR_OUT_OF_MEMORY; + } + + // Consume all the currently available rows of the result set. + nsCOMPtr row; + while (NS_SUCCEEDED(aResultSet->GetNextRow(getter_AddRefs(row))) && row) { + nsresult rv = bmSvc->ProcessFolderNodeRow(row, mOptions, &mChildren, + mAsyncBookmarkIndex); + if (NS_FAILED(rv)) { + CancelAsyncOpen(false); + return rv; + } + } + + return NS_OK; +} + +/** + * A mozIStorageStatementCallback method. Called during the async execution + * begun by FillChildrenAsync. + * + * @param aReason + * Indicates the final state of execution. + */ +NS_IMETHODIMP +nsNavHistoryFolderResultNode::HandleCompletion(uint16_t aReason) { + if (aReason == mozIStorageStatementCallback::REASON_FINISHED && + mAsyncCanceledState == NOT_CANCELED) { + // Async execution successfully completed. The container is ready to open. + + nsresult rv = OnChildrenFilled(); + NS_ENSURE_SUCCESS(rv, rv); + + mExpanded = true; + mAsyncPendingStmt = nullptr; + + // Notify observers only after mExpanded and mAsyncPendingStmt are set. + rv = NotifyOnStateChange(STATE_LOADING); + NS_ENSURE_SUCCESS(rv, rv); + } + + else if (mAsyncCanceledState == CANCELED_RESTART_NEEDED) { + // Async execution was canceled and needs to be restarted. + mAsyncCanceledState = NOT_CANCELED; + ClearChildren(false); + FillChildrenAsync(); + } + + else { + // Async execution failed or was canceled without restart. Remove all + // children and close the container, notifying observers. + mAsyncCanceledState = NOT_CANCELED; + ClearChildren(true); + CloseContainer(); + } + + return NS_OK; +} + +void nsNavHistoryFolderResultNode::ClearChildren(bool unregister) { + for (int32_t i = 0; i < mChildren.Count(); ++i) mChildren[i]->OnRemoving(); + mChildren.Clear(); + + bool needsUnregister = unregister && (mContentsValid || mAsyncPendingStmt); + if (needsUnregister && mResult && mIsRegisteredFolderObserver) { + mResult->RemoveBookmarkFolderObserver(this, mTargetFolderGuid); + mIsRegisteredFolderObserver = false; + } + mContentsValid = false; +} + +/** + * This is called to update the result when something has changed that we + * can not incrementally update. + */ +nsresult nsNavHistoryFolderResultNode::Refresh() { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + if (result->IsBatching()) { + result->requestRefresh(this); + return NS_OK; + } + + ClearChildren(true); + + if (!mExpanded) { + // When we are not expanded, we don't update, just invalidate and unhook. + return NS_OK; + } + + // Ignore errors from FillChildren, since we will still want to refresh + // the tree (there just might not be anything in it on error). ClearChildren + // has unregistered us as an observer since FillChildren will try to + // re-register us. + (void)FillChildren(); + + NOTIFY_RESULT_OBSERVERS(result, InvalidateContainer(TO_CONTAINER(this))); + return NS_OK; +} + +/** + * Implements the logic described above the constructor. This sees if we + * should do an incremental update and returns true if so. If not, it + * invalidates our children, unregisters us an observer, and returns false. + */ +bool nsNavHistoryFolderResultNode::StartIncrementalUpdate() { + // if any items are excluded, we can not do incremental updates since the + // indices from the bookmark service will not be valid + + if (!mOptions->ExcludeItems() && !mOptions->ExcludeQueries()) { + // easy case: we are visible, always do incremental update + if (mExpanded || AreChildrenVisible()) return true; + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_TRUE(result, false); + + // When any observers are attached also do incremental updates if our + // parent is visible, so that twisties are drawn correctly. + if (mParent) return result->mObservers.Length() > 0; + } + + // otherwise, we don't do incremental updates, invalidate and unregister + (void)Refresh(); + return false; +} + +/** + * This function adds aDelta to all bookmark indices between the two endpoints, + * inclusive. It is used when items are added or removed from the bookmark + * folder. + */ +void nsNavHistoryFolderResultNode::ReindexRange(int32_t aStartIndex, + int32_t aEndIndex, + int32_t aDelta) { + for (int32_t i = 0; i < mChildren.Count(); ++i) { + nsNavHistoryResultNode* node = mChildren[i]; + if (node->mBookmarkIndex >= aStartIndex && + node->mBookmarkIndex <= aEndIndex) + node->mBookmarkIndex += aDelta; + } +} + +// Used by nsNavHistoryFolderResultNode's methods below. If the container is +// notified of a bookmark event while asynchronous execution is pending, this +// restarts it and returns. +#define RESTART_AND_RETURN_IF_ASYNC_PENDING() \ + if (mAsyncPendingStmt) { \ + CancelAsyncOpen(true); \ + return NS_OK; \ + } + +nsresult nsNavHistoryFolderResultNode::OnBeginUpdateBatch() { return NS_OK; } + +nsresult nsNavHistoryFolderResultNode::OnEndUpdateBatch() { return NS_OK; } + +nsresult nsNavHistoryFolderResultNode::OnItemAdded( + int64_t aItemId, int64_t aParentFolder, int32_t aIndex, uint16_t aItemType, + nsIURI* aURI, PRTime aDateAdded, const nsACString& aGUID, + const nsACString& aParentGUID, uint16_t aSource) { + MOZ_ASSERT(aParentFolder == mTargetFolderItemId, "Got wrong bookmark update"); + + RESTART_AND_RETURN_IF_ASYNC_PENDING(); + + { + uint32_t index; + nsNavHistoryResultNode* node = FindChildById(aItemId, &index); + // Bug 1097528. + // It's possible our result registered due to a previous notification, for + // example the Library left pane could have refreshed and replaced the + // right pane as a consequence. In such a case our contents are already + // up-to-date. That's OK. + if (node) return NS_OK; + } + + bool excludeItems = mOptions->ExcludeItems(); + + // here, try to do something reasonable if the bookmark service gives us + // a bogus index. + if (aIndex < 0) { + MOZ_ASSERT_UNREACHABLE("Invalid index for item adding: <0"); + aIndex = 0; + } else if (aIndex > mChildren.Count()) { + if (!excludeItems) { + // Something wrong happened while updating indexes. + MOZ_ASSERT_UNREACHABLE( + "Invalid index for item adding: greater than " + "count"); + } + aIndex = mChildren.Count(); + } + + nsresult rv; + + // Check for query URIs, which are bookmarks, but treated as containers + // in results and views. + bool isQuery = false; + if (aItemType == nsINavBookmarksService::TYPE_BOOKMARK) { + NS_ASSERTION(aURI, "Got a null URI when we are a bookmark?!"); + nsAutoCString itemURISpec; + rv = aURI->GetSpec(itemURISpec); + NS_ENSURE_SUCCESS(rv, rv); + isQuery = IsQueryURI(itemURISpec); + } + + if (aItemType != nsINavBookmarksService::TYPE_FOLDER && !isQuery && + excludeItems) { + // don't update items when we aren't displaying them, but we still need + // to adjust bookmark indices to account for the insertion + ReindexRange(aIndex, INT32_MAX, 1); + return NS_OK; + } + + if (!StartIncrementalUpdate()) + return NS_OK; // folder was completely refreshed for us + + // adjust indices to account for insertion + ReindexRange(aIndex, INT32_MAX, 1); + + RefPtr node; + if (aItemType == nsINavBookmarksService::TYPE_BOOKMARK) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + rv = history->BookmarkIdToResultNode(aItemId, mOptions, + getter_AddRefs(node)); + NS_ENSURE_SUCCESS(rv, rv); + } else if (aItemType == nsINavBookmarksService::TYPE_FOLDER) { + nsNavBookmarks* bookmarks = nsNavBookmarks::GetBookmarksService(); + NS_ENSURE_TRUE(bookmarks, NS_ERROR_OUT_OF_MEMORY); + rv = bookmarks->ResultNodeForContainer(PromiseFlatCString(aGUID), + new nsNavHistoryQueryOptions(), + getter_AddRefs(node)); + NS_ENSURE_SUCCESS(rv, rv); + } else if (aItemType == nsINavBookmarksService::TYPE_SEPARATOR) { + node = new nsNavHistorySeparatorResultNode(); + node->mItemId = aItemId; + node->mBookmarkGuid = aGUID; + node->mDateAdded = aDateAdded; + node->mLastModified = aDateAdded; + } + + node->mBookmarkIndex = aIndex; + + if (aItemType == nsINavBookmarksService::TYPE_SEPARATOR || + GetSortType() == nsINavHistoryQueryOptions::SORT_BY_NONE) { + // insert at natural bookmarks position + return InsertChildAt(node, aIndex); + } + + // insert at sorted position + return InsertSortedChild(node); +} + +nsresult nsNavHistoryQueryResultNode::OnMobilePrefChanged(bool newValue) { + RESTART_AND_RETURN_IF_ASYNC_PENDING(); + + if (newValue) { + // If the folder is being added, simply refresh the query as that is + // simpler than re-inserting just the mobile folder. + return Refresh(); + } + + // We're removing the mobile folder, so find it. + int32_t existingIndex; + FindChildByGuid(nsLiteralCString(MOBILE_BOOKMARKS_VIRTUAL_GUID), + &existingIndex); + + if (existingIndex == -1) { + return NS_OK; + } + + return RemoveChildAt(existingIndex); +} + +nsresult nsNavHistoryFolderResultNode::OnItemRemoved( + int64_t aItemId, int64_t aParentFolder, int32_t aIndex, uint16_t aItemType, + nsIURI* aURI, const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource) { + // Folder shortcuts should not be notified removal of the target folder. + MOZ_ASSERT_IF(mItemId != mTargetFolderItemId, aItemId != mTargetFolderItemId); + // Concrete folders should not be notified their own removal. + // Note aItemId may equal mItemId for recursive folder shortcuts. + MOZ_ASSERT_IF(mItemId == mTargetFolderItemId, aItemId != mItemId); + + // In any case though, here we only care about the children removal. + if (mTargetFolderItemId == aItemId || mItemId == aItemId) return NS_OK; + + MOZ_ASSERT(aParentFolder == mTargetFolderItemId, "Got wrong bookmark update"); + + RESTART_AND_RETURN_IF_ASYNC_PENDING(); + + // don't trust the index from the bookmark service, find it ourselves. The + // sorting could be different, or the bookmark services indices and ours might + // be out of sync somehow. + uint32_t index; + nsNavHistoryResultNode* node = FindChildById(aItemId, &index); + // Bug 1097528. + // It's possible our result registered due to a previous notification, for + // example the Library left pane could have refreshed and replaced the + // right pane as a consequence. In such a case our contents are already + // up-to-date. That's OK. + if (!node) { + return NS_OK; + } + + bool excludeItems = mOptions->ExcludeItems(); + + if ((node->IsURI() || node->IsSeparator()) && excludeItems) { + // don't update items when we aren't displaying them, but we do need to + // adjust everybody's bookmark indices to account for the removal + ReindexRange(aIndex, INT32_MAX, -1); + return NS_OK; + } + + if (!StartIncrementalUpdate()) return NS_OK; // we are completely refreshed + + // shift all following indices down + ReindexRange(aIndex + 1, INT32_MAX, -1); + + return RemoveChildAt(index); +} + +nsresult nsNavHistoryResultNode::OnItemKeywordChanged( + int64_t aItemId, const nsACString& aKeyword) { + if (aItemId != mItemId) { + return NS_OK; + } + + bool shouldNotify = !mParent || mParent->AreChildrenVisible(); + if (shouldNotify) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + NOTIFY_RESULT_OBSERVERS(result, NodeKeywordChanged(this, aKeyword)); + } + + return NS_OK; +} + +nsresult nsNavHistoryResultNode::OnItemTagsChanged(int64_t aItemId, + const nsAString& aURL) { + if (aItemId != mItemId) { + return NS_OK; + } + + mTags.SetIsVoid(true); + + bool shouldNotify = !mParent || mParent->AreChildrenVisible(); + if (shouldNotify) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + NOTIFY_RESULT_OBSERVERS(result, NodeTagsChanged(this)); + } + + return NS_OK; +} + +nsresult nsNavHistoryResultNode::OnItemTimeChanged(int64_t aItemId, + const nsACString& aGUID, + PRTime aDateAdded, + PRTime aLastModified) { + if (aItemId != mItemId) { + return NS_OK; + } + + bool isDateAddedChanged = mDateAdded != aDateAdded; + bool isLastModifiedChanged = mLastModified != aLastModified; + + if (!isDateAddedChanged && !isLastModifiedChanged) { + return NS_OK; + } + + mDateAdded = aDateAdded; + mLastModified = aLastModified; + + bool shouldNotify = !mParent || mParent->AreChildrenVisible(); + if (shouldNotify) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + + if (isDateAddedChanged) { + NOTIFY_RESULT_OBSERVERS(result, NodeDateAddedChanged(this, aDateAdded)); + } + if (isLastModifiedChanged) { + NOTIFY_RESULT_OBSERVERS(result, + NodeLastModifiedChanged(this, aLastModified)); + } + } + + return NS_OK; +} + +nsresult nsNavHistoryResultNode::OnItemTitleChanged(int64_t aItemId, + const nsACString& aGUID, + const nsACString& aTitle, + PRTime aLastModified) { + if (aItemId != mItemId) { + return NS_OK; + } + + mTitle = aTitle; + mLastModified = aLastModified; + + bool shouldNotify = !mParent || mParent->AreChildrenVisible(); + if (shouldNotify) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + NOTIFY_RESULT_OBSERVERS(result, NodeTitleChanged(this, mTitle)); + } + + return NS_OK; +} + +nsresult nsNavHistoryResultNode::OnItemUrlChanged(int64_t aItemId, + const nsACString& aGUID, + const nsACString& aURL, + PRTime aLastModified) { + if (aItemId != mItemId) { + return NS_OK; + } + + // clear the tags string as well + mTags.SetIsVoid(true); + nsCString oldURI(mURI); + mURI = aURL; + mLastModified = aLastModified; + + bool shouldNotify = !mParent || mParent->AreChildrenVisible(); + if (shouldNotify) { + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + NOTIFY_RESULT_OBSERVERS(result, NodeURIChanged(this, oldURI)); + } + + if (!mParent) return NS_OK; + + // The sorting methods fall back to each other so we need to re-sort the + // result even if it's not set to sort by the given property. + int32_t ourIndex = mParent->FindChild(this); + NS_ASSERTION(ourIndex >= 0, "Could not find self in parent"); + if (ourIndex >= 0) mParent->EnsureItemPosition(ourIndex); + + return NS_OK; +} + +nsresult nsNavHistoryResultNode::OnVisitsRemoved() { + PRTime oldTime = mTime; + mTime = 0; + + nsNavHistoryResult* result = GetResult(); + NS_ENSURE_STATE(result); + NOTIFY_RESULT_OBSERVERS( + result, NodeHistoryDetailsChanged(this, oldTime, mAccessCount)); + + return NS_OK; +} + +/** + * Updates visit count and last visit time and refreshes. + */ +nsresult nsNavHistoryFolderResultNode::OnItemVisited(nsIURI* aURI, + int64_t aVisitId, + PRTime aTime) { + if (mOptions->ExcludeItems()) + return NS_OK; // don't update items when we aren't displaying them + + RESTART_AND_RETURN_IF_ASYNC_PENDING(); + + if (!StartIncrementalUpdate()) return NS_OK; + + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMArray nodes; + FindChildrenByURI(spec, &nodes); + if (!nodes.Count()) { + return NS_OK; + } + + // Update us. + int32_t oldAccessCount = mAccessCount; + ++mAccessCount; + if (aTime > mTime) mTime = aTime; + rv = ReverseUpdateStats(mAccessCount - oldAccessCount); + NS_ENSURE_SUCCESS(rv, rv); + + // Update nodes. + for (int32_t i = 0; i < nodes.Count(); ++i) { + nsNavHistoryResultNode* node = nodes[i]; + uint32_t nodeOldAccessCount = node->mAccessCount; + PRTime nodeOldTime = node->mTime; + node->mTime = aTime; + ++node->mAccessCount; + + // Update frecency for proper frecency ordering. + // TODO (bug 832617): we may avoid one query here, by providing the new + // frecency value in the notification. + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_OK); + RefPtr visitNode; + rv = history->VisitIdToResultNode(aVisitId, mOptions, + getter_AddRefs(visitNode)); + NS_ENSURE_SUCCESS(rv, rv); + if (!visitNode) { + // Certain result types manage the nodes by themselves. + return NS_OK; + } + node->mFrecency = visitNode->mFrecency; + + nsNavHistoryResult* result = GetResult(); + if (AreChildrenVisible() && !result->CanSkipHistoryDetailsNotifications()) { + // Sorting has not changed, just redraw the row if it's visible. + NOTIFY_RESULT_OBSERVERS( + result, + NodeHistoryDetailsChanged(node, nodeOldTime, nodeOldAccessCount)); + } + + // Update sorting if necessary. + uint32_t sortType = GetSortType(); + if (sortType == nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_FRECENCY_ASCENDING || + sortType == nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING) { + int32_t childIndex = FindChild(node); + NS_ASSERTION(childIndex >= 0, + "Could not find child we just got a reference to"); + if (childIndex >= 0) { + EnsureItemPosition(childIndex); + } + } + } + + return NS_OK; +} + +nsresult nsNavHistoryFolderResultNode::OnItemMoved( + int64_t aItemId, int32_t aOldIndex, int32_t aNewIndex, uint16_t aItemType, + const nsACString& aGUID, const nsACString& aOldParentGUID, + const nsACString& aNewParentGUID, uint16_t aSource, + const nsACString& aURI) { + MOZ_ASSERT(aOldParentGUID.Equals(mTargetFolderGuid) || + aNewParentGUID.Equals(mTargetFolderGuid), + "Got a bookmark message that doesn't belong to us"); + + RESTART_AND_RETURN_IF_ASYNC_PENDING(); + + bool excludeItems = mOptions->ExcludeItems(); + if (excludeItems && (aItemType == nsINavBookmarksService::TYPE_SEPARATOR || + (aItemType == nsINavBookmarksService::TYPE_BOOKMARK && + !StringBeginsWith(aURI, "place:"_ns)))) { + // This is a bookmark or a separator, so we don't need to handle this if + // we're excluding items. + return NS_OK; + } + + uint32_t index; + nsNavHistoryResultNode* node = FindChildById(aItemId, &index); + // Bug 1097528. + // It's possible our result registered due to a previous notification, for + // example the Library left pane could have refreshed and replaced the + // right pane as a consequence. In such a case our contents are already + // up-to-date. That's OK. + if (node && aNewParentGUID.Equals(mTargetFolderGuid) && + index == static_cast(aNewIndex)) + return NS_OK; + if (!node && aOldParentGUID.Equals(mTargetFolderGuid)) return NS_OK; + + if (!StartIncrementalUpdate()) + return NS_OK; // entire container was refreshed for us + + if (aNewParentGUID.Equals(aOldParentGUID)) { + // getting moved within the same folder, we don't want to do a remove and + // an add because that will lose your tree state. + + // adjust bookmark indices + ReindexRange(aOldIndex + 1, INT32_MAX, -1); + ReindexRange(aNewIndex, INT32_MAX, 1); + + MOZ_ASSERT(node, "Can't find folder that is moving!"); + if (!node) { + return NS_ERROR_FAILURE; + } + MOZ_ASSERT(index < uint32_t(mChildren.Count()), "Invalid index!"); + node->mBookmarkIndex = aNewIndex; + + // adjust position + EnsureItemPosition(index); + return NS_OK; + } + + // moving between two different folders, just do a remove and an add + nsCOMPtr itemURI; + if (aItemType == nsINavBookmarksService::TYPE_BOOKMARK) { + nsresult rv = NS_NewURI(getter_AddRefs(itemURI), aURI); + NS_ENSURE_SUCCESS(rv, rv); + } + if (aOldParentGUID.Equals(mTargetFolderGuid)) { + OnItemRemoved(aItemId, mTargetFolderItemId, aOldIndex, aItemType, itemURI, + aGUID, aOldParentGUID, aSource); + } + if (aNewParentGUID.Equals(mTargetFolderGuid)) { + OnItemAdded( + aItemId, mTargetFolderItemId, aNewIndex, aItemType, itemURI, + RoundedPRNow(), // This is a dummy dateAdded, not the real value. + aGUID, aNewParentGUID, aSource); + } + + return NS_OK; +} + +/** + * Separator nodes do not hold any data. + */ +nsNavHistorySeparatorResultNode::nsNavHistorySeparatorResultNode() + : nsNavHistoryResultNode(""_ns, ""_ns, 0, 0) {} + +NS_IMPL_CYCLE_COLLECTION_CLASS(nsNavHistoryResult) + +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsNavHistoryResult) + tmp->StopObservingOnUnlink(); + + NS_IMPL_CYCLE_COLLECTION_UNLINK(mRootNode) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mObservers) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mMobilePrefObservers) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mAllBookmarksObservers) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mHistoryObservers) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mRefreshParticipants) + NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_REFERENCE +NS_IMPL_CYCLE_COLLECTION_UNLINK_END + +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsNavHistoryResult) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRootNode) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mObservers) + for (nsNavHistoryResult::FolderObserverList* list : + tmp->mBookmarkFolderObservers.Values()) { + for (uint32_t i = 0; i < list->Length(); ++i) { + NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, + "mBookmarkFolderObservers value[i]"); + nsNavHistoryResultNode* node = list->ElementAt(i); + cb.NoteXPCOMChild(node); + } + } + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMobilePrefObservers) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAllBookmarksObservers) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mHistoryObservers) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRefreshParticipants) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(nsNavHistoryResult) +NS_IMPL_CYCLE_COLLECTING_RELEASE(nsNavHistoryResult) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsNavHistoryResult) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsINavHistoryResult) + NS_INTERFACE_MAP_STATIC_AMBIGUOUS(nsNavHistoryResult) + NS_INTERFACE_MAP_ENTRY(nsINavHistoryResult) + NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference) +NS_INTERFACE_MAP_END + +nsNavHistoryResult::nsNavHistoryResult( + nsNavHistoryContainerResultNode* aRoot, + const RefPtr& aQuery, + const RefPtr& aOptions) + : mRootNode(aRoot), + mQuery(aQuery), + mOptions(aOptions), + mNeedsToApplySortingMode(false), + mIsHistoryObserver(false), + mIsBookmarksObserver(false), + mIsMobilePrefObserver(false), + mBookmarkFolderObservers(64), + mSuppressNotifications(false), + mIsHistoryDetailsObserver(false), + mObserversWantHistoryDetails(true), + mBatchInProgress(0) { + mSortingMode = aOptions->SortingMode(); + + mRootNode->mResult = this; + MOZ_ASSERT(mRootNode->mIndentLevel == -1, + "Root node's indent level initialized wrong"); + mRootNode->FillStats(); + + AutoTArray events; + events.AppendElement(PlacesEventType::Purge_caches); + PlacesObservers::AddListener(events, this); +} + +nsNavHistoryResult::~nsNavHistoryResult() { + // Delete all heap-allocated bookmark folder observer arrays. + for (auto it = mBookmarkFolderObservers.Iter(); !it.Done(); it.Next()) { + delete it.Data(); + it.Remove(); + } +} + +void nsNavHistoryResult::StopObserving() { + AutoTArray events; + events.AppendElement(PlacesEventType::Favicon_changed); + if (mIsBookmarksObserver) { + events.AppendElement(PlacesEventType::Bookmark_added); + events.AppendElement(PlacesEventType::Bookmark_removed); + events.AppendElement(PlacesEventType::Bookmark_moved); + events.AppendElement(PlacesEventType::Bookmark_keyword_changed); + events.AppendElement(PlacesEventType::Bookmark_tags_changed); + events.AppendElement(PlacesEventType::Bookmark_time_changed); + events.AppendElement(PlacesEventType::Bookmark_title_changed); + events.AppendElement(PlacesEventType::Bookmark_url_changed); + mIsBookmarksObserver = false; + } + if (mIsMobilePrefObserver) { + Preferences::UnregisterCallback(OnMobilePrefChangedCallback, + MOBILE_BOOKMARKS_PREF, this); + mIsMobilePrefObserver = false; + } + if (mIsHistoryObserver) { + mIsHistoryObserver = false; + events.AppendElement(PlacesEventType::History_cleared); + events.AppendElement(PlacesEventType::Page_removed); + } + if (mIsHistoryDetailsObserver) { + events.AppendElement(PlacesEventType::Page_visited); + events.AppendElement(PlacesEventType::Page_title_changed); + mIsHistoryDetailsObserver = false; + } + + PlacesObservers::RemoveListener(events, this); +} + +void nsNavHistoryResult::StopObservingOnUnlink() { + StopObserving(); + + AutoTArray events; + events.AppendElement(PlacesEventType::Purge_caches); + PlacesObservers::RemoveListener(events, this); + + for (auto it = mBookmarkFolderObservers.Iter(); !it.Done(); it.Next()) { + delete it.Data(); + it.Remove(); + } +} + +bool nsNavHistoryResult::CanSkipHistoryDetailsNotifications() const { + return !mObserversWantHistoryDetails && + mOptions->QueryType() == + nsINavHistoryQueryOptions::QUERY_TYPE_BOOKMARKS && + mSortingMode != nsINavHistoryQueryOptions::SORT_BY_DATE_ASCENDING && + mSortingMode != nsINavHistoryQueryOptions::SORT_BY_DATE_DESCENDING && + mSortingMode != + nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_ASCENDING && + mSortingMode != + nsINavHistoryQueryOptions::SORT_BY_VISITCOUNT_DESCENDING && + mSortingMode != + nsINavHistoryQueryOptions::SORT_BY_FRECENCY_ASCENDING && + mSortingMode != nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING; +} + +void nsNavHistoryResult::AddHistoryObserver( + nsNavHistoryQueryResultNode* aNode) { + if (!mIsHistoryObserver) { + mIsHistoryObserver = true; + + AutoTArray events; + events.AppendElement(PlacesEventType::History_cleared); + events.AppendElement(PlacesEventType::Page_removed); + if (!mIsHistoryDetailsObserver) { + events.AppendElement(PlacesEventType::Page_visited); + events.AppendElement(PlacesEventType::Page_title_changed); + mIsHistoryDetailsObserver = true; + } + PlacesObservers::AddListener(events, this); + } + // Don't add duplicate observers. In some case we don't unregister when + // children are cleared (see ClearChildren) and the next FillChildren call + // will try to add the observer again. + if (mHistoryObservers.IndexOf(aNode) == QueryObserverList::NoIndex) { + mHistoryObservers.AppendElement(aNode); + } +} + +void nsNavHistoryResult::AddAllBookmarksObserver( + nsNavHistoryQueryResultNode* aNode) { + EnsureIsObservingBookmarks(); + // Don't add duplicate observers. In some case we don't unregister when + // children are cleared (see ClearChildren) and the next FillChildren call + // will try to add the observer again. + if (mAllBookmarksObservers.IndexOf(aNode) == QueryObserverList::NoIndex) { + mAllBookmarksObservers.AppendElement(aNode); + } +} + +void nsNavHistoryResult::AddMobilePrefsObserver( + nsNavHistoryQueryResultNode* aNode) { + if (!mIsMobilePrefObserver) { + Preferences::RegisterCallback(OnMobilePrefChangedCallback, + MOBILE_BOOKMARKS_PREF, this); + mIsMobilePrefObserver = true; + } + // Don't add duplicate observers. In some case we don't unregister when + // children are cleared (see ClearChildren) and the next FillChildren call + // will try to add the observer again. + if (mMobilePrefObservers.IndexOf(aNode) == mMobilePrefObservers.NoIndex) { + mMobilePrefObservers.AppendElement(aNode); + } +} + +void nsNavHistoryResult::AddBookmarkFolderObserver( + nsNavHistoryFolderResultNode* aNode, const nsACString& aFolderGUID) { + MOZ_ASSERT(!aFolderGUID.IsEmpty(), "aFolderGUID should not be empty"); + EnsureIsObservingBookmarks(); + // Don't add duplicate observers. In some case we don't unregister when + // children are cleared (see ClearChildren) and the next FillChildren call + // will try to add the observer again. + FolderObserverList* list = BookmarkFolderObserversForGUID(aFolderGUID, true); + if (list->IndexOf(aNode) == FolderObserverList::NoIndex) { + list->AppendElement(aNode); + } +} + +void nsNavHistoryResult::EnsureIsObservingBookmarks() { + if (mIsBookmarksObserver) { + return; + } + AutoTArray events; + events.AppendElement(PlacesEventType::Bookmark_added); + events.AppendElement(PlacesEventType::Bookmark_removed); + events.AppendElement(PlacesEventType::Bookmark_moved); + events.AppendElement(PlacesEventType::Bookmark_keyword_changed); + events.AppendElement(PlacesEventType::Bookmark_tags_changed); + events.AppendElement(PlacesEventType::Bookmark_time_changed); + events.AppendElement(PlacesEventType::Bookmark_title_changed); + events.AppendElement(PlacesEventType::Bookmark_url_changed); + // If we're not observing visits yet, also add a page-visited observer to + // serve onItemVisited. + if (!mIsHistoryObserver && !mIsHistoryDetailsObserver) { + events.AppendElement(PlacesEventType::Page_visited); + mIsHistoryDetailsObserver = true; + } + PlacesObservers::AddListener(events, this); + mIsBookmarksObserver = true; +} + +void nsNavHistoryResult::RemoveHistoryObserver( + nsNavHistoryQueryResultNode* aNode) { + mHistoryObservers.RemoveElement(aNode); +} + +void nsNavHistoryResult::RemoveAllBookmarksObserver( + nsNavHistoryQueryResultNode* aNode) { + mAllBookmarksObservers.RemoveElement(aNode); +} + +void nsNavHistoryResult::RemoveMobilePrefsObserver( + nsNavHistoryQueryResultNode* aNode) { + mMobilePrefObservers.RemoveElement(aNode); +} + +void nsNavHistoryResult::RemoveBookmarkFolderObserver( + nsNavHistoryFolderResultNode* aNode, const nsACString& aFolderGUID) { + MOZ_ASSERT(!aFolderGUID.IsEmpty(), "aFolderGUID should not be empty"); + FolderObserverList* list = BookmarkFolderObserversForGUID(aFolderGUID, false); + if (!list) return; // we don't even have an entry for that folder + list->RemoveElement(aNode); +} + +nsNavHistoryResult::FolderObserverList* +nsNavHistoryResult::BookmarkFolderObserversForGUID( + const nsACString& aFolderGUID, bool aCreate) { + FolderObserverList* list; + if (mBookmarkFolderObservers.Get(aFolderGUID, &list)) return list; + if (!aCreate) return nullptr; + + // need to create a new list + list = new FolderObserverList; + mBookmarkFolderObservers.InsertOrUpdate(aFolderGUID, list); + return list; +} + +NS_IMETHODIMP +nsNavHistoryResult::GetSortingMode(uint16_t* aSortingMode) { + *aSortingMode = mSortingMode; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::SetSortingMode(uint16_t aSortingMode) { + NS_ENSURE_STATE(mRootNode); + + if (aSortingMode > nsINavHistoryQueryOptions::SORT_BY_FRECENCY_DESCENDING) + return NS_ERROR_INVALID_ARG; + + // Keep everything in sync. + NS_ASSERTION(mOptions, "Options should always be present for a root query"); + + mSortingMode = aSortingMode; + + // If the sorting mode changed to one requiring history details, we must + // ensure to start observing. + bool addedListener = UpdateHistoryDetailsObservers(); + + if (!mRootNode->mExpanded) { + // Need to do this later when node will be expanded. + mNeedsToApplySortingMode = true; + return NS_OK; + } + + if (addedListener) { + // We must do a full refresh because the history details may be stale. + if (mRootNode->IsQuery()) { + return mRootNode->GetAsQuery()->Refresh(); + } + if (mRootNode->IsFolder()) { + return mRootNode->GetAsFolder()->Refresh(); + } + } + + // Actually do sorting. + nsNavHistoryContainerResultNode::SortComparator comparator = + nsNavHistoryContainerResultNode::GetSortingComparator(aSortingMode); + if (comparator) { + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_ERROR_OUT_OF_MEMORY); + mRootNode->RecursiveSort(comparator); + } + + NOTIFY_RESULT_OBSERVERS(this, SortingChanged(aSortingMode)); + NOTIFY_RESULT_OBSERVERS(this, InvalidateContainer(mRootNode)); + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::AddObserver(nsINavHistoryResultObserver* aObserver, + bool aOwnsWeak) { + NS_ENSURE_ARG(aObserver); + nsresult rv = mObservers.AppendWeakElementUnlessExists(aObserver, aOwnsWeak); + NS_ENSURE_SUCCESS(rv, rv); + + UpdateHistoryDetailsObservers(); + + rv = aObserver->SetResult(this); + NS_ENSURE_SUCCESS(rv, rv); + + // If we are batching, notify a fake batch start to the observers. + // Not doing so would then notify a not coupled batch end. + if (IsBatching()) { + NOTIFY_RESULT_OBSERVERS(this, Batching(true)); + } + + if (!mRootNode->IsQuery() || + mRootNode->GetAsQuery()->mLiveUpdate != QUERYUPDATE_NONE) { + // Pretty much all the views show favicons, thus observe changes to them. + AutoTArray events; + events.AppendElement(PlacesEventType::Favicon_changed); + PlacesObservers::AddListener(events, this); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::RemoveObserver(nsINavHistoryResultObserver* aObserver) { + NS_ENSURE_ARG(aObserver); + nsresult rv = mObservers.RemoveWeakElement(aObserver); + NS_ENSURE_SUCCESS(rv, rv); + UpdateHistoryDetailsObservers(); + return NS_OK; +} + +bool nsNavHistoryResult::UpdateHistoryDetailsObservers() { + bool skipHistoryDetailsNotifications = false; + // One observer set to true is enough to set mObserversWantHistoryDetails. + for (uint32_t i = 0; + i < mObservers.Length() && !skipHistoryDetailsNotifications; ++i) { + const nsCOMPtr& entry = + mObservers.ElementAt(i).GetValue(); + if (entry) { + entry->GetSkipHistoryDetailsNotifications( + &skipHistoryDetailsNotifications); + } + } + + mObserversWantHistoryDetails = !skipHistoryDetailsNotifications; + // If one observer wants history details we may have to add the listener. + if (!CanSkipHistoryDetailsNotifications()) { + if (!mIsHistoryDetailsObserver) { + AutoTArray events; + events.AppendElement(PlacesEventType::Page_visited); + events.AppendElement(PlacesEventType::Page_title_changed); + events.AppendElement(PlacesEventType::Page_removed); + PlacesObservers::AddListener(events, this); + mIsHistoryDetailsObserver = true; + return true; + } + } else { + AutoTArray events; + events.AppendElement(PlacesEventType::Page_visited); + events.AppendElement(PlacesEventType::Page_title_changed); + events.AppendElement(PlacesEventType::Page_removed); + PlacesObservers::RemoveListener(events, this); + mIsHistoryDetailsObserver = false; + } + return false; +} + +NS_IMETHODIMP +nsNavHistoryResult::GetSuppressNotifications(bool* _retval) { + *_retval = mSuppressNotifications; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::SetSuppressNotifications(bool aSuppressNotifications) { + mSuppressNotifications = aSuppressNotifications; + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::GetRoot(nsINavHistoryContainerResultNode** aRoot) { + if (!mRootNode) { + MOZ_ASSERT_UNREACHABLE("Root is null"); + *aRoot = nullptr; + return NS_ERROR_FAILURE; + } + RefPtr node(mRootNode); + node.forget(aRoot); + return NS_OK; +} + +void nsNavHistoryResult::requestRefresh( + nsNavHistoryContainerResultNode* aContainer) { + // Don't add twice the same container. + if (mRefreshParticipants.IndexOf(aContainer) == + ContainerObserverList::NoIndex) + mRefreshParticipants.AppendElement(aContainer); +} + +// Here, it is important that we create a COPY of the observer array. Some +// observers will requery themselves, which may cause the observer array to +// be modified or added to. +#define ENUMERATE_BOOKMARK_FOLDER_OBSERVERS(_folderGUID, _functionCall) \ + PR_BEGIN_MACRO \ + FolderObserverList* _fol = \ + BookmarkFolderObserversForGUID(_folderGUID, false); \ + if (_fol) { \ + FolderObserverList _listCopy(_fol->Clone()); \ + for (uint32_t _fol_i = 0; _fol_i < _listCopy.Length(); ++_fol_i) { \ + if (_listCopy[_fol_i]) _listCopy[_fol_i]->_functionCall; \ + } \ + } \ + PR_END_MACRO +#define ENUMERATE_LIST_OBSERVERS(_listType, _functionCall, _observersList, \ + _conditionCall) \ + PR_BEGIN_MACRO \ + _listType _listCopy(_observersList.Clone()); \ + for (uint32_t _obs_i = 0; _obs_i < _listCopy.Length(); ++_obs_i) { \ + if (_listCopy[_obs_i] && _listCopy[_obs_i]->_conditionCall) \ + _listCopy[_obs_i]->_functionCall; \ + } \ + PR_END_MACRO +#define ENUMERATE_QUERY_OBSERVERS(_functionCall, _observersList, \ + _conditionCall) \ + ENUMERATE_LIST_OBSERVERS(QueryObserverList, _functionCall, _observersList, \ + _conditionCall) +#define ENUMERATE_ALL_BOOKMARKS_OBSERVERS(_functionCall) \ + ENUMERATE_QUERY_OBSERVERS(_functionCall, mAllBookmarksObservers, IsQuery()) +#define ENUMERATE_HISTORY_OBSERVERS(_functionCall) \ + ENUMERATE_QUERY_OBSERVERS(_functionCall, mHistoryObservers, IsQuery()) +#define ENUMERATE_MOBILE_PREF_OBSERVERS(_functionCall) \ + ENUMERATE_QUERY_OBSERVERS(_functionCall, mMobilePrefObservers, IsQuery()) +#define ENUMERATE_BOOKMARK_CHANGED_OBSERVERS(_folderGUID, _targetId, \ + _functionCall) \ + PR_BEGIN_MACRO \ + ENUMERATE_ALL_BOOKMARKS_OBSERVERS(_functionCall); \ + FolderObserverList* _fol = \ + BookmarkFolderObserversForGUID(_folderGUID, false); \ + \ + /* \ + * Note: folder-nodes set their own bookmark observer only once they're \ + * opened, meaning we cannot optimize this code path for changes done to \ + * folder-nodes. \ + */ \ + \ + for (uint32_t _fol_i = 0; _fol && _fol_i < _fol->Length(); ++_fol_i) { \ + RefPtr _folder = _fol->ElementAt(_fol_i); \ + if (_folder) { \ + uint32_t _nodeIndex; \ + RefPtr _node = \ + _folder->FindChildById(_targetId, &_nodeIndex); \ + bool _excludeItems = _folder->mOptions->ExcludeItems(); \ + if (_node && \ + (!_excludeItems || !(_node->IsURI() || _node->IsSeparator())) && \ + _folder->StartIncrementalUpdate()) { \ + _node->_functionCall; \ + } \ + } \ + } \ + \ + PR_END_MACRO + +#define NOTIFY_REFRESH_PARTICIPANTS() \ + PR_BEGIN_MACRO \ + ENUMERATE_LIST_OBSERVERS(ContainerObserverList, Refresh(), \ + mRefreshParticipants, IsContainer()); \ + mRefreshParticipants.Clear(); \ + PR_END_MACRO + +NS_IMETHODIMP +nsNavHistoryResult::OnBeginUpdateBatch() { + // Since we could be observing both history and bookmarks, it's possible both + // notify the batch. We can safely ignore nested calls. + if (++mBatchInProgress == 1) { + ENUMERATE_HISTORY_OBSERVERS(OnBeginUpdateBatch()); + ENUMERATE_ALL_BOOKMARKS_OBSERVERS(OnBeginUpdateBatch()); + + NOTIFY_RESULT_OBSERVERS(this, Batching(true)); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsNavHistoryResult::OnEndUpdateBatch() { + // Since we could be observing both history and bookmarks, it's possible both + // notify the batch. We can safely ignore nested calls. + // Notice it's possible we are notified OnEndUpdateBatch more times than + // onBeginUpdateBatch, since the result could be created in the middle of + // nested batches. + if (--mBatchInProgress == 0) { + ENUMERATE_HISTORY_OBSERVERS(OnEndUpdateBatch()); + ENUMERATE_ALL_BOOKMARKS_OBSERVERS(OnEndUpdateBatch()); + + NOTIFY_REFRESH_PARTICIPANTS(); + NOTIFY_RESULT_OBSERVERS(this, Batching(false)); + } + + return NS_OK; +} + +nsresult nsNavHistoryResult::OnVisit(nsIURI* aURI, int64_t aVisitId, + PRTime aTime, uint32_t aTransitionType, + const nsACString& aGUID, bool aHidden, + uint32_t aVisitCount, + const nsAString& aLastKnownTitle) { + NS_ENSURE_ARG(aURI); + + // Embed visits are never shown in our views. + if (aTransitionType == nsINavHistoryService::TRANSITION_EMBED) { + return NS_OK; + } + + uint32_t added = 0; + + ENUMERATE_HISTORY_OBSERVERS( + OnVisit(aURI, aVisitId, aTime, aTransitionType, aHidden, &added)); + + // When we add visits, we don't bother telling the world that the title + // 'changed' from nothing to the first title we ever see for a history entry. + // Our consumers here might still care, though, so we have to tell them, but + // only for the first visit we add. Subsequent changes will get an usual + // ontitlechanged notification. + if (!aLastKnownTitle.IsVoid() && aVisitCount == 1) { + ENUMERATE_HISTORY_OBSERVERS(OnTitleChanged(aURI, aLastKnownTitle, aGUID)); + } + + if (!mRootNode->mExpanded) return NS_OK; + + // If this visit is accepted by an overlapped container, and not all + // overlapped containers are visible, we should still call Refresh if the + // visit falls into any of them. + bool todayIsMissing = false; + uint32_t resultType = mRootNode->mOptions->ResultType(); + if (resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY) { + uint32_t childCount; + nsresult rv = mRootNode->GetChildCount(&childCount); + NS_ENSURE_SUCCESS(rv, rv); + if (childCount) { + nsCOMPtr firstChild; + rv = mRootNode->GetChild(0, getter_AddRefs(firstChild)); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString title; + rv = firstChild->GetTitle(title); + NS_ENSURE_SUCCESS(rv, rv); + nsNavHistory* history = nsNavHistory::GetHistoryService(); + NS_ENSURE_TRUE(history, NS_OK); + nsAutoCString todayLabel; + history->GetStringFromName("finduri-AgeInDays-is-0", todayLabel); + todayIsMissing = !todayLabel.Equals(title); + } + } + + if (!added || todayIsMissing) { + // None of registered query observers has accepted our URI. This means, + // that a matching query either was not expanded or it does not exist. + if (resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_QUERY || + resultType == nsINavHistoryQueryOptions::RESULTS_AS_DATE_SITE_QUERY) { + // If the visit falls into the Today bucket and the bucket exists, it was + // just not expanded, thus there's no reason to update. + int64_t beginOfToday = nsNavHistory::NormalizeTime( + nsINavHistoryQuery::TIME_RELATIVE_TODAY, 0); + if (todayIsMissing || aTime < beginOfToday) { + (void)mRootNode->GetAsQuery()->Refresh(); + } + return NS_OK; + } + + if (resultType == nsINavHistoryQueryOptions::RESULTS_AS_SITE_QUERY) { + (void)mRootNode->GetAsQuery()->Refresh(); + return NS_OK; + } + + // We are result of a folder node, then we should run through history + // observers that are containers queries and refresh them. + // We use a copy of the observers array since requerying could potentially + // cause changes to the array. + ENUMERATE_QUERY_OBSERVERS(Refresh(), mHistoryObservers, + IsContainersQuery()); + + // Also notify onItemVisited to bookmark folder observers, that are not + // observing history. + if (!mIsHistoryObserver && mRootNode->IsFolder()) { + nsAutoCString spec; + nsresult rv = aURI->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + // Find all the folders containing the visited URI, and notify them. + nsCOMArray nodes; + mRootNode->RecursiveFindURIs(true, mRootNode, spec, &nodes); + for (int32_t i = 0; i < nodes.Count(); ++i) { + nsNavHistoryResultNode* node = nodes[i]; + ENUMERATE_BOOKMARK_FOLDER_OBSERVERS( + node->mParent->mBookmarkGuid, OnItemVisited(aURI, aVisitId, aTime)); + } + } + } + + return NS_OK; +} + +void nsNavHistoryResult::OnIconChanged(nsIURI* aURI, nsIURI* aFaviconURI, + const nsACString& aGUID) { + if (!mRootNode->mExpanded) { + return; + } + // Find all nodes for the given URI and update them. + nsAutoCString spec; + if (NS_SUCCEEDED(aURI->GetSpec(spec))) { + bool onlyOneEntry = + mOptions->QueryType() == + nsINavHistoryQueryOptions::QUERY_TYPE_HISTORY && + mOptions->ResultType() == nsINavHistoryQueryOptions::RESULTS_AS_URI; + mRootNode->UpdateURIs(true, onlyOneEntry, false, spec, setFaviconCallback, + nullptr); + } +} + +bool nsNavHistoryResult::IsBulkPageRemovedEvent( + const PlacesEventSequence& aEvents) { + if (IsBatching() || aEvents.Length() <= MAX_PAGE_REMOVES_BEFORE_REFRESH) { + return false; + } + for (const auto& event : aEvents) { + if (event->Type() != PlacesEventType::Page_removed) return false; + } + return true; +} + +void nsNavHistoryResult::HandlePlacesEvent(const PlacesEventSequence& aEvents) { + if (IsBulkPageRemovedEvent(aEvents)) { + ENUMERATE_HISTORY_OBSERVERS(Refresh()); + return; + } + + for (const auto& event : aEvents) { + switch (event->Type()) { + case PlacesEventType::Favicon_changed: { + const dom::PlacesFavicon* faviconEvent = event->AsPlacesFavicon(); + if (NS_WARN_IF(!faviconEvent)) { + continue; + } + nsCOMPtr uri, faviconUri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), faviconEvent->mUrl)); + if (!uri) { + continue; + } + MOZ_ALWAYS_SUCCEEDS( + NS_NewURI(getter_AddRefs(faviconUri), faviconEvent->mFaviconUrl)); + if (!faviconUri) { + continue; + } + OnIconChanged(uri, faviconUri, faviconEvent->mPageGuid); + break; + } + case PlacesEventType::Page_visited: { + const dom::PlacesVisit* visit = event->AsPlacesVisit(); + if (NS_WARN_IF(!visit)) { + continue; + } + + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), visit->mUrl)); + if (!uri) { + continue; + } + OnVisit(uri, visit->mVisitId, visit->mVisitTime * 1000, + visit->mTransitionType, visit->mPageGuid, visit->mHidden, + visit->mVisitCount, visit->mLastKnownTitle); + break; + } + case PlacesEventType::Bookmark_added: { + const dom::PlacesBookmarkAddition* item = + event->AsPlacesBookmarkAddition(); + if (NS_WARN_IF(!item)) { + continue; + } + + nsCOMPtr uri; + if (item->mItemType == nsINavBookmarksService::TYPE_BOOKMARK) { + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), item->mUrl)); + if (!uri) { + continue; + } + } + + ENUMERATE_BOOKMARK_FOLDER_OBSERVERS( + item->mParentGuid, + OnItemAdded(item->mId, item->mParentId, item->mIndex, + item->mItemType, uri, item->mDateAdded * 1000, + item->mGuid, item->mParentGuid, item->mSource)); + ENUMERATE_HISTORY_OBSERVERS( + OnItemAdded(item->mId, item->mParentId, item->mIndex, + item->mItemType, uri, item->mDateAdded * 1000, + item->mGuid, item->mParentGuid, item->mSource)); + ENUMERATE_ALL_BOOKMARKS_OBSERVERS( + OnItemAdded(item->mId, item->mParentId, item->mIndex, + item->mItemType, uri, item->mDateAdded * 1000, + item->mGuid, item->mParentGuid, item->mSource)); + break; + } + case PlacesEventType::Bookmark_removed: { + const dom::PlacesBookmarkRemoved* item = + event->AsPlacesBookmarkRemoved(); + if (NS_WARN_IF(!item)) { + continue; + } + + nsCOMPtr uri; + + if (item->mIsDescendantRemoval) { + continue; + } + ENUMERATE_BOOKMARK_FOLDER_OBSERVERS( + item->mParentGuid, + OnItemRemoved(item->mId, item->mParentId, item->mIndex, + item->mItemType, uri, item->mGuid, item->mParentGuid, + item->mSource)); + ENUMERATE_ALL_BOOKMARKS_OBSERVERS(OnItemRemoved( + item->mId, item->mParentId, item->mIndex, item->mItemType, uri, + item->mGuid, item->mParentGuid, item->mSource)); + ENUMERATE_HISTORY_OBSERVERS(OnItemRemoved( + item->mId, item->mParentId, item->mIndex, item->mItemType, uri, + item->mGuid, item->mParentGuid, item->mSource)); + break; + } + case PlacesEventType::Bookmark_moved: { + const dom::PlacesBookmarkMoved* item = event->AsPlacesBookmarkMoved(); + if (NS_WARN_IF(!item)) { + continue; + } + + NS_ConvertUTF16toUTF8 url(item->mUrl); + + ENUMERATE_BOOKMARK_FOLDER_OBSERVERS( + item->mOldParentGuid, + OnItemMoved(item->mId, item->mOldIndex, item->mIndex, + item->mItemType, item->mGuid, item->mOldParentGuid, + item->mParentGuid, item->mSource, url)); + if (!item->mParentGuid.Equals(item->mOldParentGuid)) { + ENUMERATE_BOOKMARK_FOLDER_OBSERVERS( + item->mParentGuid, + OnItemMoved(item->mId, item->mOldIndex, item->mIndex, + item->mItemType, item->mGuid, item->mOldParentGuid, + item->mParentGuid, item->mSource, url)); + } + ENUMERATE_ALL_BOOKMARKS_OBSERVERS( + OnItemMoved(item->mId, item->mOldIndex, item->mIndex, + item->mItemType, item->mGuid, item->mOldParentGuid, + item->mParentGuid, item->mSource, url)); + ENUMERATE_HISTORY_OBSERVERS( + OnItemMoved(item->mId, item->mOldIndex, item->mIndex, + item->mItemType, item->mGuid, item->mOldParentGuid, + item->mParentGuid, item->mSource, url)); + break; + } + case PlacesEventType::Bookmark_keyword_changed: { + const dom::PlacesBookmarkKeyword* keywordEvent = + event->AsPlacesBookmarkKeyword(); + if (NS_WARN_IF(!keywordEvent)) { + continue; + } + ENUMERATE_BOOKMARK_CHANGED_OBSERVERS( + keywordEvent->mParentGuid, keywordEvent->mId, + OnItemKeywordChanged(keywordEvent->mId, keywordEvent->mKeyword)); + break; + } + case PlacesEventType::Bookmark_tags_changed: { + const dom::PlacesBookmarkTags* tagsEvent = + event->AsPlacesBookmarkTags(); + if (NS_WARN_IF(!tagsEvent)) { + continue; + } + ENUMERATE_BOOKMARK_CHANGED_OBSERVERS( + tagsEvent->mParentGuid, tagsEvent->mId, + OnItemTagsChanged(tagsEvent->mId, tagsEvent->mUrl)); + break; + } + case PlacesEventType::Bookmark_time_changed: { + const dom::PlacesBookmarkTime* timeEvent = + event->AsPlacesBookmarkTime(); + if (NS_WARN_IF(!timeEvent)) { + continue; + } + ENUMERATE_BOOKMARK_CHANGED_OBSERVERS( + timeEvent->mParentGuid, timeEvent->mId, + OnItemTimeChanged(timeEvent->mId, timeEvent->mGuid, + timeEvent->mDateAdded * 1000, + timeEvent->mLastModified * 1000)); + break; + } + case PlacesEventType::Bookmark_title_changed: { + const dom::PlacesBookmarkTitle* titleEvent = + event->AsPlacesBookmarkTitle(); + if (NS_WARN_IF(!titleEvent)) { + continue; + } + + NS_ConvertUTF16toUTF8 title(titleEvent->mTitle); + ENUMERATE_BOOKMARK_CHANGED_OBSERVERS( + titleEvent->mParentGuid, titleEvent->mId, + OnItemTitleChanged(titleEvent->mId, titleEvent->mGuid, title, + titleEvent->mLastModified * 1000)); + break; + } + case PlacesEventType::Bookmark_url_changed: { + const dom::PlacesBookmarkUrl* urlEvent = event->AsPlacesBookmarkUrl(); + if (NS_WARN_IF(!urlEvent)) { + continue; + } + + NS_ConvertUTF16toUTF8 url(urlEvent->mUrl); + ENUMERATE_BOOKMARK_CHANGED_OBSERVERS( + urlEvent->mParentGuid, urlEvent->mId, + OnItemUrlChanged(urlEvent->mId, urlEvent->mGuid, url, + urlEvent->mLastModified * 1000)); + break; + } + case PlacesEventType::Page_title_changed: { + const PlacesVisitTitle* titleEvent = event->AsPlacesVisitTitle(); + if (NS_WARN_IF(!titleEvent)) { + continue; + } + + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), titleEvent->mUrl)); + if (!uri) { + continue; + } + + ENUMERATE_HISTORY_OBSERVERS( + OnTitleChanged(uri, titleEvent->mTitle, titleEvent->mPageGuid)); + break; + } + case PlacesEventType::History_cleared: { + ENUMERATE_HISTORY_OBSERVERS(OnClearHistory()); + break; + } + case PlacesEventType::Page_removed: { + const PlacesVisitRemoved* removeEvent = event->AsPlacesVisitRemoved(); + if (NS_WARN_IF(!removeEvent)) { + continue; + } + + nsCOMPtr uri; + MOZ_ALWAYS_SUCCEEDS(NS_NewURI(getter_AddRefs(uri), removeEvent->mUrl)); + if (!uri) { + continue; + } + + if (removeEvent->mIsRemovedFromStore) { + ENUMERATE_HISTORY_OBSERVERS(OnPageRemovedFromStore( + uri, removeEvent->mPageGuid, removeEvent->mReason)); + } else { + ENUMERATE_HISTORY_OBSERVERS( + OnPageRemovedVisits(uri, removeEvent->mIsPartialVisistsRemoval, + removeEvent->mPageGuid, removeEvent->mReason, + removeEvent->mTransitionType)); + + if (!removeEvent->mIsPartialVisistsRemoval && mRootNode) { + mRootNode->OnVisitsRemoved(uri); + } + } + + break; + } + case PlacesEventType::Purge_caches: { + mRootNode->Refresh(); + break; + } + default: { + MOZ_ASSERT_UNREACHABLE( + "Receive notification of a type not subscribed to."); + } + } + } +} + +void nsNavHistoryResult::OnMobilePrefChanged() { + ENUMERATE_MOBILE_PREF_OBSERVERS( + OnMobilePrefChanged(Preferences::GetBool(MOBILE_BOOKMARKS_PREF, false))); +} + +void nsNavHistoryResult::OnMobilePrefChangedCallback(const char* prefName, + void* self) { + MOZ_ASSERT(!strcmp(prefName, MOBILE_BOOKMARKS_PREF), + "We only expected Mobile Bookmarks pref change."); + + static_cast(self)->OnMobilePrefChanged(); +} diff --git a/toolkit/components/places/nsNavHistoryResult.h b/toolkit/components/places/nsNavHistoryResult.h new file mode 100644 index 0000000000..5b6d478bdc --- /dev/null +++ b/toolkit/components/places/nsNavHistoryResult.h @@ -0,0 +1,855 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The definitions of objects that make up a history query result set. This file + * should only be included by nsNavHistory.h, include that if you want these + * classes. + */ + +#ifndef nsNavHistoryResult_h_ +#define nsNavHistoryResult_h_ + +#include "INativePlacesEventCallback.h" +#include "nsTArray.h" +#include "nsMaybeWeakPtr.h" +#include "nsInterfaceHashtable.h" +#include "nsTHashMap.h" +#include "nsCycleCollectionParticipant.h" +#include "mozilla/storage.h" +#include "Helpers.h" + +class nsNavHistory; +class nsNavHistoryQuery; +class nsNavHistoryQueryOptions; + +class nsNavHistoryContainerResultNode; +class nsNavHistoryFolderResultNode; +class nsNavHistoryQueryResultNode; + +/** + * hashkey wrapper using int64_t KeyType + * + * @see nsTHashtable::EntryType for specification + * + * This just truncates the 64-bit int to a 32-bit one for using a hash number. + * It is used for bookmark folder IDs, which should be way less than 2^32. + */ +class nsTrimInt64HashKey : public PLDHashEntryHdr { + public: + typedef const int64_t& KeyType; + typedef const int64_t* KeyTypePointer; + + explicit nsTrimInt64HashKey(KeyTypePointer aKey) : mValue(*aKey) {} + nsTrimInt64HashKey(const nsTrimInt64HashKey& toCopy) + : mValue(toCopy.mValue) {} + ~nsTrimInt64HashKey() = default; + + KeyType GetKey() const { return mValue; } + bool KeyEquals(KeyTypePointer aKey) const { return *aKey == mValue; } + + static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } + static PLDHashNumber HashKey(KeyTypePointer aKey) { + return static_cast((*aKey) & UINT32_MAX); + } + enum { ALLOW_MEMMOVE = true }; + + private: + const int64_t mValue; +}; + +// nsNavHistoryResult +// +// nsNavHistory creates this object and fills in mChildren (by getting +// it through GetTopLevel()). Then FilledAllResults() is called to finish +// object initialization. + +#define NS_NAVHISTORYRESULT_IID \ + { \ + 0x455d1d40, 0x1b9b, 0x40e6, { \ + 0xa6, 0x41, 0x8b, 0xb7, 0xe8, 0x82, 0x23, 0x87 \ + } \ + } + +class nsNavHistoryResult final + : public nsSupportsWeakReference, + public nsINavHistoryResult, + public mozilla::places::INativePlacesEventCallback { + public: + NS_DECLARE_STATIC_IID_ACCESSOR(NS_NAVHISTORYRESULT_IID) + + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_NSINAVHISTORYRESULT + NS_DECL_CYCLE_COLLECTION_CLASS_AMBIGUOUS(nsNavHistoryResult, + nsINavHistoryResult) + + void AddHistoryObserver(nsNavHistoryQueryResultNode* aNode); + void AddBookmarkFolderObserver(nsNavHistoryFolderResultNode* aNode, + const nsACString& aFolderGUID); + void AddAllBookmarksObserver(nsNavHistoryQueryResultNode* aNode); + void AddMobilePrefsObserver(nsNavHistoryQueryResultNode* aNode); + void RemoveHistoryObserver(nsNavHistoryQueryResultNode* aNode); + void RemoveBookmarkFolderObserver(nsNavHistoryFolderResultNode* aNode, + const nsACString& aFolderGUID); + void RemoveAllBookmarksObserver(nsNavHistoryQueryResultNode* aNode); + void RemoveMobilePrefsObserver(nsNavHistoryQueryResultNode* aNode); + void StopObserving(); + void EnsureIsObservingBookmarks(); + + nsresult OnVisit(nsIURI* aURI, int64_t aVisitId, PRTime aTime, + uint32_t aTransitionType, const nsACString& aGUID, + bool aHidden, uint32_t aVisitCount, + const nsAString& aLastKnownTitle); + + void OnIconChanged(nsIURI* aURI, nsIURI* aFaviconURI, + const nsACString& aGUID); + + explicit nsNavHistoryResult(nsNavHistoryContainerResultNode* mRoot, + const RefPtr& aQuery, + const RefPtr& aOptions); + + RefPtr mRootNode; + + RefPtr mQuery; + RefPtr mOptions; + + // One of nsNavHistoryQueryOptions.SORY_BY_* This is initialized to + // mOptions.sortingMode, but may be overridden if the user clicks on one of + // the columns. + uint16_t mSortingMode; + // If root node is closed and we try to apply a sortingMode, it would not + // work. So we will apply it when the node will be reopened and populated. + // This var states the fact we need to apply sortingMode in such a situation. + bool mNeedsToApplySortingMode; + + // node observers + bool mIsHistoryObserver; + bool mIsBookmarksObserver; + bool mIsMobilePrefObserver; + + typedef nsTArray > QueryObserverList; + QueryObserverList mHistoryObservers; + QueryObserverList mAllBookmarksObservers; + QueryObserverList mMobilePrefObservers; + + typedef nsTArray > FolderObserverList; + nsTHashMap mBookmarkFolderObservers; + FolderObserverList* BookmarkFolderObserversForGUID(const nsACString& aGUID, + bool aCreate); + + typedef nsTArray > + ContainerObserverList; + + void RecursiveExpandCollapse(nsNavHistoryContainerResultNode* aContainer, + bool aExpand); + + void InvalidateTree(); + + nsMaybeWeakPtrArray mObservers; + bool mSuppressNotifications; + + // Tracks whether observers for history details were added. + bool mIsHistoryDetailsObserver; + // Tracks whether any result observer is interested in history details + // updates. + bool mObserversWantHistoryDetails; + /** + * Updates mObserversWantHistoryDetails when observers are added/removed. + * @returns Whether we started observing for history changes. + */ + bool UpdateHistoryDetailsObservers(); + // Whether NodeHistoryDetailsChanged can be skipped. + bool CanSkipHistoryDetailsNotifications() const; + + ContainerObserverList mRefreshParticipants; + void requestRefresh(nsNavHistoryContainerResultNode* aContainer); + + void HandlePlacesEvent(const PlacesEventSequence& aEvents) override; + + // Optimisation: refreshing containers is much faster than incremental + // updates when handling multiple Page_removed events. + bool IsBulkPageRemovedEvent(const PlacesEventSequence& aEvents); + + void OnMobilePrefChanged(); + + bool IsBatching() const { return mBatchInProgress > 0; }; + + static void OnMobilePrefChangedCallback(const char* prefName, void* self); + + protected: + virtual ~nsNavHistoryResult(); + + private: + // Number of batch processes currently running. IsBatching() returns true if + // this value is greater than or equal to 1. Also, when this value changes to + // 1 from 0, batching() in nsINavHistoryResultObserver is called with + // parameter as true, when changes to 0, that means finishing all batch + // processes, batching() is called with false. + uint32_t mBatchInProgress; + + // Stop all observers upon unlinking. + void StopObservingOnUnlink(); +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsNavHistoryResult, NS_NAVHISTORYRESULT_IID) + +// nsNavHistoryResultNode +// +// This is the base class for every node in a result set. The result itself +// is a node (nsNavHistoryResult inherits from this), as well as every +// leaf and branch on the tree. + +#define NS_NAVHISTORYRESULTNODE_IID \ + { \ + 0x54b61d38, 0x57c1, 0x11da, { \ + 0x95, 0xb8, 0x00, 0x13, 0x21, 0xc9, 0xf6, 0x9e \ + } \ + } + +// These are all the simple getters, they can be used for the result node +// implementation and all subclasses. More complex are GetIcon, GetParent +// (which depends on the definition of container result node), and GetUri +// (which is overridded for lazy construction for some containers). +#define NS_IMPLEMENT_SIMPLE_RESULTNODE \ + NS_IMETHOD GetTitle(nsACString& aTitle) override { \ + aTitle = mTitle; \ + return NS_OK; \ + } \ + NS_IMETHOD GetAccessCount(uint32_t* aAccessCount) override { \ + *aAccessCount = mAccessCount; \ + return NS_OK; \ + } \ + NS_IMETHOD GetTime(PRTime* aTime) override { \ + *aTime = mTime; \ + return NS_OK; \ + } \ + NS_IMETHOD GetIndentLevel(int32_t* aIndentLevel) override { \ + *aIndentLevel = mIndentLevel; \ + return NS_OK; \ + } \ + NS_IMETHOD GetBookmarkIndex(int32_t* aIndex) override { \ + *aIndex = mBookmarkIndex; \ + return NS_OK; \ + } \ + NS_IMETHOD GetDateAdded(PRTime* aDateAdded) override { \ + *aDateAdded = mDateAdded; \ + return NS_OK; \ + } \ + NS_IMETHOD GetLastModified(PRTime* aLastModified) override { \ + *aLastModified = mLastModified; \ + return NS_OK; \ + } \ + NS_IMETHOD GetItemId(int64_t* aId) override { \ + *aId = mItemId; \ + return NS_OK; \ + } + +// This is used by the base classes instead of +// NS_FORWARD_NSINAVHISTORYRESULTNODE(nsNavHistoryResultNode) because they +// need to redefine GetType and GetUri rather than forwarding them. This +// implements all the simple getters instead of forwarding because they are so +// short and we can save a virtual function call. +// +// (GetUri is redefined only by QueryResultNode and FolderResultNode because +// the query might not necessarily be parsed. The rest just return the node's +// buffer.) +#define NS_FORWARD_COMMON_RESULTNODE_TO_BASE \ + NS_IMPLEMENT_SIMPLE_RESULTNODE \ + NS_IMETHOD GetIcon(nsACString& aIcon) override { \ + return nsNavHistoryResultNode::GetIcon(aIcon); \ + } \ + NS_IMETHOD GetParent(nsINavHistoryContainerResultNode** aParent) override { \ + return nsNavHistoryResultNode::GetParent(aParent); \ + } \ + NS_IMETHOD GetParentResult(nsINavHistoryResult** aResult) override { \ + return nsNavHistoryResultNode::GetParentResult(aResult); \ + } \ + NS_IMETHOD GetTags(nsAString& aTags) override { \ + return nsNavHistoryResultNode::GetTags(aTags); \ + } \ + NS_IMETHOD GetPageGuid(nsACString& aPageGuid) override { \ + return nsNavHistoryResultNode::GetPageGuid(aPageGuid); \ + } \ + NS_IMETHOD GetBookmarkGuid(nsACString& aBookmarkGuid) override { \ + return nsNavHistoryResultNode::GetBookmarkGuid(aBookmarkGuid); \ + } \ + NS_IMETHOD GetVisitId(int64_t* aVisitId) override { \ + return nsNavHistoryResultNode::GetVisitId(aVisitId); \ + } \ + NS_IMETHOD GetFromVisitId(int64_t* aFromVisitId) override { \ + return nsNavHistoryResultNode::GetFromVisitId(aFromVisitId); \ + } \ + NS_IMETHOD GetVisitType(uint32_t* aVisitType) override { \ + return nsNavHistoryResultNode::GetVisitType(aVisitType); \ + } + +class nsNavHistoryResultNode : public nsINavHistoryResultNode { + public: + nsNavHistoryResultNode(const nsACString& aURI, const nsACString& aTitle, + uint32_t aAccessCount, PRTime aTime); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_NAVHISTORYRESULTNODE_IID) + + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_CLASS(nsNavHistoryResultNode) + + NS_IMPLEMENT_SIMPLE_RESULTNODE + NS_IMETHOD GetIcon(nsACString& aIcon) override; + NS_IMETHOD GetParent(nsINavHistoryContainerResultNode** aParent) override; + NS_IMETHOD GetParentResult(nsINavHistoryResult** aResult) override; + NS_IMETHOD GetType(uint32_t* type) override { + *type = nsNavHistoryResultNode::RESULT_TYPE_URI; + return NS_OK; + } + NS_IMETHOD GetUri(nsACString& aURI) override { + aURI = mURI; + return NS_OK; + } + NS_IMETHOD GetTags(nsAString& aTags) override; + NS_IMETHOD GetPageGuid(nsACString& aPageGuid) override; + NS_IMETHOD GetBookmarkGuid(nsACString& aBookmarkGuid) override; + NS_IMETHOD GetVisitId(int64_t* aVisitId) override; + NS_IMETHOD GetFromVisitId(int64_t* aFromVisitId) override; + NS_IMETHOD GetVisitType(uint32_t* aVisitType) override; + + virtual void OnRemoving(); + + nsresult OnItemKeywordChanged(int64_t aItemId, const nsACString& aKeyword); + nsresult OnItemTagsChanged(int64_t aItemId, const nsAString& aURL); + nsresult OnItemTimeChanged(int64_t aItemId, const nsACString& aGUID, + PRTime aDateAdded, PRTime aLastModified); + nsresult OnItemTitleChanged(int64_t aItemId, const nsACString& aGUID, + const nsACString& aTitle, PRTime aLastModified); + nsresult OnItemUrlChanged(int64_t aItemId, const nsACString& aGUID, + const nsACString& aURL, PRTime aLastModified); + + virtual nsresult OnMobilePrefChanged(bool newValue) { return NS_OK; }; + + nsresult OnVisitsRemoved(); + + protected: + virtual ~nsNavHistoryResultNode() = default; + + public: + nsNavHistoryResult* GetResult(); + + // These functions test the type. We don't use a virtual function since that + // would take a vtable slot for every one of (potentially very many) nodes. + // Note that GetType() already has a vtable slot because its on the iface. + bool IsTypeContainer(uint32_t type) { + return type == nsINavHistoryResultNode::RESULT_TYPE_QUERY || + type == nsINavHistoryResultNode::RESULT_TYPE_FOLDER || + type == nsINavHistoryResultNode::RESULT_TYPE_FOLDER_SHORTCUT; + } + bool IsContainer() { + uint32_t type; + GetType(&type); + return IsTypeContainer(type); + } + static bool IsTypeURI(uint32_t type) { + return type == nsINavHistoryResultNode::RESULT_TYPE_URI; + } + bool IsURI() { + uint32_t type; + GetType(&type); + return IsTypeURI(type); + } + static bool IsTypeFolder(uint32_t type) { + return type == nsINavHistoryResultNode::RESULT_TYPE_FOLDER || + type == nsINavHistoryResultNode::RESULT_TYPE_FOLDER_SHORTCUT; + } + bool IsFolder() { + uint32_t type; + GetType(&type); + return IsTypeFolder(type); + } + static bool IsTypeQuery(uint32_t type) { + return type == nsINavHistoryResultNode::RESULT_TYPE_QUERY; + } + bool IsQuery() { + uint32_t type; + GetType(&type); + return IsTypeQuery(type); + } + bool IsSeparator() { + uint32_t type; + GetType(&type); + return type == nsINavHistoryResultNode::RESULT_TYPE_SEPARATOR; + } + nsNavHistoryContainerResultNode* GetAsContainer() { + NS_ASSERTION(IsContainer(), "Not a container"); + return reinterpret_cast(this); + } + nsNavHistoryFolderResultNode* GetAsFolder() { + NS_ASSERTION(IsFolder(), "Not a folder"); + return reinterpret_cast(this); + } + nsNavHistoryQueryResultNode* GetAsQuery() { + NS_ASSERTION(IsQuery(), "Not a query"); + return reinterpret_cast(this); + } + + RefPtr mParent; + nsCString mURI; // not necessarily valid for containers, call GetUri + nsCString mTitle; + nsString mTags; + bool mAreTagsSorted; + uint32_t mAccessCount; + int64_t mTime; + int32_t mBookmarkIndex; + int64_t mItemId; + int64_t mVisitId; + int64_t mFromVisitId; + PRTime mDateAdded; + PRTime mLastModified; + + // The indent level of this node. The root node will have a value of -1. The + // root's children will have a value of 0, and so on. + int32_t mIndentLevel; + + // Frecency of the page. Valid only for URI nodes. + int32_t mFrecency; + + // Hidden status of the page. Valid only for URI nodes. + bool mHidden; + + // Transition type used when this node represents a single visit. + uint32_t mTransitionType; + + // Unique Id of the page. + nsCString mPageGuid; + + // Unique Id of the bookmark. + nsCString mBookmarkGuid; +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsNavHistoryResultNode, + NS_NAVHISTORYRESULTNODE_IID) + +// nsNavHistoryContainerResultNode +// +// This is the base class for all nodes that can have children. It is +// overridden for nodes that are dynamically populated such as queries and +// folders. It is used directly for simple containers such as host groups +// in history views. + +// derived classes each provide their own implementation of has children and +// forward the rest to us using this macro +#define NS_FORWARD_CONTAINERNODE_EXCEPT_HASCHILDREN \ + NS_IMETHOD GetState(uint16_t* _state) override { \ + return nsNavHistoryContainerResultNode::GetState(_state); \ + } \ + NS_IMETHOD GetContainerOpen(bool* aContainerOpen) override { \ + return nsNavHistoryContainerResultNode::GetContainerOpen(aContainerOpen); \ + } \ + NS_IMETHOD SetContainerOpen(bool aContainerOpen) override { \ + return nsNavHistoryContainerResultNode::SetContainerOpen(aContainerOpen); \ + } \ + NS_IMETHOD GetChildCount(uint32_t* aChildCount) override { \ + return nsNavHistoryContainerResultNode::GetChildCount(aChildCount); \ + } \ + NS_IMETHOD GetChild(uint32_t index, nsINavHistoryResultNode** _retval) \ + override { \ + return nsNavHistoryContainerResultNode::GetChild(index, _retval); \ + } \ + NS_IMETHOD GetChildIndex(nsINavHistoryResultNode* aNode, uint32_t* _retval) \ + override { \ + return nsNavHistoryContainerResultNode::GetChildIndex(aNode, _retval); \ + } + +#define NS_NAVHISTORYCONTAINERRESULTNODE_IID \ + { \ + 0x6e3bf8d3, 0x22aa, 0x4065, { \ + 0x86, 0xbc, 0x37, 0x46, 0xb5, 0xb3, 0x2c, 0xe8 \ + } \ + } + +class nsNavHistoryContainerResultNode + : public nsNavHistoryResultNode, + public nsINavHistoryContainerResultNode { + public: + nsNavHistoryContainerResultNode(const nsACString& aURI, + const nsACString& aTitle, PRTime aTime, + uint32_t aContainerType, + nsNavHistoryQueryOptions* aOptions); + + virtual nsresult Refresh(); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_NAVHISTORYCONTAINERRESULTNODE_IID) + + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(nsNavHistoryContainerResultNode, + nsNavHistoryResultNode) + NS_FORWARD_COMMON_RESULTNODE_TO_BASE + NS_IMETHOD GetType(uint32_t* type) override { + *type = mContainerType; + return NS_OK; + } + NS_IMETHOD GetUri(nsACString& aURI) override { + aURI = mURI; + return NS_OK; + } + NS_DECL_NSINAVHISTORYCONTAINERRESULTNODE + + public: + virtual void OnRemoving() override; + + nsresult OnVisitsRemoved(nsIURI* aURI); + + bool AreChildrenVisible(); + + // Overridded by descendents to populate. + virtual nsresult OpenContainer(); + nsresult CloseContainer(bool aSuppressNotifications = false); + + virtual nsresult OpenContainerAsync(); + + // This points to the result that owns this container. All containers have + // their result pointer set so we can quickly get to the result without having + // to walk the tree. Yet, this also saves us from storing a million pointers + // for every leaf node to the result. + RefPtr mResult; + + // For example, RESULT_TYPE_QUERY. Query and Folder results override GetType + // so this is not used, but is still kept in sync. + uint32_t mContainerType; + + // When there are children, this stores the open state in the tree + // this is set to the default in the constructor. + bool mExpanded; + + // Filled in by the result type generator in nsNavHistory. + nsCOMArray mChildren; + + // mOriginalOptions is the options object used to _define_ this specific + // container node. It may differ from mOptions, that is the options used + // to _fill_ this container node, because mOptions may be modified by + // the direct parent of this container node, see SetAsParentOfNode. For + // example, if the parent has excludeItems, options will have it too, even if + // originally this object was not defined with that option. + RefPtr mOriginalOptions; + RefPtr mOptions; + + void FillStats(); + // Sets this container as parent of aNode, propagating the appropriate + // options. + void SetAsParentOfNode(nsNavHistoryResultNode* aNode); + nsresult ReverseUpdateStats(int32_t aAccessCountChange); + + // Sorting methods. + typedef nsCOMArray::TComparatorFunc SortComparator; + virtual uint16_t GetSortType(); + + static SortComparator GetSortingComparator(uint16_t aSortType); + virtual void RecursiveSort(SortComparator aComparator); + uint32_t FindInsertionPoint(nsNavHistoryResultNode* aNode, + SortComparator aComparator, bool* aItemExists); + bool DoesChildNeedResorting(uint32_t aIndex, SortComparator aComparator); + + static int32_t SortComparison_StringLess(const nsAString& a, + const nsAString& b); + + static int32_t SortComparison_Bookmark(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_TitleLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_TitleGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_DateLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_DateGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_URILess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_URIGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_VisitCountLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_VisitCountGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_DateAddedLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_DateAddedGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_LastModifiedLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_LastModifiedGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_TagsLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_TagsGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_FrecencyLess(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + static int32_t SortComparison_FrecencyGreater(nsNavHistoryResultNode* a, + nsNavHistoryResultNode* b, + void* closure); + + // finding children: THESE DO NOT ADDREF + nsNavHistoryResultNode* FindChildByURI(const nsACString& aSpec, + uint32_t* aNodeIndex); + void FindChildrenByURI(const nsCString& aSpec, + nsCOMArray* aMatches); + // returns the index of the given node, -1 if not found + int32_t FindChild(nsNavHistoryResultNode* aNode) { + return mChildren.IndexOf(aNode); + } + + nsNavHistoryResultNode* FindChildByGuid(const nsACString& guid, + int32_t* nodeIndex); + + nsNavHistoryResultNode* FindChildById(int64_t aItemId, uint32_t* aNodeIndex); + + nsresult InsertChildAt(nsNavHistoryResultNode* aNode, int32_t aIndex); + nsresult InsertSortedChild(nsNavHistoryResultNode* aNode, + bool aIgnoreDuplicates = false); + bool EnsureItemPosition(uint32_t aIndex); + + nsresult RemoveChildAt(int32_t aIndex); + + void RecursiveFindURIs(bool aOnlyOne, + nsNavHistoryContainerResultNode* aContainer, + const nsCString& aSpec, + nsCOMArray* aMatches); + bool UpdateURIs(bool aRecursive, bool aOnlyOne, bool aUpdateSort, + const nsCString& aSpec, + nsresult (*aCallback)(nsNavHistoryResultNode*, const void*, + const nsNavHistoryResult*), + const void* aClosure); + nsresult ChangeTitles(nsIURI* aURI, const nsACString& aNewTitle, + bool aRecursive, bool aOnlyOne); + + protected: + virtual ~nsNavHistoryContainerResultNode(); + + enum AsyncCanceledState { NOT_CANCELED, CANCELED, CANCELED_RESTART_NEEDED }; + + void CancelAsyncOpen(bool aRestart); + nsresult NotifyOnStateChange(uint16_t aOldState); + + nsCOMPtr mAsyncPendingStmt; + AsyncCanceledState mAsyncCanceledState; +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsNavHistoryContainerResultNode, + NS_NAVHISTORYCONTAINERRESULTNODE_IID) + +// nsNavHistoryQueryResultNode +// +// Overridden container type for complex queries over history and/or +// bookmarks. This keeps itself in sync by listening to history and +// bookmark notifications. + +class nsNavHistoryQueryResultNode final + : public nsNavHistoryContainerResultNode, + public nsINavHistoryQueryResultNode { + public: + nsNavHistoryQueryResultNode(const nsACString& aTitle, PRTime aTime, + const nsACString& aQueryURI, + const RefPtr& aQuery, + const RefPtr& aOptions); + + NS_DECL_ISUPPORTS_INHERITED + NS_FORWARD_COMMON_RESULTNODE_TO_BASE + NS_IMETHOD GetType(uint32_t* type) override { + *type = nsNavHistoryResultNode::RESULT_TYPE_QUERY; + return NS_OK; + } + NS_IMETHOD GetUri(nsACString& aURI) override; // does special lazy creation + NS_FORWARD_CONTAINERNODE_EXCEPT_HASCHILDREN + NS_IMETHOD GetHasChildren(bool* aHasChildren) override; + NS_DECL_NSINAVHISTORYQUERYRESULTNODE + + virtual nsresult OnMobilePrefChanged(bool newValue) override; + + bool CanExpand(); + bool IsContainersQuery(); + + virtual nsresult OpenContainer() override; + + nsresult OnItemAdded(int64_t aItemId, int64_t aParentId, int32_t aIndex, + uint16_t aItemType, nsIURI* aURI, PRTime aDateAdded, + const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource); + nsresult OnItemRemoved(int64_t aItemId, int64_t aParentFolder, int32_t aIndex, + uint16_t aItemType, nsIURI* aURI, + const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource); + nsresult OnItemMoved(int64_t aFolder, int32_t aOldIndex, int32_t aNewIndex, + uint16_t aItemType, const nsACString& aGUID, + const nsACString& aOldParentGUID, + const nsACString& aNewParentGUID, uint16_t aSource, + const nsACString& aURI); + nsresult OnItemTagsChanged(int64_t aItemId, const nsAString& aURL); + nsresult OnItemTimeChanged(int64_t aItemId, const nsACString& aGUID, + PRTime aDateAdded, PRTime aLastModified); + nsresult OnItemTitleChanged(int64_t aItemId, const nsACString& aGUID, + const nsACString& aTitle, PRTime aLastModified); + nsresult OnItemUrlChanged(int64_t aItemId, const nsACString& aGUID, + const nsACString& aURL, PRTime aLastModified); + + // The internal version has an output aAdded parameter, it is incremented by + // query nodes when the visited uri belongs to them. If no such query exists, + // the history result creates a new query node dynamically. + nsresult OnVisit(nsIURI* aURI, int64_t aVisitId, PRTime aTime, + uint32_t aTransitionType, bool aHidden, uint32_t* aAdded); + nsresult OnTitleChanged(nsIURI* aURI, const nsAString& aPageTitle, + const nsACString& aGUID); + nsresult OnClearHistory(); + nsresult OnPageRemovedFromStore(nsIURI* aURI, const nsACString& aGUID, + uint16_t aReason); + nsresult OnPageRemovedVisits(nsIURI* aURI, bool aPartialRemoval, + const nsACString& aGUID, uint16_t aReason, + uint32_t aTransitionType); + + virtual void OnRemoving() override; + + nsresult OnBeginUpdateBatch(); + nsresult OnEndUpdateBatch(); + + public: + RefPtr mQuery; + uint32_t mLiveUpdate; // one of QUERYUPDATE_* in nsNavHistory.h + bool mHasSearchTerms; + + // safe options getter, ensures query is parsed + nsNavHistoryQueryOptions* Options(); + + // this indicates whether the query contents are valid, they don't go away + // after the container is closed until a notification comes in + bool mContentsValid; + + nsresult FillChildren(); + void ClearChildren(bool unregister); + nsresult Refresh() override; + + virtual uint16_t GetSortType() override; + virtual void RecursiveSort(SortComparator aComparator) override; + + nsresult NotifyIfTagsChanged(nsIURI* aURI); + + uint32_t mBatchChanges; + + // Tracks transition type filters. + nsTArray mTransitions; + + protected: + virtual ~nsNavHistoryQueryResultNode(); +}; + +// nsNavHistoryFolderResultNode +// +// Overridden container type for bookmark folders. It will keep the contents +// of the folder in sync with the bookmark service. + +class nsNavHistoryFolderResultNode final + : public nsNavHistoryContainerResultNode, + public nsINavHistoryQueryResultNode, + public mozilla::places::WeakAsyncStatementCallback { + public: + nsNavHistoryFolderResultNode(const nsACString& aTitle, + nsNavHistoryQueryOptions* options, + int64_t aFolderId); + + NS_DECL_ISUPPORTS_INHERITED + NS_FORWARD_COMMON_RESULTNODE_TO_BASE + NS_IMETHOD GetType(uint32_t* type) override { + if (mTargetFolderItemId != mItemId) { + *type = nsNavHistoryResultNode::RESULT_TYPE_FOLDER_SHORTCUT; + } else { + *type = nsNavHistoryResultNode::RESULT_TYPE_FOLDER; + } + return NS_OK; + } + NS_IMETHOD GetUri(nsACString& aURI) override; + NS_FORWARD_CONTAINERNODE_EXCEPT_HASCHILDREN + NS_IMETHOD GetHasChildren(bool* aHasChildren) override; + NS_DECL_NSINAVHISTORYQUERYRESULTNODE + + virtual nsresult OpenContainer() override; + + virtual nsresult OpenContainerAsync() override; + NS_DECL_ASYNCSTATEMENTCALLBACK + + nsresult OnItemAdded(int64_t aItemId, int64_t aParentId, int32_t aIndex, + uint16_t aItemType, nsIURI* aURI, PRTime aDateAdded, + const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource); + nsresult OnItemRemoved(int64_t aItemId, int64_t aParentFolder, int32_t aIndex, + uint16_t aItemType, nsIURI* aURI, + const nsACString& aGUID, const nsACString& aParentGUID, + uint16_t aSource); + nsresult OnItemMoved(int64_t aFolder, int32_t aOldIndex, int32_t aNewIndex, + uint16_t aItemType, const nsACString& aGUID, + const nsACString& aOldParentGUID, + const nsACString& aNewParentGUID, uint16_t aSource, + const nsACString& aURI); + nsresult OnItemVisited(nsIURI* aURI, int64_t aVisitId, PRTime aTime); + + virtual void OnRemoving() override; + + // this indicates whether the folder contents are valid, they don't go away + // after the container is closed until a notification comes in + bool mContentsValid; + + // If the node is generated from a place:folder=X query, this is the target + // folder id and GUID. For regular folder nodes, they are set to the same + // values as mItemId and mBookmarkGuid. For more complex queries, they are set + // to -1/an empty string. + int64_t mTargetFolderItemId; + nsCString mTargetFolderGuid; + + nsresult FillChildren(); + void ClearChildren(bool aUnregister); + nsresult Refresh() override; + + bool StartIncrementalUpdate(); + void ReindexRange(int32_t aStartIndex, int32_t aEndIndex, int32_t aDelta); + + nsresult OnBeginUpdateBatch(); + nsresult OnEndUpdateBatch(); + + protected: + virtual ~nsNavHistoryFolderResultNode(); + + private: + nsresult OnChildrenFilled(); + void EnsureRegisteredAsFolderObserver(); + nsresult FillChildrenAsync(); + + bool mIsRegisteredFolderObserver; + int32_t mAsyncBookmarkIndex; +}; + +// nsNavHistorySeparatorResultNode +// +// Separator result nodes do not hold any data. +class nsNavHistorySeparatorResultNode : public nsNavHistoryResultNode { + public: + nsNavHistorySeparatorResultNode(); + + NS_IMETHOD GetType(uint32_t* type) override { + *type = nsNavHistoryResultNode::RESULT_TYPE_SEPARATOR; + return NS_OK; + } +}; + +#endif // nsNavHistoryResult_h_ diff --git a/toolkit/components/places/nsPlacesIndexes.h b/toolkit/components/places/nsPlacesIndexes.h new file mode 100644 index 0000000000..c073691c19 --- /dev/null +++ b/toolkit/components/places/nsPlacesIndexes.h @@ -0,0 +1,118 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsPlacesIndexes_h__ +#define nsPlacesIndexes_h__ + +#define CREATE_PLACES_IDX(__name, __table, __columns, __type) \ + nsLiteralCString("CREATE " __type " INDEX IF NOT EXISTS " __table "_" __name \ + " ON " __table " (" __columns ")") + +/** + * moz_places + */ +#define CREATE_IDX_MOZ_PLACES_URL_HASH \ + CREATE_PLACES_IDX("url_hashindex", "moz_places", "url_hash", "") + +#define CREATE_IDX_MOZ_PLACES_REVHOST \ + CREATE_PLACES_IDX("hostindex", "moz_places", "rev_host", "") + +#define CREATE_IDX_MOZ_PLACES_VISITCOUNT \ + CREATE_PLACES_IDX("visitcount", "moz_places", "visit_count", "") + +#define CREATE_IDX_MOZ_PLACES_FRECENCY \ + CREATE_PLACES_IDX("frecencyindex", "moz_places", "frecency", "") + +#define CREATE_IDX_MOZ_PLACES_LASTVISITDATE \ + CREATE_PLACES_IDX("lastvisitdateindex", "moz_places", "last_visit_date", "") + +#define CREATE_IDX_MOZ_PLACES_GUID \ + CREATE_PLACES_IDX("guid_uniqueindex", "moz_places", "guid", "UNIQUE") + +#define CREATE_IDX_MOZ_PLACES_ORIGIN_ID \ + CREATE_PLACES_IDX("originidindex", "moz_places", "origin_id", "") + +#define CREATE_IDX_MOZ_PLACES_ALT_FRECENCY \ + CREATE_PLACES_IDX("altfrecencyindex", "moz_places", "alt_frecency", "") + +/** + * moz_historyvisits + */ + +#define CREATE_IDX_MOZ_HISTORYVISITS_PLACEDATE \ + CREATE_PLACES_IDX("placedateindex", "moz_historyvisits", \ + "place_id, visit_date", "") + +#define CREATE_IDX_MOZ_HISTORYVISITS_FROMVISIT \ + CREATE_PLACES_IDX("fromindex", "moz_historyvisits", "from_visit", "") + +#define CREATE_IDX_MOZ_HISTORYVISITS_VISITDATE \ + CREATE_PLACES_IDX("dateindex", "moz_historyvisits", "visit_date", "") + +/** + * moz_bookmarks + */ + +#define CREATE_IDX_MOZ_BOOKMARKS_PLACETYPE \ + CREATE_PLACES_IDX("itemindex", "moz_bookmarks", "fk, type", "") + +#define CREATE_IDX_MOZ_BOOKMARKS_PARENTPOSITION \ + CREATE_PLACES_IDX("parentindex", "moz_bookmarks", "parent, position", "") + +#define CREATE_IDX_MOZ_BOOKMARKS_PLACELASTMODIFIED \ + CREATE_PLACES_IDX("itemlastmodifiedindex", "moz_bookmarks", \ + "fk, lastModified", "") + +#define CREATE_IDX_MOZ_BOOKMARKS_DATEADDED \ + CREATE_PLACES_IDX("dateaddedindex", "moz_bookmarks", "dateAdded", "") + +#define CREATE_IDX_MOZ_BOOKMARKS_GUID \ + CREATE_PLACES_IDX("guid_uniqueindex", "moz_bookmarks", "guid", "UNIQUE") + +/** + * moz_annos + */ + +#define CREATE_IDX_MOZ_ANNOS_PLACEATTRIBUTE \ + CREATE_PLACES_IDX("placeattributeindex", "moz_annos", \ + "place_id, anno_attribute_id", "UNIQUE") + +/** + * moz_items_annos + */ + +#define CREATE_IDX_MOZ_ITEMSANNOS_PLACEATTRIBUTE \ + CREATE_PLACES_IDX("itemattributeindex", "moz_items_annos", \ + "item_id, anno_attribute_id", "UNIQUE") + +/** + * moz_keywords + */ + +#define CREATE_IDX_MOZ_KEYWORDS_PLACEPOSTDATA \ + CREATE_PLACES_IDX("placepostdata_uniqueindex", "moz_keywords", \ + "place_id, post_data", "UNIQUE") + +// moz_pages_w_icons + +#define CREATE_IDX_MOZ_PAGES_W_ICONS_ICONURLHASH \ + CREATE_PLACES_IDX("urlhashindex", "moz_pages_w_icons", "page_url_hash", "") + +// moz_icons + +#define CREATE_IDX_MOZ_ICONS_ICONURLHASH \ + CREATE_PLACES_IDX("iconurlhashindex", "moz_icons", "fixed_icon_url_hash", "") + +// moz_places_metadata +#define CREATE_IDX_MOZ_PLACES_METADATA_PLACECREATED \ + CREATE_PLACES_IDX("placecreated_uniqueindex", "moz_places_metadata", \ + "place_id, created_at", "UNIQUE") + +#define CREATE_IDX_MOZ_PLACES_METADATA_REFERRER \ + CREATE_PLACES_IDX("referrerindex", "moz_places_metadata", \ + "referrer_place_id", "") + +#endif // nsPlacesIndexes_h__ diff --git a/toolkit/components/places/nsPlacesMacros.h b/toolkit/components/places/nsPlacesMacros.h new file mode 100644 index 0000000000..9d192eddb6 --- /dev/null +++ b/toolkit/components/places/nsPlacesMacros.h @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#define PLACES_FACTORY_SINGLETON_IMPLEMENTATION(_className, _sInstance) \ + _className* _className::_sInstance = nullptr; \ + \ + already_AddRefed<_className> _className::GetSingleton() { \ + if (_sInstance) { \ + RefPtr<_className> ret = _sInstance; \ + return ret.forget(); \ + } \ + _sInstance = new _className(); \ + RefPtr<_className> ret = _sInstance; \ + if (NS_FAILED(_sInstance->Init())) { \ + /* Null out ret before _sInstance so the destructor doesn't assert */ \ + ret = nullptr; \ + _sInstance = nullptr; \ + return nullptr; \ + } \ + return ret.forget(); \ + } diff --git a/toolkit/components/places/nsPlacesTables.h b/toolkit/components/places/nsPlacesTables.h new file mode 100644 index 0000000000..c81afbe854 --- /dev/null +++ b/toolkit/components/places/nsPlacesTables.h @@ -0,0 +1,323 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __nsPlacesTables_h__ +#define __nsPlacesTables_h__ + +#define CREATE_MOZ_PLACES \ + nsLiteralCString( \ + "CREATE TABLE moz_places ( " \ + " id INTEGER PRIMARY KEY" \ + ", url LONGVARCHAR" \ + ", title LONGVARCHAR" \ + ", rev_host LONGVARCHAR" \ + ", visit_count INTEGER DEFAULT 0" \ + ", hidden INTEGER DEFAULT 0 NOT NULL" \ + ", typed INTEGER DEFAULT 0 NOT NULL" \ + ", frecency INTEGER DEFAULT -1 NOT NULL" \ + ", last_visit_date INTEGER " \ + ", guid TEXT" \ + ", foreign_count INTEGER DEFAULT 0 NOT NULL" \ + ", url_hash INTEGER DEFAULT 0 NOT NULL " \ + ", description TEXT" \ + ", preview_image_url TEXT" \ + ", site_name TEXT" \ + ", origin_id INTEGER REFERENCES moz_origins(id)" \ + ", recalc_frecency INTEGER NOT NULL DEFAULT 0" \ + ", alt_frecency INTEGER" \ + ", recalc_alt_frecency INTEGER NOT NULL DEFAULT 0" \ + ")") + +#define CREATE_MOZ_HISTORYVISITS \ + nsLiteralCString( \ + "CREATE TABLE moz_historyvisits (" \ + " id INTEGER PRIMARY KEY" \ + ", from_visit INTEGER" \ + ", place_id INTEGER" \ + ", visit_date INTEGER" \ + ", visit_type INTEGER" \ + ", session INTEGER" \ + ", source INTEGER DEFAULT 0 NOT NULL" \ + ", triggeringPlaceId INTEGER" \ + ")") + +#define CREATE_MOZ_INPUTHISTORY \ + nsLiteralCString( \ + "CREATE TABLE moz_inputhistory (" \ + " place_id INTEGER NOT NULL" \ + ", input LONGVARCHAR NOT NULL" \ + ", use_count INTEGER" \ + ", PRIMARY KEY (place_id, input)" \ + ")") + +// Note: flags, expiration, type, dateAdded and lastModified should be +// considered deprecated but are kept to ease backwards compatibility. +#define CREATE_MOZ_ANNOS \ + nsLiteralCString( \ + "CREATE TABLE moz_annos (" \ + " id INTEGER PRIMARY KEY" \ + ", place_id INTEGER NOT NULL" \ + ", anno_attribute_id INTEGER" \ + ", content LONGVARCHAR" \ + ", flags INTEGER DEFAULT 0" \ + ", expiration INTEGER DEFAULT 0" \ + ", type INTEGER DEFAULT 0" \ + ", dateAdded INTEGER DEFAULT 0" \ + ", lastModified INTEGER DEFAULT 0" \ + ")") + +#define CREATE_MOZ_ANNO_ATTRIBUTES \ + nsLiteralCString( \ + "CREATE TABLE moz_anno_attributes (" \ + " id INTEGER PRIMARY KEY" \ + ", name VARCHAR(32) UNIQUE NOT NULL" \ + ")") + +#define CREATE_MOZ_ITEMS_ANNOS \ + nsLiteralCString( \ + "CREATE TABLE moz_items_annos (" \ + " id INTEGER PRIMARY KEY" \ + ", item_id INTEGER NOT NULL" \ + ", anno_attribute_id INTEGER" \ + ", content LONGVARCHAR" \ + ", flags INTEGER DEFAULT 0" \ + ", expiration INTEGER DEFAULT 0" \ + ", type INTEGER DEFAULT 0" \ + ", dateAdded INTEGER DEFAULT 0" \ + ", lastModified INTEGER DEFAULT 0" \ + ")") + +#define CREATE_MOZ_BOOKMARKS \ + nsLiteralCString( \ + "CREATE TABLE moz_bookmarks (" \ + " id INTEGER PRIMARY KEY" \ + ", type INTEGER" \ + ", fk INTEGER DEFAULT NULL" /* place_id */ \ + ", parent INTEGER" \ + ", position INTEGER" \ + ", title LONGVARCHAR" \ + ", keyword_id INTEGER" \ + ", folder_type TEXT" \ + ", dateAdded INTEGER" \ + ", lastModified INTEGER" \ + ", guid TEXT" /* The sync status is determined from the change source. \ + We set this to SYNC_STATUS_NEW = 1 for new local \ + bookmarks, and SYNC_STATUS_NORMAL = 2 for bookmarks \ + from other devices. Uploading a local bookmark for the \ + first time changes its status to SYNC_STATUS_NORMAL. \ + For bookmarks restored from a backup, we set \ + SYNC_STATUS_UNKNOWN = 0, indicating that Sync should \ + reconcile them with bookmarks on the server. If Sync is \ + disconnected or never set up, all bookmarks will stay \ + in SYNC_STATUS_NEW. \ + */ \ + ", syncStatus INTEGER NOT NULL DEFAULT 0" /* This field is incremented \ + for every bookmark change \ + that should trigger a sync. \ + It's a counter instead of a \ + Boolean so that we can \ + track changes made during a \ + sync, and queue them for \ + the next sync. Changes made \ + by Sync don't bump the \ + counter, to avoid sync \ + loops. If Sync is \ + disconnected, we'll reset \ + the counter to 1 for all \ + bookmarks. \ + */ \ + ", syncChangeCounter INTEGER NOT NULL DEFAULT 1" \ + ")") + +// This table stores tombstones for bookmarks with SYNC_STATUS_NORMAL. We +// upload tombstones during a sync, and delete them from this table on success. +// If Sync is disconnected, we'll delete all stored tombstones. If Sync is +// never set up, we'll never write new tombstones, since all bookmarks will stay +// in SYNC_STATUS_NEW. +#define CREATE_MOZ_BOOKMARKS_DELETED \ + nsLiteralCString( \ + "CREATE TABLE moz_bookmarks_deleted (" \ + " guid TEXT PRIMARY KEY" \ + ", dateRemoved INTEGER NOT NULL DEFAULT 0" \ + ")") + +#define CREATE_MOZ_KEYWORDS \ + nsLiteralCString( \ + "CREATE TABLE moz_keywords (" \ + " id INTEGER PRIMARY KEY AUTOINCREMENT" \ + ", keyword TEXT UNIQUE" \ + ", place_id INTEGER" \ + ", post_data TEXT" \ + ")") + +#define CREATE_MOZ_ORIGINS \ + nsLiteralCString( \ + "CREATE TABLE moz_origins ( " \ + "id INTEGER PRIMARY KEY, " \ + "prefix TEXT NOT NULL, " \ + "host TEXT NOT NULL, " \ + "frecency INTEGER NOT NULL, " \ + "recalc_frecency INTEGER NOT NULL DEFAULT 0, " \ + "alt_frecency INTEGER, " \ + "recalc_alt_frecency INTEGER NOT NULL DEFAULT 0, " \ + "UNIQUE (prefix, host) " \ + ")") + +// Note: this should be kept up-to-date with the definition in +// nsPlacesAutoComplete.js. +#define CREATE_MOZ_OPENPAGES_TEMP \ + nsLiteralCString( \ + "CREATE TEMP TABLE moz_openpages_temp (" \ + " url TEXT" \ + ", userContextId INTEGER" \ + ", open_count INTEGER" \ + ", PRIMARY KEY (url, userContextId)" \ + ")") + +// This table is used, along with moz_places_afterinsert_trigger, to update +// origins after places removals. During an INSERT into moz_places, origins are +// accumulated in this table, then a DELETE FROM moz_updateoriginsinsert_temp +// will take care of updating the moz_origins table for every new origin. See +// CREATE_PLACES_AFTERINSERT_TRIGGER in nsPlacestriggers.h for details. +#define CREATE_UPDATEORIGINSINSERT_TEMP \ + nsLiteralCString( \ + "CREATE TEMP TABLE moz_updateoriginsinsert_temp ( " \ + "place_id INTEGER PRIMARY KEY, " \ + "prefix TEXT NOT NULL, " \ + "host TEXT NOT NULL, " \ + "frecency INTEGER NOT NULL " \ + ") ") + +// This table is used in a similar way to moz_updateoriginsinsert_temp, but for +// deletes, and triggered via moz_places_afterdelete_trigger. +// +// When rows are added to this table, moz_places.origin_id may be null. That's +// why this table uses prefix + host as its primary key, not origin_id. +#define CREATE_UPDATEORIGINSDELETE_TEMP \ + nsLiteralCString( \ + "CREATE TEMP TABLE moz_updateoriginsdelete_temp ( " \ + "prefix TEXT NOT NULL, " \ + "host TEXT NOT NULL, " \ + "frecency_delta INTEGER NOT NULL, " \ + "PRIMARY KEY (prefix, host) " \ + ") WITHOUT ROWID ") + +// This table is used in a similar way to moz_updateoriginsinsert_temp, but for +// updates to places' frecencies, and triggered via +// moz_places_afterupdate_frecency_trigger. +// +// When rows are added to this table, moz_places.origin_id may be null. That's +// why this table uses prefix + host as its primary key, not origin_id. +#define CREATE_UPDATEORIGINSUPDATE_TEMP \ + nsLiteralCString( \ + "CREATE TEMP TABLE moz_updateoriginsupdate_temp ( " \ + "prefix TEXT NOT NULL, " \ + "host TEXT NOT NULL, " \ + "frecency_delta INTEGER NOT NULL, " \ + "PRIMARY KEY (prefix, host) " \ + ") WITHOUT ROWID ") + +// This table would not be strictly needed for functionality since it's just +// mimicking moz_places, though it's great for database portability. +// With this we don't have to take care into account a bunch of database +// mismatch cases, where places.sqlite could be mixed up with a favicons.sqlite +// created with a different places.sqlite (not just in case of a user messing +// up with the profile, but also in case of corruption). +#define CREATE_MOZ_PAGES_W_ICONS \ + nsLiteralCString( \ + "CREATE TABLE moz_pages_w_icons ( " \ + "id INTEGER PRIMARY KEY, " \ + "page_url TEXT NOT NULL, " \ + "page_url_hash INTEGER NOT NULL " \ + ") ") + +// This table retains the icons data. The hashes url is "fixed" (thus the scheme +// and www are trimmed in most cases) so we can quickly query for root icon urls +// like "domain/favicon.ico". +// We are considering squared icons for simplicity, so storing only one size. +// For svg payloads, width will be set to 65535 (UINT16_MAX). +#define CREATE_MOZ_ICONS \ + nsLiteralCString( \ + "CREATE TABLE moz_icons ( " \ + "id INTEGER PRIMARY KEY, " \ + "icon_url TEXT NOT NULL, " \ + "fixed_icon_url_hash INTEGER NOT NULL, " \ + "width INTEGER NOT NULL DEFAULT 0, " \ + "root INTEGER NOT NULL DEFAULT 0, " \ + "color INTEGER, " \ + "expire_ms INTEGER NOT NULL DEFAULT 0, " \ + "data BLOB " \ + ") ") + +// This table maintains relations between icons and pages. +// Each page can have multiple icons, and each icon can be used by multiple +// pages. +#define CREATE_MOZ_ICONS_TO_PAGES \ + nsLiteralCString( \ + "CREATE TABLE moz_icons_to_pages ( " \ + "page_id INTEGER NOT NULL, " \ + "icon_id INTEGER NOT NULL, " \ + "expire_ms INTEGER NOT NULL DEFAULT 0, " \ + "PRIMARY KEY (page_id, icon_id), " \ + "FOREIGN KEY (page_id) REFERENCES moz_pages_w_icons ON DELETE CASCADE, " \ + "FOREIGN KEY (icon_id) REFERENCES moz_icons ON DELETE CASCADE " \ + ") WITHOUT ROWID ") + +// This table holds key-value metadata for Places and its consumers. Sync stores +// the sync IDs for the bookmarks and history collections in this table, and the +// last sync time for history. +#define CREATE_MOZ_META \ + nsLiteralCString( \ + "CREATE TABLE moz_meta (" \ + "key TEXT PRIMARY KEY, " \ + "value NOT NULL" \ + ") WITHOUT ROWID ") + +// Keys in the moz_meta table. +#define MOZ_META_KEY_ORIGIN_FRECENCY_COUNT "origin_frecency_count" +#define MOZ_META_KEY_ORIGIN_FRECENCY_SUM "origin_frecency_sum" +#define MOZ_META_KEY_ORIGIN_FRECENCY_SUM_OF_SQUARES \ + "origin_frecency_sum_of_squares" + +// This table holds history interactions that will be used to achieve improved +// history recalls. +#define CREATE_MOZ_PLACES_METADATA \ + nsLiteralCString( \ + "CREATE TABLE moz_places_metadata (" \ + "id INTEGER PRIMARY KEY, " \ + "place_id INTEGER NOT NULL, " \ + "referrer_place_id INTEGER, " \ + "created_at INTEGER NOT NULL DEFAULT 0, " \ + "updated_at INTEGER NOT NULL DEFAULT 0, " \ + "total_view_time INTEGER NOT NULL DEFAULT 0, " \ + "typing_time INTEGER NOT NULL DEFAULT 0, " \ + "key_presses INTEGER NOT NULL DEFAULT 0, " \ + "scrolling_time INTEGER NOT NULL DEFAULT 0, " \ + "scrolling_distance INTEGER NOT NULL DEFAULT 0, " \ + "document_type INTEGER NOT NULL DEFAULT 0, " \ + "search_query_id INTEGER, " \ + "FOREIGN KEY (place_id) REFERENCES moz_places(id) ON DELETE CASCADE, " \ + "FOREIGN KEY (referrer_place_id) REFERENCES moz_places(id) ON DELETE " \ + "CASCADE, " \ + "FOREIGN KEY(search_query_id) REFERENCES " \ + "moz_places_metadata_search_queries(id) ON DELETE CASCADE " \ + "CHECK(place_id != referrer_place_id) " \ + ")") + +#define CREATE_MOZ_PLACES_METADATA_SEARCH_QUERIES \ + nsLiteralCString( \ + "CREATE TABLE IF NOT EXISTS moz_places_metadata_search_queries ( " \ + "id INTEGER PRIMARY KEY, " \ + "terms TEXT NOT NULL UNIQUE " \ + ")") + +#define CREATE_MOZ_PREVIEWS_TOMBSTONES \ + nsLiteralCString( \ + "CREATE TABLE IF NOT EXISTS moz_previews_tombstones ( " \ + " hash TEXT PRIMARY KEY " \ + ") WITHOUT ROWID") + +#endif // __nsPlacesTables_h__ diff --git a/toolkit/components/places/nsPlacesTriggers.h b/toolkit/components/places/nsPlacesTriggers.h new file mode 100644 index 0000000000..78eacdb581 --- /dev/null +++ b/toolkit/components/places/nsPlacesTriggers.h @@ -0,0 +1,415 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsPlacesTables.h" + +#ifndef __nsPlacesTriggers_h__ +# define __nsPlacesTriggers_h__ + +/** + * These visit types are excluded from visit_count: + * 0 - invalid + * 4 - EMBED + * 7 - DOWNLOAD + * 8 - FRAMED_LINK + * 9 - RELOAD + **/ +# define VISIT_COUNT_INC(field) \ + "(CASE WHEN " field " IN (0, 4, 7, 8, 9) THEN 0 ELSE 1 END) " + +/** + * This triggers update visit_count and last_visit_date based on historyvisits + * table changes. + */ +# define CREATE_HISTORYVISITS_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_historyvisits_afterinsert_v2_trigger " \ + "AFTER INSERT ON moz_historyvisits FOR EACH ROW " \ + "BEGIN " \ + "SELECT invalidate_days_of_history();" \ + "SELECT store_last_inserted_id('moz_historyvisits', NEW.id); " \ + "UPDATE moz_places SET " \ + "visit_count = visit_count + " VISIT_COUNT_INC("NEW.visit_type") ", " \ + "recalc_frecency = (frecency <> 0), " \ + "recalc_alt_frecency = (frecency <> 0), " \ + "last_visit_date = MAX(IFNULL(last_visit_date, 0), NEW.visit_date) " \ + "WHERE id = NEW.place_id;" \ + "END") + +# define CREATE_HISTORYVISITS_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_historyvisits_afterdelete_v2_trigger " \ + "AFTER DELETE ON moz_historyvisits FOR EACH ROW " \ + "BEGIN " \ + "SELECT invalidate_days_of_history();" \ + "UPDATE moz_places SET " \ + "visit_count = visit_count - " VISIT_COUNT_INC("OLD.visit_type") ", " \ + "recalc_frecency = (frecency <> 0), " \ + "recalc_alt_frecency = (frecency <> 0), " \ + "last_visit_date = (SELECT visit_date FROM moz_historyvisits " \ + "WHERE place_id = OLD.place_id " \ + "ORDER BY visit_date DESC LIMIT 1) " \ + "WHERE id = OLD.place_id;" \ + "END") + +// This macro is a helper for the next several triggers. It updates the origin +// frecency stats. Use it as follows. Before changing an origin's frecency, +// call the macro and pass "-" (subtraction) as the argument. That will update +// the stats by deducting the origin's current contribution to them. And then +// after you change the origin's frecency, call the macro again, this time +// passing "+" (addition) as the argument. That will update the stats by adding +// the origin's new contribution to them. +# define UPDATE_ORIGIN_FRECENCY_STATS(op) \ + "INSERT OR REPLACE INTO moz_meta(key, value) " \ + "SELECT '" MOZ_META_KEY_ORIGIN_FRECENCY_COUNT \ + "', " \ + "IFNULL((SELECT value FROM moz_meta WHERE key = " \ + "'" MOZ_META_KEY_ORIGIN_FRECENCY_COUNT "'), 0) " op \ + " CAST(frecency > 0 AS INT) " \ + "FROM moz_origins WHERE prefix = OLD.prefix AND host = OLD.host " \ + "UNION " \ + "SELECT '" MOZ_META_KEY_ORIGIN_FRECENCY_SUM \ + "', " \ + "IFNULL((SELECT value FROM moz_meta WHERE key = " \ + "'" MOZ_META_KEY_ORIGIN_FRECENCY_SUM "'), 0) " op \ + " MAX(frecency, 0) " \ + "FROM moz_origins WHERE prefix = OLD.prefix AND host = OLD.host " \ + "UNION " \ + "SELECT '" MOZ_META_KEY_ORIGIN_FRECENCY_SUM_OF_SQUARES \ + "', " \ + "IFNULL((SELECT value FROM moz_meta WHERE key = " \ + "'" MOZ_META_KEY_ORIGIN_FRECENCY_SUM_OF_SQUARES "'), 0) " op \ + " (MAX(frecency, 0) * MAX(frecency, 0)) " \ + "FROM moz_origins WHERE prefix = OLD.prefix AND host = OLD.host " + +// The next several triggers are a workaround for the lack of FOR EACH STATEMENT +// in Sqlite, until bug 871908 can be fixed properly. +// +// While doing inserts or deletes into moz_places, we accumulate the affected +// origins into a temp table. Afterwards, we delete everything from the temp +// table, causing the AFTER DELETE trigger to fire for it, which will then +// update moz_origins and the origin frecency stats. As a consequence, we also +// do this for updates to moz_places.frecency in order to make sure that changes +// to origins are serialized. +// +// Note this way we lose atomicity, crashing between the 2 queries may break the +// tables' coherency. So it's better to run those DELETE queries in a single +// transaction. Regardless, this is still better than hanging the browser for +// several minutes on a fast machine. + +// This trigger runs on inserts into moz_places. +# define CREATE_PLACES_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_afterinsert_trigger " \ + "AFTER INSERT ON moz_places FOR EACH ROW " \ + "BEGIN " \ + "SELECT store_last_inserted_id('moz_places', NEW.id); " \ + "INSERT OR IGNORE INTO moz_updateoriginsinsert_temp (place_id, " \ + "prefix, " \ + "host, frecency) " \ + "VALUES (NEW.id, get_prefix(NEW.url), get_host_and_port(NEW.url), " \ + "NEW.frecency); " \ + "END") +// This trigger corresponds to the previous trigger. It runs on deletes on +// moz_updateoriginsinsert_temp -- logically, after inserts on moz_places. +# define CREATE_UPDATEORIGINSINSERT_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_updateoriginsinsert_afterdelete_trigger " \ + "AFTER DELETE ON moz_updateoriginsinsert_temp FOR EACH ROW " \ + "BEGIN " \ + /* Deduct the origin's current contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("-") "; " \ + "INSERT INTO moz_origins (prefix, host, frecency, recalc_alt_frecency) " \ + "VALUES (OLD.prefix, OLD.host, MAX(OLD.frecency, 0), 1) " \ + "ON CONFLICT(prefix, host) DO UPDATE " \ + "SET frecency = frecency + OLD.frecency " \ + "WHERE OLD.frecency > 0; " \ + /* Add the origin's new contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("+") "; " \ + "UPDATE moz_places SET origin_id = ( " \ + "SELECT id " \ + "FROM moz_origins " \ + "WHERE prefix = OLD.prefix AND host = OLD.host " \ + ") " \ + "WHERE id = OLD.place_id; " \ + "END" \ +) + +// This trigger runs on deletes on moz_places. +# define CREATE_PLACES_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_afterdelete_trigger " \ + "AFTER DELETE ON moz_places FOR EACH ROW " \ + "BEGIN " \ + "INSERT INTO moz_updateoriginsdelete_temp (prefix, host, " \ + "frecency_delta) " \ + "VALUES (get_prefix(OLD.url), get_host_and_port(OLD.url), " \ + "-MAX(OLD.frecency, 0)) " \ + "ON CONFLICT(prefix, host) DO UPDATE " \ + "SET frecency_delta = frecency_delta - OLD.frecency " \ + "WHERE OLD.frecency > 0; " \ + "END ") + +// This is an alternate version of CREATE_PLACES_AFTERDELETE_TRIGGER, with +// support for previews tombstones. Only one of these should be used at the +// same time +# define CREATE_PLACES_AFTERDELETE_WPREVIEWS_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_afterdelete_wpreviews_trigger " \ + "AFTER DELETE ON moz_places FOR EACH ROW " \ + "BEGIN " \ + "INSERT INTO moz_updateoriginsdelete_temp (prefix, host, " \ + "frecency_delta) " \ + "VALUES (get_prefix(OLD.url), get_host_and_port(OLD.url), " \ + "-MAX(OLD.frecency, 0)) " \ + "ON CONFLICT(prefix, host) DO UPDATE " \ + "SET frecency_delta = frecency_delta - OLD.frecency " \ + "WHERE OLD.frecency > 0; " \ + "INSERT OR IGNORE INTO moz_previews_tombstones VALUES " \ + "(md5hex(OLD.url));" \ + "END ") + +// This trigger corresponds to the previous trigger. It runs on deletes on +// moz_updateoriginsdelete_temp -- logically, after deletes on moz_places. +# define CREATE_UPDATEORIGINSDELETE_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_updateoriginsdelete_afterdelete_trigger " \ + "AFTER DELETE ON moz_updateoriginsdelete_temp FOR EACH ROW " \ + "BEGIN " \ + /* Deduct the origin's current contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("-") "; " \ + "UPDATE moz_origins SET frecency = frecency + OLD.frecency_delta, " \ + "recalc_frecency = 0 " \ + "WHERE prefix = OLD.prefix AND host = OLD.host; " \ + "DELETE FROM moz_origins " \ + "WHERE prefix = OLD.prefix AND host = OLD.host AND NOT EXISTS ( " \ + "SELECT id FROM moz_places " \ + "WHERE origin_id = moz_origins.id " \ + "LIMIT 1 " \ + "); " \ + /* Add the origin's new contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("+") "; " \ + "DELETE FROM moz_icons WHERE id IN ( " \ + "SELECT id FROM moz_icons " \ + "WHERE fixed_icon_url_hash = hash(fixup_url(OLD.host || '/favicon.ico')) " \ + "AND fixup_url(icon_url) = fixup_url(OLD.host || '/favicon.ico') " \ + "AND NOT EXISTS (SELECT 1 FROM moz_origins WHERE host = OLD.host " \ + "OR host = fixup_url(OLD.host)) " \ + "EXCEPT " \ + "SELECT icon_id FROM moz_icons_to_pages " \ + "); " \ + "END" \ +) + +// This trigger runs on updates to moz_places.frecency. +// +// However, we skip this when frecency changes are due to frecency decay since +// (1) decay updates all frecencies at once, so this trigger would run for each +// moz_place, which would be expensive; and (2) decay does not change the +// ordering of frecencies since all frecencies decay by the same percentage. +# define CREATE_PLACES_AFTERUPDATE_FRECENCY_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_afterupdate_frecency_trigger " \ + "AFTER UPDATE OF frecency ON moz_places FOR EACH ROW " \ + "WHEN NOT is_frecency_decaying() " \ + "BEGIN " \ + "INSERT INTO moz_updateoriginsupdate_temp (prefix, host, " \ + "frecency_delta) " \ + "VALUES (get_prefix(NEW.url), get_host_and_port(NEW.url), " \ + "MAX(NEW.frecency, 0) - MAX(OLD.frecency, 0)) " \ + "ON CONFLICT(prefix, host) DO UPDATE " \ + "SET frecency_delta = frecency_delta + EXCLUDED.frecency_delta; " \ + "UPDATE moz_places SET recalc_frecency = 0 WHERE id = NEW.id; " \ + "UPDATE moz_origins SET recalc_frecency = 1, recalc_alt_frecency = 1 " \ + "WHERE id = NEW.origin_id; " \ + "END ") +// This trigger corresponds to the previous trigger. It runs on deletes on +// moz_updateoriginsupdate_temp -- logically, after updates to +// moz_places.frecency. +# define CREATE_UPDATEORIGINSUPDATE_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_updateoriginsupdate_afterdelete_trigger " \ + "AFTER DELETE ON moz_updateoriginsupdate_temp FOR EACH ROW " \ + "BEGIN " \ + /* Deduct the origin's current contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("-") "; " \ + "UPDATE moz_origins " \ + "SET frecency = frecency + OLD.frecency_delta, " \ + "recalc_frecency = 0 " \ + "WHERE prefix = OLD.prefix AND host = OLD.host; " \ + /* Add the origin's new contribution to frecency stats */ \ + UPDATE_ORIGIN_FRECENCY_STATS("+") "; " \ + "END" \ +) + +// Runs when recalc_frecency is set to 1 +# define CREATE_PLACES_AFTERUPDATE_RECALC_FRECENCY_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_afterupdate_recalc_frecency_trigger " \ + "AFTER UPDATE OF recalc_frecency ON moz_places FOR EACH ROW " \ + "WHEN NEW.recalc_frecency = 1 " \ + "BEGIN " \ + " SELECT set_should_start_frecency_recalculation();" \ + "END") + +/** + * This trigger removes a row from moz_openpages_temp when open_count reaches 0. + * + * @note this should be kept up-to-date with the definition in + * nsPlacesAutoComplete.js + */ +# define CREATE_REMOVEOPENPAGE_CLEANUP_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMPORARY TRIGGER moz_openpages_temp_afterupdate_trigger " \ + "AFTER UPDATE OF open_count ON moz_openpages_temp FOR EACH ROW " \ + "WHEN NEW.open_count = 0 " \ + "BEGIN " \ + "DELETE FROM moz_openpages_temp " \ + "WHERE url = NEW.url " \ + "AND userContextId = NEW.userContextId;" \ + "END") + +/** + * Any bookmark operation should recalculate frecency, apart from place: + * queries. + */ +# define IS_PLACE_QUERY \ + " url_hash BETWEEN hash('place', 'prefix_lo') " \ + " AND hash('place', 'prefix_hi') " + +# define CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_bookmarks_foreign_count_afterdelete_trigger " \ + "AFTER DELETE ON moz_bookmarks FOR EACH ROW " \ + "BEGIN " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count - 1 " \ + ", recalc_frecency = NOT " IS_PLACE_QUERY \ + ", recalc_alt_frecency = NOT " IS_PLACE_QUERY \ + "WHERE id = OLD.fk;" \ + "END") + +/** + * Currently expiration skips anything with frecency = -1, since that is the + * default value for new page insertions. Unfortunately adding and immediately + * removing a bookmark will generate a page with frecency = -1 that would never + * be expired until visited. + * As a temporary workaround we set frecency to 1 on bookmark addition if it was + * set to -1. This is not elegant, but it will be fixed by Bug 1475582 once + * removing bookmarks will immediately take care of removing orphan pages. + * Note setting frecency resets recalc_frecency, so do it first. + */ +# define CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_bookmarks_foreign_count_afterinsert_trigger " \ + "AFTER INSERT ON moz_bookmarks FOR EACH ROW " \ + "BEGIN " \ + "SELECT store_last_inserted_id('moz_bookmarks', NEW.id); " \ + "SELECT note_sync_change() WHERE NEW.syncChangeCounter > 0; " \ + "UPDATE moz_places " \ + "SET frecency = 1 WHERE frecency = -1 AND NOT " IS_PLACE_QUERY \ + ";" \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count + 1 " \ + ", hidden = " IS_PLACE_QUERY \ + ", recalc_frecency = NOT " IS_PLACE_QUERY \ + ", recalc_alt_frecency = NOT " IS_PLACE_QUERY \ + "WHERE id = NEW.fk;" \ + "END") + +# define CREATE_BOOKMARKS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_bookmarks_foreign_count_afterupdate_trigger " \ + "AFTER UPDATE OF fk, syncChangeCounter ON moz_bookmarks FOR EACH ROW " \ + "BEGIN " \ + "SELECT note_sync_change() " \ + "WHERE NEW.syncChangeCounter <> OLD.syncChangeCounter; " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count + 1 " \ + ", hidden = " IS_PLACE_QUERY \ + ", recalc_frecency = NOT " IS_PLACE_QUERY \ + ", recalc_alt_frecency = NOT " IS_PLACE_QUERY \ + "WHERE OLD.fk <> NEW.fk AND id = NEW.fk;" \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count - 1 " \ + ", recalc_frecency = NOT " IS_PLACE_QUERY \ + ", recalc_alt_frecency = NOT " IS_PLACE_QUERY \ + "WHERE OLD.fk <> NEW.fk AND id = OLD.fk;" \ + "END") + +# define CREATE_KEYWORDS_FOREIGNCOUNT_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_keywords_foreign_count_afterdelete_trigger " \ + "AFTER DELETE ON moz_keywords FOR EACH ROW " \ + "BEGIN " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count - 1 " \ + "WHERE id = OLD.place_id;" \ + "END") + +# define CREATE_KEYWORDS_FOREIGNCOUNT_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_keywords_foreign_count_afterinsert_trigger " \ + "AFTER INSERT ON moz_keywords FOR EACH ROW " \ + "BEGIN " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count + 1 " \ + "WHERE id = NEW.place_id;" \ + "END") + +# define CREATE_KEYWORDS_FOREIGNCOUNT_AFTERUPDATE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_keywords_foreign_count_afterupdate_trigger " \ + "AFTER UPDATE OF place_id ON moz_keywords FOR EACH ROW " \ + "BEGIN " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count + 1 " \ + "WHERE id = NEW.place_id; " \ + "UPDATE moz_places " \ + "SET foreign_count = foreign_count - 1 " \ + "WHERE id = OLD.place_id; " \ + "END") + +# define CREATE_ICONS_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_icons_afterinsert_v1_trigger " \ + "AFTER INSERT ON moz_icons FOR EACH ROW " \ + "BEGIN " \ + "SELECT store_last_inserted_id('moz_icons', NEW.id); " \ + "END") + +# define CREATE_BOOKMARKS_DELETED_AFTERINSERT_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_bookmarks_deleted_afterinsert_v1_trigger " \ + "AFTER INSERT ON moz_bookmarks_deleted FOR EACH ROW " \ + "BEGIN " \ + "SELECT note_sync_change(); " \ + "END") + +# define CREATE_BOOKMARKS_DELETED_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_bookmarks_deleted_afterdelete_v1_trigger " \ + "AFTER DELETE ON moz_bookmarks_deleted FOR EACH ROW " \ + "BEGIN " \ + "SELECT note_sync_change(); " \ + "END") + +// This trigger removes orphan search terms when interactions are removed from +// the metadata table. +# define CREATE_PLACES_METADATA_AFTERDELETE_TRIGGER \ + nsLiteralCString( \ + "CREATE TEMP TRIGGER moz_places_metadata_afterdelete_trigger " \ + "AFTER DELETE ON moz_places_metadata " \ + "FOR EACH ROW " \ + "BEGIN " \ + "DELETE FROM moz_places_metadata_search_queries " \ + "WHERE id = OLD.search_query_id AND NOT EXISTS (" \ + "SELECT id FROM moz_places_metadata " \ + "WHERE search_query_id = OLD.search_query_id " \ + "); " \ + "END") + +#endif // __nsPlacesTriggers_h__ diff --git a/toolkit/components/places/tests/PlacesTestUtils.sys.mjs b/toolkit/components/places/tests/PlacesTestUtils.sys.mjs new file mode 100644 index 0000000000..e2e792c24d --- /dev/null +++ b/toolkit/components/places/tests/PlacesTestUtils.sys.mjs @@ -0,0 +1,616 @@ +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + TestUtils: "resource://testing-common/TestUtils.sys.mjs", +}); + +export var PlacesTestUtils = Object.freeze({ + /** + * Asynchronously adds visits to a page. + * + * @param {*} aPlaceInfo + * A string URL, nsIURI, Window.URL object, info object (explained + * below), or an array of any of those. Info objects describe the + * visits to add more fully than URLs/URIs alone and look like this: + * + * { + * uri|url: href, URL or nsIURI of the page, + * [optional] transition: one of the TRANSITION_* from nsINavHistoryService, + * [optional] title: title of the page, + * [optional] visitDate: visit date, either in microseconds from the epoch or as a date object + * [optional] referrer: nsIURI of the referrer for this visit + * } + * + * @return {Promise} + * @resolves When all visits have been added successfully. + * @rejects JavaScript exception. + */ + async addVisits(placeInfo) { + let places = []; + let infos = []; + + if (Array.isArray(placeInfo)) { + places.push(...placeInfo); + } else { + places.push(placeInfo); + } + + // Create a PageInfo for each entry. + let lastStoredVisit; + for (let obj of places) { + let place; + if ( + obj instanceof Ci.nsIURI || + URL.isInstance(obj) || + typeof obj == "string" + ) { + place = { uri: obj }; + } else if (typeof obj == "object" && (obj.uri || obj.url)) { + place = obj; + } else { + throw new Error("Unsupported type passed to addVisits"); + } + + let referrer = place.referrer + ? lazy.PlacesUtils.toURI(place.referrer) + : null; + let info = { url: place.uri || place.url }; + let spec = + info.url instanceof Ci.nsIURI ? info.url.spec : new URL(info.url).href; + info.title = "title" in place ? place.title : "test visit for " + spec; + let visitDate = place.visitDate; + if (visitDate) { + if (visitDate.constructor.name != "Date") { + // visitDate should be in microseconds. It's easy to do the wrong thing + // and pass milliseconds, so we lazily check for that. + // While it's not easily distinguishable, since both are integers, we + // can check if the value is very far in the past, and assume it's + // probably a mistake. + if (visitDate <= Date.now()) { + throw new Error( + "AddVisits expects a Date object or _micro_seconds!" + ); + } + visitDate = lazy.PlacesUtils.toDate(visitDate); + } + } else { + visitDate = new Date(); + } + info.visits = [ + { + transition: place.transition, + date: visitDate, + referrer, + }, + ]; + infos.push(info); + if ( + !place.transition || + place.transition != lazy.PlacesUtils.history.TRANSITIONS.EMBED + ) { + lastStoredVisit = info; + } + } + await lazy.PlacesUtils.history.insertMany(infos); + if (lastStoredVisit) { + await lazy.TestUtils.waitForCondition( + () => lazy.PlacesUtils.history.fetch(lastStoredVisit.url), + "Ensure history has been updated and is visible to read-only connections" + ); + } + }, + + /* + * Add Favicons + * + * @param {Map} faviconURLs keys are page URLs, values are their + * associated favicon URLs. + */ + + async addFavicons(faviconURLs) { + let faviconPromises = []; + + // If no favicons were provided, we do not want to continue on + if (!faviconURLs) { + throw new Error("No favicon URLs were provided"); + } + for (let [key, val] of faviconURLs) { + if (!val) { + throw new Error("URL does not exist"); + } + faviconPromises.push( + new Promise((resolve, reject) => { + let uri = Services.io.newURI(key); + let faviconURI = Services.io.newURI(val); + try { + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + uri, + faviconURI, + false, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + resolve, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } catch (ex) { + reject(ex); + } + }) + ); + } + await Promise.all(faviconPromises); + }, + + /** + * Clears any favicons stored in the database. + */ + async clearFavicons() { + return new Promise(resolve => { + Services.obs.addObserver(function observer() { + Services.obs.removeObserver(observer, "places-favicons-expired"); + resolve(); + }, "places-favicons-expired"); + lazy.PlacesUtils.favicons.expireAllFavicons(); + }); + }, + + /** + * Adds a bookmark to the database. This should only be used when you need to + * add keywords. Otherwise, use `PlacesUtils.bookmarks.insert()`. + * @param {string} aBookmarkObj.uri + * @param {string} [aBookmarkObj.title] + * @param {string} [aBookmarkObj.keyword] + */ + async addBookmarkWithDetails(aBookmarkObj) { + await lazy.PlacesUtils.bookmarks.insert({ + parentGuid: lazy.PlacesUtils.bookmarks.unfiledGuid, + title: aBookmarkObj.title || "A bookmark", + url: aBookmarkObj.uri, + }); + + if (aBookmarkObj.keyword) { + await lazy.PlacesUtils.keywords.insert({ + keyword: aBookmarkObj.keyword, + url: + aBookmarkObj.uri instanceof Ci.nsIURI + ? aBookmarkObj.uri.spec + : aBookmarkObj.uri, + postData: aBookmarkObj.postData, + }); + } + + if (aBookmarkObj.tags) { + let uri = + aBookmarkObj.uri instanceof Ci.nsIURI + ? aBookmarkObj.uri + : Services.io.newURI(aBookmarkObj.uri); + lazy.PlacesUtils.tagging.tagURI(uri, aBookmarkObj.tags); + } + }, + + /** + * Waits for all pending async statements on the default connection. + * + * @return {Promise} + * @resolves When all pending async statements finished. + * @rejects Never. + * + * @note The result is achieved by asynchronously executing a query requiring + * a write lock. Since all statements on the same connection are + * serialized, the end of this write operation means that all writes are + * complete. Note that WAL makes so that writers don't block readers, but + * this is a problem only across different connections. + */ + promiseAsyncUpdates() { + return lazy.PlacesUtils.withConnectionWrapper( + "promiseAsyncUpdates", + async function (db) { + try { + await db.executeCached("BEGIN EXCLUSIVE"); + await db.executeCached("COMMIT"); + } catch (ex) { + // If we fail to start a transaction, it's because there is already one. + // In such a case we should not try to commit the existing transaction. + } + } + ); + }, + + /** + * Asynchronously checks if an address is found in the database. + * @param aURI + * nsIURI or address to look for. + * + * @return {Promise} + * @resolves Returns true if the page is found. + * @rejects JavaScript exception. + */ + async isPageInDB(aURI) { + return ( + (await this.getDatabaseValue("moz_places", "id", { url: aURI })) !== + undefined + ); + }, + + /** + * Asynchronously checks how many visits exist for a specified page. + * @param aURI + * nsIURI or address to look for. + * + * @return {Promise} + * @resolves Returns the number of visits found. + * @rejects JavaScript exception. + */ + async visitsInDB(aURI) { + let url = aURI instanceof Ci.nsIURI ? aURI.spec : aURI; + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + `SELECT count(*) FROM moz_historyvisits v + JOIN moz_places h ON h.id = v.place_id + WHERE url_hash = hash(:url) AND url = :url`, + { url } + ); + return rows[0].getResultByIndex(0); + }, + + /** + * Marks all syncable bookmarks as synced by setting their sync statuses to + * "NORMAL", resetting their change counters, and removing all tombstones. + * Used by tests to avoid calling `PlacesSyncUtils.bookmarks.pullChanges` + * and `PlacesSyncUtils.bookmarks.pushChanges`. + * + * @resolves When all bookmarks have been updated. + * @rejects JavaScript exception. + */ + markBookmarksAsSynced() { + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesTestUtils: markBookmarksAsSynced", + function (db) { + return db.executeTransaction(async function () { + await db.executeCached( + `WITH RECURSIVE + syncedItems(id) AS ( + SELECT b.id FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ) + UPDATE moz_bookmarks + SET syncChangeCounter = 0, + syncStatus = :syncStatus + WHERE id IN syncedItems`, + { syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL } + ); + await db.executeCached("DELETE FROM moz_bookmarks_deleted"); + }); + } + ); + }, + + /** + * Sets sync fields for multiple bookmarks. + * @param aStatusInfos + * One or more objects with the following properties: + * { [required] guid: The bookmark's GUID, + * syncStatus: An `nsINavBookmarksService::SYNC_STATUS_*` constant, + * syncChangeCounter: The sync change counter value, + * lastModified: The last modified time, + * dateAdded: The date added time. + * } + * + * @resolves When all bookmarks have been updated. + * @rejects JavaScript exception. + */ + setBookmarkSyncFields(...aFieldInfos) { + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesTestUtils: setBookmarkSyncFields", + function (db) { + return db.executeTransaction(async function () { + for (let info of aFieldInfos) { + if (!lazy.PlacesUtils.isValidGuid(info.guid)) { + throw new Error(`Invalid GUID: ${info.guid}`); + } + await db.executeCached( + `UPDATE moz_bookmarks + SET syncStatus = IFNULL(:syncStatus, syncStatus), + syncChangeCounter = IFNULL(:syncChangeCounter, syncChangeCounter), + lastModified = IFNULL(:lastModified, lastModified), + dateAdded = IFNULL(:dateAdded, dateAdded) + WHERE guid = :guid`, + { + guid: info.guid, + syncChangeCounter: info.syncChangeCounter, + syncStatus: "syncStatus" in info ? info.syncStatus : null, + lastModified: + "lastModified" in info + ? lazy.PlacesUtils.toPRTime(info.lastModified) + : null, + dateAdded: + "dateAdded" in info + ? lazy.PlacesUtils.toPRTime(info.dateAdded) + : null, + } + ); + } + }); + } + ); + }, + + async fetchBookmarkSyncFields(...aGuids) { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let results = []; + for (let guid of aGuids) { + let rows = await db.executeCached( + ` + SELECT syncStatus, syncChangeCounter, lastModified, dateAdded + FROM moz_bookmarks + WHERE guid = :guid`, + { guid } + ); + if (!rows.length) { + throw new Error(`Bookmark ${guid} does not exist`); + } + results.push({ + guid, + syncStatus: rows[0].getResultByName("syncStatus"), + syncChangeCounter: rows[0].getResultByName("syncChangeCounter"), + lastModified: lazy.PlacesUtils.toDate( + rows[0].getResultByName("lastModified") + ), + dateAdded: lazy.PlacesUtils.toDate( + rows[0].getResultByName("dateAdded") + ), + }); + } + return results; + }, + + async fetchSyncTombstones() { + let db = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached(` + SELECT guid, dateRemoved + FROM moz_bookmarks_deleted + ORDER BY guid`); + return rows.map(row => ({ + guid: row.getResultByName("guid"), + dateRemoved: lazy.PlacesUtils.toDate(row.getResultByName("dateRemoved")), + })); + }, + + /** + * Returns a promise that waits until happening Places events specified by + * notification parameter. + * + * @param {string} notification + * Available values are: + * bookmark-added + * bookmark-removed + * bookmark-moved + * bookmark-guid_changed + * bookmark-keyword_changed + * bookmark-tags_changed + * bookmark-time_changed + * bookmark-title_changed + * bookmark-url_changed + * favicon-changed + * history-cleared + * page-removed + * page-title-changed + * page-visited + * pages-rank-changed + * purge-caches + * @param {Function} conditionFn [optional] + * If need some more condition to wait, please use conditionFn. + * This is an optional, but if set, should returns true when the wait + * condition is met. + * @return {Promise} + * A promise that resolved if the wait condition is met. + * The resolved value is an array of PlacesEvent object. + */ + waitForNotification(notification, conditionFn) { + return new Promise(resolve => { + function listener(events) { + if (!conditionFn || conditionFn(events)) { + PlacesObservers.removeListener([notification], listener); + resolve(events); + } + } + PlacesObservers.addListener([notification], listener); + }); + }, + + /** + * A debugging helper that dumps the contents of an SQLite table. + * + * @param {String} table + * The table name. + * @param {Sqlite.OpenedConnection} [db] + * The mirror database connection. + * @param {String[]} [columns] + * Clumns to be printed, defaults to all. + */ + async dumpTable({ table, db, columns }) { + if (!table) { + throw new Error("Must pass a `table` name"); + } + if (!db) { + db = await lazy.PlacesUtils.promiseDBConnection(); + } + if (!columns) { + columns = (await db.execute(`PRAGMA table_info('${table}')`)).map(r => + r.getResultByName("name") + ); + } + let results = [columns.join("\t")]; + + let rows = await db.execute(`SELECT ${columns.join()} FROM ${table}`); + dump(`>> Table ${table} contains ${rows.length} rows\n`); + + for (let row of rows) { + let numColumns = row.numEntries; + let rowValues = []; + for (let i = 0; i < numColumns; ++i) { + let value = "N/A"; + switch (row.getTypeOfIndex(i)) { + case Ci.mozIStorageValueArray.VALUE_TYPE_NULL: + value = "NULL"; + break; + case Ci.mozIStorageValueArray.VALUE_TYPE_INTEGER: + value = row.getInt64(i); + break; + case Ci.mozIStorageValueArray.VALUE_TYPE_FLOAT: + value = row.getDouble(i); + break; + case Ci.mozIStorageValueArray.VALUE_TYPE_TEXT: + value = JSON.stringify(row.getString(i)); + break; + } + rowValues.push(value.toString().padStart(columns[i].length, " ")); + } + results.push(rowValues.join("\t")); + } + results.push("\n"); + dump(results.join("\n")); + }, + + /** + * Removes all stored metadata. + */ + clearMetadata() { + return lazy.PlacesUtils.withConnectionWrapper( + "PlacesTestUtils: clearMetadata", + async db => { + await db.execute(`DELETE FROM moz_meta`); + lazy.PlacesUtils.metadata.cache.clear(); + } + ); + }, + + /** + * Clear moz_inputhistory table. + */ + async clearInputHistory() { + await lazy.PlacesUtils.withConnectionWrapper( + "test:clearInputHistory", + db => { + return db.executeCached("DELETE FROM moz_inputhistory"); + } + ); + }, + + /** + * Clear moz_historyvisits table. + */ + async clearHistoryVisits() { + await lazy.PlacesUtils.withConnectionWrapper( + "test:clearHistoryVisits", + db => { + return db.executeCached("DELETE FROM moz_historyvisits"); + } + ); + }, + + /** + * Compares 2 place: URLs ignoring the order of their params. + * @param url1 First URL to compare + * @param url2 Second URL to compare + * @return whether the URLs are the same + */ + ComparePlacesURIs(url1, url2) { + url1 = url1 instanceof Ci.nsIURI ? url1.spec : new URL(url1); + if (url1.protocol != "place:") { + throw new Error("Expected a place: uri, got " + url1.href); + } + url2 = url2 instanceof Ci.nsIURI ? url2.spec : new URL(url2); + if (url2.protocol != "place:") { + throw new Error("Expected a place: uri, got " + url2.href); + } + let tokens1 = url1.pathname.split("&").sort().join("&"); + let tokens2 = url2.pathname.split("&").sort().join("&"); + if (tokens1 != tokens2) { + dump(`Failed comparison between:\n${tokens1}\n${tokens2}\n`); + return false; + } + return true; + }, + + /** + * Retrieves a single value from a specified field in a database table, based + * on the given conditions. + * @param {string} table - The name of the database table to query. + * @param {string} field - The name of the field to retrieve a value from. + * @param {Object} conditions - An object containing the conditions to filter + * the query results. The keys represent the names of the columns to filter + * by, and the values represent the filter values. + * @return {Promise} A Promise that resolves to the value of the specified + * field from the database table, or null if the query returns no results. + * @throws If more than one result is found for the given conditions. + */ + async getDatabaseValue(table, field, conditions) { + let { fragment: where, params } = this._buildWhereClause(table, conditions); + let query = `SELECT ${field} FROM ${table} ${where}`; + let conn = await lazy.PlacesUtils.promiseDBConnection(); + let rows = await conn.executeCached(query, params); + if (rows.length > 1) { + throw new Error( + "getDatabaseValue doesn't support returning multiple results" + ); + } + return rows[0]?.getResultByName(field); + }, + + /** + * Updates specified fields in a database table, based on the given + * conditions. + * @param {string} table - The name of the database table to add to. + * @param {string} fields - an object with field, value pairs + * @param {Object} [conditions] - An object containing the conditions to filter + * the query results. The keys represent the names of the columns to filter + * by, and the values represent the filter values. + * @return {Promise} A Promise that resolves to the number of affected rows. + * @throws If no rows were affected. + */ + async updateDatabaseValues(table, fields, conditions = {}) { + let { fragment: where, params } = this._buildWhereClause(table, conditions); + let query = `UPDATE ${table} SET ${Object.keys(fields) + .map(f => f + " = :" + f) + .join()} ${where} RETURNING rowid`; + params = Object.assign(fields, params); + return lazy.PlacesUtils.withConnectionWrapper( + "setDatabaseValue", + async conn => { + let rows = await conn.executeCached(query, params); + if (!rows.length) { + throw new Error("setDatabaseValue didn't update any value"); + } + return rows.length; + } + ); + }, + + _buildWhereClause(table, conditions) { + let fragments = []; + let params = {}; + for (let [column, value] of Object.entries(conditions)) { + if (column == "url") { + if (value instanceof Ci.nsIURI) { + value = value.spec; + } else if (URL.isInstance(value)) { + value = value.href; + } + } + if (column == "url" && table == "moz_places") { + fragments.push("url_hash = hash(:url) AND url = :url"); + } else { + fragments.push(`${column} = :${column}`); + } + params[column] = value; + } + return { + fragment: fragments.length ? `WHERE ${fragments.join(" AND ")}` : "", + params, + }; + }, +}); diff --git a/toolkit/components/places/tests/bookmarks/bookmarks_long_tag.json b/toolkit/components/places/tests/bookmarks/bookmarks_long_tag.json new file mode 100644 index 0000000000..25fef61eb2 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/bookmarks_long_tag.json @@ -0,0 +1,55 @@ +{ + "guid": "root________", + "index": 0, + "id": 1, + "type": "text/x-moz-place-container", + "dateAdded": 1554906792778, + "lastModified": 1554906792778, + "root": "placesRoot", + "children": [ + { + "guid": "unfiled_____", + "index": 0, + "id": 2, + "type": "text/x-moz-place-container", + "dateAdded": 1554906792778, + "lastModified": 1554906792778, + "root": "unfiledBookmarksFolder", + "children": [ + { + "guid": "___guid1____", + "index": 0, + "id": 3, + "charset": "UTF-16", + "tags": "tag0", + "type": "text/x-moz-place", + "dateAdded": 1554906792778, + "lastModified": 1554906792778, + "uri": "http://test0.com/" + }, + { + "guid": "___guid2____", + "index": 1, + "id": 4, + "charset": "UTF-16", + "tags": "tag1,a0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", + "type": "text/x-moz-place", + "dateAdded": 1554906792778, + "lastModified": 1554906792778, + "uri": "http://test1.com/" + }, + { + "guid": "___guid3____", + "index": 2, + "id": 5, + "charset": "UTF-16", + "tags": "tag2", + "type": "text/x-moz-place", + "dateAdded": 1554906792778, + "lastModified": 1554906792778, + "uri": "http://test2.com/" + } + ] + } + ] +} diff --git a/toolkit/components/places/tests/bookmarks/head_bookmarks.js b/toolkit/components/places/tests/bookmarks/head_bookmarks.js new file mode 100644 index 0000000000..e306b4080d --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/head_bookmarks.js @@ -0,0 +1,150 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Import common head. +{ + /* import-globals-from ../head_common.js */ + let commonFile = do_get_file("../head_common.js", false); + let uri = Services.io.newFileURI(commonFile); + Services.scriptloader.loadSubScript(uri.spec, this); +} + +// Put any other stuff relative to this test folder below. + +function expectPlacesObserverNotifications( + types, + checkAllArgs = true, + skipDescendants = false +) { + let notifications = []; + let listener = events => { + for (let event of events) { + switch (event.type) { + case "bookmark-added": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + parentId: event.parentId, + index: event.index, + url: event.url || undefined, + title: event.title, + dateAdded: new Date(event.dateAdded), + guid: event.guid, + parentGuid: event.parentGuid, + source: event.source, + isTagging: event.isTagging, + }); + break; + case "bookmark-removed": + if ( + !( + skipDescendants && + event.isDescendantRemoval && + !PlacesUtils.bookmarks.userContentRoots.includes(event.parentGuid) + ) + ) { + if (checkAllArgs) { + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + parentId: event.parentId, + index: event.index, + url: event.url || null, + guid: event.guid, + parentGuid: event.parentGuid, + source: event.source, + isTagging: event.isTagging, + }); + } else { + notifications.push({ + type: event.type, + guid: event.guid, + }); + } + } + break; + case "bookmark-moved": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + source: event.source, + index: event.index, + oldParentGuid: event.oldParentGuid, + oldIndex: event.oldIndex, + isTagging: event.isTagging, + }); + break; + case "bookmark-tags-changed": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + tags: event.tags, + lastModified: new Date(event.lastModified), + source: event.source, + isTagging: event.isTagging, + }); + break; + case "bookmark-time-changed": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + dateAdded: new Date(event.dateAdded), + lastModified: new Date(event.lastModified), + source: event.source, + isTagging: event.isTagging, + }); + break; + case "bookmark-title-changed": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + title: event.title, + lastModified: new Date(event.lastModified), + source: event.source, + isTagging: event.isTagging, + }); + break; + case "bookmark-url-changed": + notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + source: event.source, + isTagging: event.isTagging, + lastModified: new Date(event.lastModified), + }); + break; + } + } + }; + PlacesUtils.observers.addListener(types, listener); + return { + check(expectedNotifications) { + PlacesUtils.observers.removeListener(types, listener); + Assert.deepEqual(notifications, expectedNotifications); + }, + }; +} diff --git a/toolkit/components/places/tests/bookmarks/test_1016953-renaming-uncompressed.js b/toolkit/components/places/tests/bookmarks/test_1016953-renaming-uncompressed.js new file mode 100644 index 0000000000..3f74430296 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_1016953-renaming-uncompressed.js @@ -0,0 +1,119 @@ +/* Bug 1016953 - When a previous bookmark backup exists with the same hash +regardless of date, an automatic backup should attempt to either rename it to +today's date if the backup was for an old date or leave it alone if it was for +the same date. However if the file ext was json it will accidentally rename it +to jsonlz4 while keeping the json contents +*/ + +add_task(async function test_same_date_same_hash() { + // If old file has been created on the same date and has the same hash + // the file should be left alone + let backupFolder = await PlacesBackups.getBackupFolder(); + // Save to profile dir to obtain hash and nodeCount to append to filename + let tempPath = PathUtils.join( + PathUtils.profileDir, + "bug10169583_bookmarks.json" + ); + let { count, hash } = await BookmarkJSONUtils.exportToFile(tempPath); + + // Save JSON file in backup folder with hash appended + let dateObj = new Date(); + let filename = + "bookmarks-" + + PlacesBackups.toISODateString(dateObj) + + "_" + + count + + "_" + + hash + + ".json"; + let backupFile = PathUtils.join(backupFolder, filename); + await IOUtils.move(tempPath, backupFile); + + // Force a compressed backup which fallbacks to rename + await PlacesBackups.create(); + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + // check to ensure not renamed to jsonlz4 + Assert.equal(mostRecentBackupFile, backupFile); + // inspect contents and check if valid json + info("Check is valid JSON"); + // We initially wrote an uncompressed file, and although a backup was triggered + // it did not rewrite the file, so this is uncompressed. + await IOUtils.readJSON(mostRecentBackupFile); + + // Cleanup + await IOUtils.remove(backupFile); + await IOUtils.remove(tempPath); + PlacesBackups._backupFiles = null; // To force re-cache of backupFiles +}); + +add_task(async function test_same_date_diff_hash() { + // If the old file has been created on the same date, but has a different hash + // the existing file should be overwritten with the newer compressed version + let backupFolder = await PlacesBackups.getBackupFolder(); + let tempPath = PathUtils.join( + PathUtils.profileDir, + "bug10169583_bookmarks.json" + ); + let { count } = await BookmarkJSONUtils.exportToFile(tempPath); + let dateObj = new Date(); + let filename = + "bookmarks-" + + PlacesBackups.toISODateString(dateObj) + + "_" + + count + + "_differentHash==.json"; + let backupFile = PathUtils.join(backupFolder, filename); + await IOUtils.move(tempPath, backupFile); + await PlacesBackups.create(); // Force compressed backup + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + + // Decode lz4 compressed file to json and check if json is valid + info("Check is valid JSON"); + await IOUtils.readJSON(mostRecentBackupFile, { decompress: true }); + + // Cleanup + await IOUtils.remove(mostRecentBackupFile); + await IOUtils.remove(tempPath); + PlacesBackups._backupFiles = null; // To force re-cache of backupFiles +}); + +add_task(async function test_diff_date_same_hash() { + // If the old file has been created on an older day but has the same hash + // it should be renamed with today's date without altering the contents. + let backupFolder = await PlacesBackups.getBackupFolder(); + let tempPath = PathUtils.join( + PathUtils.profileDir, + "bug10169583_bookmarks.json" + ); + let { count, hash } = await BookmarkJSONUtils.exportToFile(tempPath); + let oldDate = new Date(2014, 1, 1); + let curDate = new Date(); + let oldFilename = + "bookmarks-" + + PlacesBackups.toISODateString(oldDate) + + "_" + + count + + "_" + + hash + + ".json"; + let newFilename = + "bookmarks-" + + PlacesBackups.toISODateString(curDate) + + "_" + + count + + "_" + + hash + + ".json"; + let backupFile = PathUtils.join(backupFolder, oldFilename); + let newBackupFile = PathUtils.join(backupFolder, newFilename); + await IOUtils.move(tempPath, backupFile); + + // Ensure file has been renamed correctly + await PlacesBackups.create(); + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + Assert.equal(mostRecentBackupFile, newBackupFile); + + // Cleanup + await IOUtils.remove(mostRecentBackupFile); + await IOUtils.remove(tempPath); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_1017502-bookmarks_foreign_count.js b/toolkit/components/places/tests/bookmarks/test_1017502-bookmarks_foreign_count.js new file mode 100644 index 0000000000..47955a4ea4 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_1017502-bookmarks_foreign_count.js @@ -0,0 +1,117 @@ +/* -*- tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Bug 1017502 - Add a foreign_count column to moz_places +This tests, tests the triggers that adjust the foreign_count when a bookmark is +added or removed and also the maintenance task to fix wrong counts. +*/ + +const T_URI = Services.io.newURI( + "https://www.mozilla.org/firefox/nightly/firstrun/" +); + +async function getForeignCountForURL(conn, url) { + await PlacesTestUtils.promiseAsyncUpdates(); + url = url instanceof Ci.nsIURI ? url.spec : url; + let rows = await conn.executeCached( + `SELECT foreign_count FROM moz_places WHERE url_hash = hash(:t_url) + AND url = :t_url`, + { t_url: url } + ); + return rows[0].getResultByName("foreign_count"); +} + +add_task(async function add_remove_change_bookmark_test() { + let conn = await PlacesUtils.promiseDBConnection(); + + // Simulate a visit to the url + await PlacesTestUtils.addVisits(T_URI); + Assert.equal(await getForeignCountForURL(conn, T_URI), 0); + + // Add 1st bookmark which should increment foreign_count by 1 + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "First Run", + url: T_URI, + }); + Assert.equal(await getForeignCountForURL(conn, T_URI), 1); + + // Add 2nd bookmark + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "First Run", + url: T_URI, + }); + Assert.equal(await getForeignCountForURL(conn, T_URI), 2); + + // Remove 2nd bookmark which should decrement foreign_count by 1 + await PlacesUtils.bookmarks.remove(bm2); + Assert.equal(await getForeignCountForURL(conn, T_URI), 1); + + // Change first bookmark's URI + const URI2 = Services.io.newURI("http://www.mozilla.org"); + bm1.url = URI2; + bm1 = await PlacesUtils.bookmarks.update(bm1); + // Check foreign count for original URI + Assert.equal(await getForeignCountForURL(conn, T_URI), 0); + // Check foreign count for new URI + Assert.equal(await getForeignCountForURL(conn, URI2), 1); + + // Cleanup - Remove changed bookmark + await PlacesUtils.bookmarks.remove(bm1); + Assert.equal(await getForeignCountForURL(conn, URI2), 0); +}); + +add_task(async function maintenance_foreign_count_test() { + let conn = await PlacesUtils.promiseDBConnection(); + + // Simulate a visit to the url + await PlacesTestUtils.addVisits(T_URI); + + // Adjust the foreign_count for the added entry to an incorrect value + await new Promise(resolve => { + let stmt = DBConn().createAsyncStatement( + `UPDATE moz_places SET foreign_count = 10 WHERE url_hash = hash(:t_url) + AND url = :t_url ` + ); + stmt.params.t_url = T_URI.spec; + stmt.executeAsync({ + handleCompletion() { + resolve(); + }, + }); + stmt.finalize(); + }); + Assert.equal(await getForeignCountForURL(conn, T_URI), 10); + + // Run maintenance + const { PlacesDBUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesDBUtils.sys.mjs" + ); + await PlacesDBUtils.maintenanceOnIdle(); + + // Check if the foreign_count has been adjusted to the correct value + Assert.equal(await getForeignCountForURL(conn, T_URI), 0); +}); + +add_task(async function add_remove_tags_test() { + let conn = await PlacesUtils.promiseDBConnection(); + + await PlacesTestUtils.addVisits(T_URI); + Assert.equal(await getForeignCountForURL(conn, T_URI), 0); + + // Check foreign count incremented by 1 for a single tag + PlacesUtils.tagging.tagURI(T_URI, ["test tag"]); + Assert.equal(await getForeignCountForURL(conn, T_URI), 1); + + // Check foreign count is incremented by 2 for two tags + PlacesUtils.tagging.tagURI(T_URI, ["one", "two"]); + Assert.equal(await getForeignCountForURL(conn, T_URI), 3); + + // Check foreign count is set to 0 when all tags are removed + PlacesUtils.tagging.untagURI(T_URI, ["test tag", "one", "two"]); + Assert.equal(await getForeignCountForURL(conn, T_URI), 0); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_1129529.js b/toolkit/components/places/tests/bookmarks/test_1129529.js new file mode 100644 index 0000000000..8fa4731823 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_1129529.js @@ -0,0 +1,24 @@ +// Test that importing bookmark data where a bookmark has a tag longer than 100 +// chars imports everything except the tags for that bookmark. +add_task(async function () { + let bookmarksFile = PathUtils.join( + do_get_cwd().path, + "bookmarks_long_tag.json" + ); + let bookmarksUrl = PathUtils.toFileURI(bookmarksFile); + + await BookmarkJSONUtils.importFromURL(bookmarksUrl); + + let [bookmarks] = await PlacesBackups.getBookmarksTree(); + let unsortedBookmarks = bookmarks.children[2].children; + Assert.equal(unsortedBookmarks.length, 3); + + for (let i = 0; i < unsortedBookmarks.length; ++i) { + let bookmark = unsortedBookmarks[i]; + Assert.equal(bookmark.charset, "UTF-16"); + Assert.equal(bookmark.dateAdded, 1554906792000); + Assert.equal(bookmark.lastModified, 1554906792000); + Assert.equal(bookmark.uri, `http://test${i}.com/`); + Assert.equal(bookmark.tags, `tag${i}`); + } +}); diff --git a/toolkit/components/places/tests/bookmarks/test_384228.js b/toolkit/components/places/tests/bookmarks/test_384228.js new file mode 100644 index 0000000000..0db46353b7 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_384228.js @@ -0,0 +1,93 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * test querying for bookmarks in multiple folders. + */ +add_task(async function search_bookmark_in_folder() { + let testFolder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "bug 384228 test folder 1", + }); + Assert.equal(testFolder1.index, 0); + + let testFolder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "bug 384228 test folder 2", + }); + Assert.equal(testFolder2.index, 1); + + let testFolder3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "bug 384228 test folder 3", + }); + Assert.equal(testFolder3.index, 2); + + let b1 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder1.guid, + url: "http://foo.tld/", + title: "title b1 (folder 1)", + }); + Assert.equal(b1.index, 0); + + let b2 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder1.guid, + url: "http://foo.tld/", + title: "title b2 (folder 1)", + }); + Assert.equal(b2.index, 1); + + let b3 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder2.guid, + url: "http://foo.tld/", + title: "title b3 (folder 2)", + }); + Assert.equal(b3.index, 0); + + let b4 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder3.guid, + url: "http://foo.tld/", + title: "title b4 (folder 3)", + }); + Assert.equal(b4.index, 0); + + // also test recursive search + let testFolder1_1 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder1.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "bug 384228 test folder 1.1", + }); + Assert.equal(testFolder1_1.index, 2); + + let b5 = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder1_1.guid, + url: "http://foo.tld/", + title: "title b5 (folder 1.1)", + }); + Assert.equal(b5.index, 0); + + // query folder 1, folder 2 and get 4 bookmarks + let hs = PlacesUtils.history; + let options = hs.getNewQueryOptions(); + let query = hs.getNewQuery(); + query.searchTerms = "title"; + options.queryType = options.QUERY_TYPE_BOOKMARKS; + query.setParents([testFolder1.guid, testFolder2.guid]); + let rootNode = hs.executeQuery(query, options).root; + rootNode.containerOpen = true; + + // should not match item from folder 3 + Assert.equal(rootNode.childCount, 4); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b1.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b3.guid); + Assert.equal(rootNode.getChild(3).bookmarkGuid, b5.guid); + + rootNode.containerOpen = false; +}); diff --git a/toolkit/components/places/tests/bookmarks/test_385829.js b/toolkit/components/places/tests/bookmarks/test_385829.js new file mode 100644 index 0000000000..9de7c6da17 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_385829.js @@ -0,0 +1,180 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function search_bookmark_by_lastModified_dateDated() { + // test search on folder with various sorts and max results + // see bug #385829 for more details + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "bug 385829 test", + }); + + let now = new Date(); + // ensure some unique values for date added and last modified + // for date added: b1 < b2 < b3 < b4 + // for last modified: b1 > b2 > b3 > b4 + let b1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://a1.com/", + title: "1 title", + dateAdded: new Date(now.getTime() + 1000), + }); + let b2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://a2.com/", + title: "2 title", + dateAdded: new Date(now.getTime() + 2000), + }); + let b3 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://a3.com/", + title: "3 title", + dateAdded: new Date(now.getTime() + 3000), + }); + let b4 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://a4.com/", + title: "4 title", + dateAdded: new Date(now.getTime() + 4000), + }); + + // make sure lastModified is larger than dateAdded + let modifiedTime = new Date(now.getTime() + 5000); + await PlacesUtils.bookmarks.update({ + guid: b1.guid, + lastModified: new Date(modifiedTime.getTime() + 4000), + }); + await PlacesUtils.bookmarks.update({ + guid: b2.guid, + lastModified: new Date(modifiedTime.getTime() + 3000), + }); + await PlacesUtils.bookmarks.update({ + guid: b3.guid, + lastModified: new Date(modifiedTime.getTime() + 2000), + }); + await PlacesUtils.bookmarks.update({ + guid: b4.guid, + lastModified: new Date(modifiedTime.getTime() + 1000), + }); + + let hs = PlacesUtils.history; + let options = hs.getNewQueryOptions(); + let query = hs.getNewQuery(); + options.queryType = options.QUERY_TYPE_BOOKMARKS; + options.maxResults = 3; + query.setParents([folder.guid]); + + let result = hs.executeQuery(query, options); + let rootNode = result.root; + rootNode.containerOpen = true; + + // test SORT_BY_DATEADDED_ASCENDING (live update) + result.sortingMode = options.SORT_BY_DATEADDED_ASCENDING; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b1.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b3.guid); + Assert.ok(rootNode.getChild(0).dateAdded < rootNode.getChild(1).dateAdded); + Assert.ok(rootNode.getChild(1).dateAdded < rootNode.getChild(2).dateAdded); + + // test SORT_BY_DATEADDED_DESCENDING (live update) + result.sortingMode = options.SORT_BY_DATEADDED_DESCENDING; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b3.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b1.guid); + Assert.ok(rootNode.getChild(0).dateAdded > rootNode.getChild(1).dateAdded); + Assert.ok(rootNode.getChild(1).dateAdded > rootNode.getChild(2).dateAdded); + + // test SORT_BY_LASTMODIFIED_ASCENDING (live update) + result.sortingMode = options.SORT_BY_LASTMODIFIED_ASCENDING; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b3.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b1.guid); + Assert.ok( + rootNode.getChild(0).lastModified < rootNode.getChild(1).lastModified + ); + Assert.ok( + rootNode.getChild(1).lastModified < rootNode.getChild(2).lastModified + ); + + // test SORT_BY_LASTMODIFIED_DESCENDING (live update) + result.sortingMode = options.SORT_BY_LASTMODIFIED_DESCENDING; + + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b1.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b3.guid); + Assert.ok( + rootNode.getChild(0).lastModified > rootNode.getChild(1).lastModified + ); + Assert.ok( + rootNode.getChild(1).lastModified > rootNode.getChild(2).lastModified + ); + rootNode.containerOpen = false; + + // test SORT_BY_DATEADDED_ASCENDING + options.sortingMode = options.SORT_BY_DATEADDED_ASCENDING; + result = hs.executeQuery(query, options); + rootNode = result.root; + rootNode.containerOpen = true; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b1.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b3.guid); + Assert.ok(rootNode.getChild(0).dateAdded < rootNode.getChild(1).dateAdded); + Assert.ok(rootNode.getChild(1).dateAdded < rootNode.getChild(2).dateAdded); + rootNode.containerOpen = false; + + // test SORT_BY_DATEADDED_DESCENDING + options.sortingMode = options.SORT_BY_DATEADDED_DESCENDING; + result = hs.executeQuery(query, options); + rootNode = result.root; + rootNode.containerOpen = true; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b4.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b3.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b2.guid); + Assert.ok(rootNode.getChild(0).dateAdded > rootNode.getChild(1).dateAdded); + Assert.ok(rootNode.getChild(1).dateAdded > rootNode.getChild(2).dateAdded); + rootNode.containerOpen = false; + + // test SORT_BY_LASTMODIFIED_ASCENDING + options.sortingMode = options.SORT_BY_LASTMODIFIED_ASCENDING; + result = hs.executeQuery(query, options); + rootNode = result.root; + rootNode.containerOpen = true; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b4.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b3.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b2.guid); + Assert.ok( + rootNode.getChild(0).lastModified < rootNode.getChild(1).lastModified + ); + Assert.ok( + rootNode.getChild(1).lastModified < rootNode.getChild(2).lastModified + ); + rootNode.containerOpen = false; + + // test SORT_BY_LASTMODIFIED_DESCENDING + options.sortingMode = options.SORT_BY_LASTMODIFIED_DESCENDING; + result = hs.executeQuery(query, options); + rootNode = result.root; + rootNode.containerOpen = true; + Assert.equal(rootNode.childCount, 3); + Assert.equal(rootNode.getChild(0).bookmarkGuid, b1.guid); + Assert.equal(rootNode.getChild(1).bookmarkGuid, b2.guid); + Assert.equal(rootNode.getChild(2).bookmarkGuid, b3.guid); + Assert.ok( + rootNode.getChild(0).lastModified > rootNode.getChild(1).lastModified + ); + Assert.ok( + rootNode.getChild(1).lastModified > rootNode.getChild(2).lastModified + ); + rootNode.containerOpen = false; +}); diff --git a/toolkit/components/places/tests/bookmarks/test_388695.js b/toolkit/components/places/tests/bookmarks/test_388695.js new file mode 100644 index 0000000000..337d8176bd --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_388695.js @@ -0,0 +1,45 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Get bookmark service +let bm = PlacesUtils.bookmarks; + +// Test that Bookmarks fetch properly orders its results based on +// the last modified value. Note we cannot rely on dateAdded due to +// the low PR_Now() resolution. + +add_task(async function sort_bookmark_by_relevance() { + let now = new Date(); + let modifiedTime = new Date(now.setHours(now.getHours() - 2)); + + let url = "http://foo.tld.com/"; + let parentGuid = ( + await bm.insert({ + type: bm.TYPE_FOLDER, + title: "test folder", + parentGuid: bm.unfiledGuid, + }) + ).guid; + let item1Guid = (await bm.insert({ url, parentGuid })).guid; + let item2Guid = ( + await bm.insert({ + url, + parentGuid, + dateAdded: modifiedTime, + lastModified: modifiedTime, + }) + ).guid; + let bms = []; + await bm.fetch({ url }, bm1 => bms.push(bm1)); + Assert.equal(bms[0].guid, item1Guid); + Assert.equal(bms[1].guid, item2Guid); + await bm.update({ guid: item2Guid, title: "modified" }); + + let bms1 = []; + await bm.fetch({ url }, bm2 => bms1.push(bm2)); + Assert.equal(bms1[0].guid, item2Guid); + Assert.equal(bms1[1].guid, item1Guid); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_393498.js b/toolkit/components/places/tests/bookmarks/test_393498.js new file mode 100644 index 0000000000..ab7cf57f61 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_393498.js @@ -0,0 +1,161 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var observer = { + handlePlacesEvents(events) { + for (const event of events) { + switch (event.type) { + case "bookmark-added": { + this._itemAddedId = event.id; + this._itemAddedParent = event.parentId; + this._itemAddedIndex = event.index; + break; + } + case "bookmark-time-changed": { + this._itemTimeChangedGuid = event.guid; + this._itemTimeChangedDateAdded = event.dateAdded; + this._itemTimeChangedLastModified = event.lastModified; + break; + } + case "bookmark-title-changed": { + this._itemTitleChangedId = event.id; + this._itemTitleChangedTitle = event.title; + break; + } + } + } + }, +}; + +observer.handlePlacesEvents = observer.handlePlacesEvents.bind(observer); +PlacesUtils.observers.addListener( + ["bookmark-added", "bookmark-time-changed", "bookmark-title-changed"], + observer.handlePlacesEvents +); + +registerCleanupFunction(function () { + PlacesUtils.observers.removeListener( + ["bookmark-added", "bookmark-time-changed", "bookmark-title-changed"], + observer.handlePlacesEvents + ); +}); + +// Returns do_check_eq with .getTime() added onto parameters +function do_check_date_eq(t1, t2) { + return Assert.equal(t1.getTime(), t2.getTime()); +} + +add_task(async function test_bookmark_update_notifications() { + // We set times in the past to workaround a timing bug due to virtual + // machines and the skew between PR_Now() and Date.now(), see bug 427142 and + // bug 858377 for details. + const PAST_DATE = new Date(Date.now() - 86400000); + + // Insert a new bookmark. + let testFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "test Folder", + parentGuid: PlacesUtils.bookmarks.menuGuid, + }); + + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: testFolder.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://google.com/", + title: "a bookmark", + }); + + // Sanity check. + Assert.ok(observer.itemChangedProperty === undefined); + + // Set dateAdded in the past and verify the changes. + await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + dateAdded: PAST_DATE, + }); + + Assert.equal(observer._itemTimeChangedGuid, bookmark.guid); + Assert.equal(observer._itemTimeChangedDateAdded, PAST_DATE.getTime()); + + // After just inserting, modified should be the same as dateAdded. + do_check_date_eq(bookmark.lastModified, bookmark.dateAdded); + + let updatedBookmark = await PlacesUtils.bookmarks.fetch({ + guid: bookmark.guid, + }); + + do_check_date_eq(updatedBookmark.dateAdded, PAST_DATE); + + // Set lastModified in the past and verify the changes. + updatedBookmark = await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + lastModified: PAST_DATE, + }); + + Assert.equal(observer._itemTimeChangedGuid, bookmark.guid); + Assert.equal(observer._itemTimeChangedLastModified, PAST_DATE.getTime()); + do_check_date_eq(updatedBookmark.lastModified, PAST_DATE); + + // Set bookmark title + updatedBookmark = await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + title: "Google", + }); + + // Test notifications. + Assert.equal( + observer._itemTitleChangedId, + await PlacesUtils.promiseItemId(bookmark.guid) + ); + Assert.equal(observer._itemTitleChangedTitle, "Google"); + + // Check lastModified has been updated. + Assert.ok(is_time_ordered(PAST_DATE, updatedBookmark.lastModified.getTime())); + + // Check that node properties are updated. + let root = PlacesUtils.getFolderContents(testFolder.guid).root; + Assert.equal(root.childCount, 1); + let childNode = root.getChild(0); + + // confirm current dates match node properties + Assert.equal( + PlacesUtils.toPRTime(updatedBookmark.dateAdded), + childNode.dateAdded + ); + Assert.equal( + PlacesUtils.toPRTime(updatedBookmark.lastModified), + childNode.lastModified + ); + + // Test live update of lastModified when setting title. + updatedBookmark = await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + lastModified: PAST_DATE, + title: "Google", + }); + + // Check lastModified has been updated. + Assert.ok(is_time_ordered(PAST_DATE, childNode.lastModified)); + // Test that node value matches db value. + Assert.equal( + PlacesUtils.toPRTime(updatedBookmark.lastModified), + childNode.lastModified + ); + + // Test live update of the exposed date apis. + await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + dateAdded: PAST_DATE, + }); + Assert.equal(childNode.dateAdded, PlacesUtils.toPRTime(PAST_DATE)); + await PlacesUtils.bookmarks.update({ + guid: bookmark.guid, + lastModified: PAST_DATE, + }); + Assert.equal(childNode.lastModified, PlacesUtils.toPRTime(PAST_DATE)); + + root.containerOpen = false; +}); diff --git a/toolkit/components/places/tests/bookmarks/test_405938_restore_queries.js b/toolkit/components/places/tests/bookmarks/test_405938_restore_queries.js new file mode 100644 index 0000000000..c893f1db3f --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_405938_restore_queries.js @@ -0,0 +1,253 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var tests = []; + +/* + +Backup/restore tests example: + +var myTest = { + populate: function () { ... add bookmarks ... }, + validate: function () { ... query for your bookmarks ... } +} + +this.push(myTest); + +*/ + +/* + +test summary: +- create folders with content +- create a query bookmark for those folders +- backs up bookmarks +- restores bookmarks +- confirms that the query has the new ids for the same folders + +scenarios: +- 1 folder (folder shortcut) +- n folders (single query) +- n folders (multiple queries) + +*/ + +var test = { + _testRootId: null, + _testRootTitle: "test root", + _folderGuids: [], + _bookmarkURIs: [], + _count: 3, + _extraBookmarksCount: 10, + + populate: async function populate() { + // folder to hold this test + await PlacesUtils.bookmarks.eraseEverything(); + + let testFolderItems = []; + // Set a date 60 seconds ago, so that we can set newer bookmarks later. + let dateAdded = new Date(new Date() - 60000); + + // create test folders each with a bookmark + for (let i = 0; i < this._count; i++) { + this._folderGuids.push(PlacesUtils.history.makeGuid()); + testFolderItems.push({ + guid: this._folderGuids[i], + title: `folder${i}`, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + dateAdded, + children: [ + { + dateAdded, + url: `http://${i}`, + title: `bookmark${i}`, + }, + ], + }); + } + + let bookmarksTree = { + guid: PlacesUtils.bookmarks.toolbarGuid, + children: [ + { + dateAdded, + title: this._testRootTitle, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children: testFolderItems, + }, + ], + }; + + let insertedBookmarks = await PlacesUtils.bookmarks.insertTree( + bookmarksTree + ); + + // create a query URI with 1 folder (ie: folder shortcut) + this._queryURI1 = `place:parent=${this._folderGuids[0]}&queryType=1`; + this._queryTitle1 = "query1"; + await PlacesUtils.bookmarks.insert({ + parentGuid: insertedBookmarks[0].guid, + dateAdded, + url: this._queryURI1, + title: this._queryTitle1, + }); + + // create a query URI with _count folders + this._queryURI2 = `place:parent=${this._folderGuids.join( + "&parent=" + )}&queryType=1`; + this._queryTitle2 = "query2"; + await PlacesUtils.bookmarks.insert({ + parentGuid: insertedBookmarks[0].guid, + dateAdded, + url: this._queryURI2, + title: this._queryTitle2, + }); + + // Create a query URI for most recent bookmarks with NO folders specified. + this._queryURI3 = + "place:queryType=1&sort=12&maxResults=10&excludeQueries=1"; + this._queryTitle3 = "query3"; + await PlacesUtils.bookmarks.insert({ + parentGuid: insertedBookmarks[0].guid, + dateAdded, + url: this._queryURI3, + title: this._queryTitle3, + }); + }, + + clean() {}, + + validate: async function validate(addExtras) { + if (addExtras) { + // Throw a wrench in the works by inserting some new bookmarks, + // ensuring folder ids won't be the same, when restoring. + let date = new Date() - this._extraBookmarksCount * 1000; + for (let i = 0; i < this._extraBookmarksCount; i++) { + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: uri("http://aaaa" + i), + dateAdded: new Date(date + (this._extraBookmarksCount - i) * 1000), + }); + } + } + + var toolbar = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid, + false, + true + ).root; + Assert.equal(toolbar.childCount, 1); + + var folderNode = toolbar.getChild(0); + Assert.equal(folderNode.type, folderNode.RESULT_TYPE_FOLDER); + Assert.equal(folderNode.title, this._testRootTitle); + folderNode.QueryInterface(Ci.nsINavHistoryQueryResultNode); + folderNode.containerOpen = true; + + // |_count| folders + the query nodes + Assert.equal(folderNode.childCount, this._count + 3); + + for (let i = 0; i < this._count; i++) { + var subFolder = folderNode.getChild(i); + Assert.equal(subFolder.title, "folder" + i); + subFolder.QueryInterface(Ci.nsINavHistoryContainerResultNode); + subFolder.containerOpen = true; + Assert.equal(subFolder.childCount, 1); + var child = subFolder.getChild(0); + Assert.equal(child.title, "bookmark" + i); + Assert.ok(uri(child.uri).equals(uri("http://" + i))); + } + + // validate folder shortcut + this.validateQueryNode1(folderNode.getChild(this._count)); + + // validate folders query + this.validateQueryNode2(folderNode.getChild(this._count + 1)); + + // validate recent folders query + this.validateQueryNode3(folderNode.getChild(this._count + 2)); + + // clean up + folderNode.containerOpen = false; + toolbar.containerOpen = false; + }, + + validateQueryNode1: function validateQueryNode1(aNode) { + Assert.equal(aNode.title, this._queryTitle1); + Assert.ok(PlacesUtils.nodeIsFolder(aNode)); + + aNode.QueryInterface(Ci.nsINavHistoryContainerResultNode); + aNode.containerOpen = true; + Assert.equal(aNode.childCount, 1); + var child = aNode.getChild(0); + Assert.ok(uri(child.uri).equals(uri("http://0"))); + Assert.equal(child.title, "bookmark0"); + aNode.containerOpen = false; + }, + + validateQueryNode2: function validateQueryNode2(aNode) { + Assert.equal(aNode.title, this._queryTitle2); + Assert.ok(PlacesUtils.nodeIsQuery(aNode)); + + aNode.QueryInterface(Ci.nsINavHistoryContainerResultNode); + aNode.containerOpen = true; + Assert.equal(aNode.childCount, this._count); + for (var i = 0; i < aNode.childCount; i++) { + var child = aNode.getChild(i); + Assert.ok(uri(child.uri).equals(uri("http://" + i))); + Assert.equal(child.title, "bookmark" + i); + } + aNode.containerOpen = false; + }, + + validateQueryNode3(aNode) { + Assert.equal(aNode.title, this._queryTitle3); + Assert.ok(PlacesUtils.nodeIsQuery(aNode)); + + aNode.QueryInterface(Ci.nsINavHistoryContainerResultNode); + aNode.containerOpen = true; + // The query will list the extra bookmarks added at the start of validate. + Assert.equal(aNode.childCount, this._extraBookmarksCount); + for (var i = 0; i < aNode.childCount; i++) { + var child = aNode.getChild(i); + Assert.equal(child.uri, `http://aaaa${i}/`); + } + aNode.containerOpen = false; + }, +}; +tests.push(test); + +add_task(async function () { + // make json file + let jsonFile = PathUtils.join(PathUtils.profileDir, "bookmarks.json"); + + // populate db + for (let singleTest of tests) { + await singleTest.populate(); + // sanity + await singleTest.validate(true); + } + + // export json to file + await BookmarkJSONUtils.exportToFile(jsonFile); + + // clean + for (let singleTest of tests) { + singleTest.clean(); + } + + // restore json file + await BookmarkJSONUtils.importFromFile(jsonFile, { replace: true }); + + // validate + for (let singleTest of tests) { + await singleTest.validate(false); + } + + // clean up + await IOUtils.remove(jsonFile); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_424958-json-quoted-folders.js b/toolkit/components/places/tests/bookmarks/test_424958-json-quoted-folders.js new file mode 100644 index 0000000000..2a7ce3f003 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_424958-json-quoted-folders.js @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const FOLDER_TITLE = '"quoted folder"'; + +function checkQuotedFolder() { + let toolbar = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid + ).root; + + // test for our quoted folder + Assert.equal(toolbar.childCount, 1); + var folderNode = toolbar.getChild(0); + Assert.equal(folderNode.type, folderNode.RESULT_TYPE_FOLDER); + Assert.equal(folderNode.title, FOLDER_TITLE); + + // clean up + toolbar.containerOpen = false; +} + +add_task(async function () { + // make json file + let jsonFile = PathUtils.join(PathUtils.profileDir, "bookmarks.json"); + + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: FOLDER_TITLE, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + checkQuotedFolder(); + + // export json to file + await BookmarkJSONUtils.exportToFile(jsonFile); + + await PlacesUtils.bookmarks.remove(folder.guid); + + // restore json file + await BookmarkJSONUtils.importFromFile(jsonFile, { replace: true }); + + checkQuotedFolder(); + + // clean up + await IOUtils.remove(jsonFile); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_448584.js b/toolkit/components/places/tests/bookmarks/test_448584.js new file mode 100644 index 0000000000..07665327b1 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_448584.js @@ -0,0 +1,90 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Get database connection +try { + var mDBConn = PlacesUtils.history.DBConnection; +} catch (ex) { + do_throw("Could not get database connection\n"); +} + +/* + This test is: + - don't try to add invalid uri nodes to a JSON backup +*/ + +const ITEM_TITLE = "invalid uri"; +const ITEM_URL = "http://test.mozilla.org"; + +function validateResults(expectedValidItemsCount) { + var query = PlacesUtils.history.getNewQuery(); + query.setParents([PlacesUtils.bookmarks.toolbarGuid]); + var options = PlacesUtils.history.getNewQueryOptions(); + var result = PlacesUtils.history.executeQuery(query, options); + + var toolbar = result.root; + toolbar.containerOpen = true; + + // test for our bookmark + Assert.equal(toolbar.childCount, expectedValidItemsCount); + for (var i = 0; i < toolbar.childCount; i++) { + var folderNode = toolbar.getChild(0); + Assert.equal(folderNode.type, folderNode.RESULT_TYPE_URI); + Assert.equal(folderNode.title, ITEM_TITLE); + } + + // clean up + toolbar.containerOpen = false; +} + +add_task(async function () { + // make json file + let jsonFile = PathUtils.join(PathUtils.profileDir, "bookmarks.json"); + + // populate db + // add a valid bookmark + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: ITEM_TITLE, + url: ITEM_URL, + }); + + let badBookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: ITEM_TITLE, + url: ITEM_URL, + }); + // sanity + validateResults(2); + // Something in the code went wrong and we finish up losing the place, so + // the bookmark uri becomes null. + var sql = "UPDATE moz_bookmarks SET fk = 1337 WHERE guid = ?1"; + var stmt = mDBConn.createStatement(sql); + stmt.bindByIndex(0, badBookmark.guid); + try { + stmt.execute(); + } finally { + stmt.finalize(); + } + + await BookmarkJSONUtils.exportToFile(jsonFile); + + // clean + await PlacesUtils.bookmarks.remove(badBookmark); + + // restore json file + try { + await BookmarkJSONUtils.importFromFile(jsonFile, { replace: true }); + } catch (ex) { + do_throw("couldn't import the exported file: " + ex); + } + + // validate + validateResults(1); + + // clean up + await IOUtils.remove(jsonFile); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_458683.js b/toolkit/components/places/tests/bookmarks/test_458683.js new file mode 100644 index 0000000000..d31fca66e1 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_458683.js @@ -0,0 +1,111 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + This test is: + - don't block while doing backup and restore if tag containers contain + bogus items (separators, folders) +*/ + +const ITEM_TITLE = "invalid uri"; +const ITEM_URL = "http://test.mozilla.org/"; +const TAG_NAME = "testTag"; + +function validateResults() { + let toolbar = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid + ).root; + // test for our bookmark + Assert.equal(toolbar.childCount, 1); + for (var i = 0; i < toolbar.childCount; i++) { + var folderNode = toolbar.getChild(0); + Assert.equal(folderNode.type, folderNode.RESULT_TYPE_URI); + Assert.equal(folderNode.title, ITEM_TITLE); + } + toolbar.containerOpen = false; + + // test for our tag + var tags = PlacesUtils.tagging.getTagsForURI(Services.io.newURI(ITEM_URL)); + Assert.equal(tags.length, 1); + Assert.equal(tags[0], TAG_NAME); +} + +add_task(async function () { + let jsonFile = PathUtils.join(PathUtils.profileDir, "bookmarks.json"); + + // add a valid bookmark + let item = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: ITEM_TITLE, + url: ITEM_URL, + }); + + // create a tag + PlacesUtils.tagging.tagURI(Services.io.newURI(ITEM_URL), [TAG_NAME]); + // get tag folder id + let tagRoot = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.tagsGuid + ).root; + Assert.equal(tagRoot.childCount, 1); + let tagItemGuid = PlacesUtils.asContainer(tagRoot.getChild(0)).bookmarkGuid; + tagRoot.containerOpen = false; + + function insert({ type, parentGuid }) { + return PlacesUtils.withConnectionWrapper( + "test_458683: insert", + async db => { + await db.executeCached( + `INSERT INTO moz_bookmarks (type, parent, position, guid) + VALUES (:type, + (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid), + (SELECT MAX(position) + 1 FROM moz_bookmarks WHERE parent = (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid)), + GENERATE_GUID())`, + { type, parentGuid } + ); + } + ); + } + + // add a separator and a folder inside tag folder + // We must insert these manually, because the new bookmarking API doesn't + // support inserting invalid items into the tag folder. + await insert({ + parentGuid: tagItemGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + await insert({ + parentGuid: tagItemGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // add a separator and a folder inside tag root + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.tagsGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "test tags root folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // sanity + validateResults(); + + await BookmarkJSONUtils.exportToFile(jsonFile); + + // clean + PlacesUtils.tagging.untagURI(Services.io.newURI(ITEM_URL), [TAG_NAME]); + await PlacesUtils.bookmarks.remove(item); + + // restore json file + await BookmarkJSONUtils.importFromFile(jsonFile, { replace: true }); + + validateResults(); + + // clean up + await IOUtils.remove(jsonFile); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_466303-json-remove-backups.js b/toolkit/components/places/tests/bookmarks/test_466303-json-remove-backups.js new file mode 100644 index 0000000000..4596ed93b2 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_466303-json-remove-backups.js @@ -0,0 +1,86 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Since PlacesBackups.getbackupFiles() is a lazy getter, these tests must +// run in the given order, to avoid making it out-of-sync. + +async function countChildren(path) { + let children = await IOUtils.getChildren(path); + let count = 0; + let lastBackupPath = null; + for (let entry of children) { + count++; + if (PlacesBackups.filenamesRegex.test(PathUtils.filename(entry))) { + lastBackupPath = entry; + } + } + return { count, lastBackupPath }; +} + +add_task(async function check_max_backups_is_respected() { + // Get bookmarkBackups directory + let backupFolder = await PlacesBackups.getBackupFolder(); + + // Create 2 json dummy backups in the past. + let oldJsonPath = PathUtils.join(backupFolder, "bookmarks-2008-01-01.json"); + await IOUtils.writeUTF8(oldJsonPath, ""); + Assert.ok(await IOUtils.exists(oldJsonPath)); + + let jsonPath = PathUtils.join(backupFolder, "bookmarks-2008-01-31.json"); + await IOUtils.writeUTF8(jsonPath, ""); + Assert.ok(await IOUtils.exists(jsonPath)); + + // Export bookmarks to JSON. + // Allow 2 backups, the older one should be removed. + await PlacesBackups.create(2); + + let { count, lastBackupPath } = await countChildren(backupFolder); + Assert.equal(count, 2); + Assert.notEqual(lastBackupPath, null); + Assert.equal(false, await IOUtils.exists(oldJsonPath)); + Assert.ok(await IOUtils.exists(jsonPath)); +}); + +add_task(async function check_max_backups_greater_than_backups() { + // Get bookmarkBackups directory + let backupFolder = await PlacesBackups.getBackupFolder(); + + // Export bookmarks to JSON. + // Allow 3 backups, none should be removed. + await PlacesBackups.create(3); + + let { count, lastBackupPath } = await countChildren(backupFolder); + Assert.equal(count, 2); + Assert.notEqual(lastBackupPath, null); +}); + +add_task(async function check_max_backups_null() { + // Get bookmarkBackups directory + let backupFolder = await PlacesBackups.getBackupFolder(); + + // Export bookmarks to JSON. + // Allow infinite backups, none should be removed, a new one is not created + // since one for today already exists. + await PlacesBackups.create(null); + + let { count, lastBackupPath } = await countChildren(backupFolder); + Assert.equal(count, 2); + Assert.notEqual(lastBackupPath, null); +}); + +add_task(async function check_max_backups_undefined() { + // Get bookmarkBackups directory + let backupFolder = await PlacesBackups.getBackupFolder(); + + // Export bookmarks to JSON. + // Allow infinite backups, none should be removed, a new one is not created + // since one for today already exists. + await PlacesBackups.create(); + + let { count, lastBackupPath } = await countChildren(backupFolder); + Assert.equal(count, 2); + Assert.notEqual(lastBackupPath, null); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_477583_json-backup-in-future.js b/toolkit/components/places/tests/bookmarks/test_477583_json-backup-in-future.js new file mode 100644 index 0000000000..ab4f4a02d5 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_477583_json-backup-in-future.js @@ -0,0 +1,56 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +add_task(async function test_json_backup_in_future() { + let backupFolder = await PlacesBackups.getBackupFolder(); + let bookmarksBackupDir = new FileUtils.File(backupFolder); + // Remove all files from backups folder. + let files = bookmarksBackupDir.directoryEntries; + while (files.hasMoreElements()) { + let entry = files.nextFile; + entry.remove(false); + } + + // Create a json dummy backup in the future. + let dateObj = new Date(); + dateObj.setYear(dateObj.getFullYear() + 1); + let name = PlacesBackups.getFilenameForDate(dateObj); + Assert.equal( + name, + "bookmarks-" + PlacesBackups.toISODateString(dateObj) + ".json" + ); + files = bookmarksBackupDir.directoryEntries; + while (files.hasMoreElements()) { + let entry = files.nextFile; + if (PlacesBackups.filenamesRegex.test(entry.leafName)) { + entry.remove(false); + } + } + + let futureBackupFile = bookmarksBackupDir.clone(); + futureBackupFile.append(name); + futureBackupFile.create(Ci.nsIFile.NORMAL_FILE_TYPE, 0o600); + Assert.ok(futureBackupFile.exists()); + + Assert.equal((await PlacesBackups.getBackupFiles()).length, 0); + + await PlacesBackups.create(); + // Check that a backup for today has been created. + Assert.equal((await PlacesBackups.getBackupFiles()).length, 1); + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + Assert.notEqual(mostRecentBackupFile, null); + Assert.ok( + PlacesBackups.filenamesRegex.test(PathUtils.filename(mostRecentBackupFile)) + ); + + // Check that future backup has been removed. + Assert.ok(!futureBackupFile.exists()); + + // Cleanup. + mostRecentBackupFile = new FileUtils.File(mostRecentBackupFile); + mostRecentBackupFile.remove(false); + Assert.ok(!mostRecentBackupFile.exists()); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_818584-discard-duplicate-backups.js b/toolkit/components/places/tests/bookmarks/test_818584-discard-duplicate-backups.js new file mode 100644 index 0000000000..be5d53f8c6 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_818584-discard-duplicate-backups.js @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Checks that automatically created bookmark backups are discarded if they are + * duplicate of an existing ones. + */ +add_task(async function () { + // Create a backup for yesterday in the backups folder. + let backupFolder = await PlacesBackups.getBackupFolder(); + let dateObj = new Date(); + dateObj.setDate(dateObj.getDate() - 1); + let oldBackupName = PlacesBackups.getFilenameForDate(dateObj); + let oldBackup = PathUtils.join(backupFolder, oldBackupName); + let { count: count, hash: hash } = await BookmarkJSONUtils.exportToFile( + oldBackup + ); + Assert.ok(count > 0); + Assert.equal(hash.length, 24); + oldBackupName = oldBackupName.replace( + /\.json/, + "_" + count + "_" + hash + ".json" + ); + await IOUtils.move(oldBackup, PathUtils.join(backupFolder, oldBackupName)); + + // Create a backup. + // This should just rename the existing backup, so in the end there should be + // only one backup with today's date. + await PlacesBackups.create(); + + // Get the hash of the generated backup + let backupFiles = await PlacesBackups.getBackupFiles(); + Assert.equal(backupFiles.length, 1); + + let matches = PathUtils.filename(backupFiles[0]).match( + PlacesBackups.filenamesRegex + ); + Assert.equal(matches[1], PlacesBackups.toISODateString(new Date())); + Assert.equal(matches[2], count); + Assert.equal(matches[3], hash); + + // Add a bookmark and create another backup. + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "foo", + url: "http://foo.com", + }); + + // We must enforce a backup since one for today already exists. The forced + // backup will replace the existing one. + await PlacesBackups.create(undefined, true); + Assert.equal(backupFiles.length, 1); + let recentBackup = await PlacesBackups.getMostRecentBackup(); + Assert.notEqual(recentBackup, PathUtils.join(backupFolder, oldBackupName)); + matches = PathUtils.filename(recentBackup).match( + PlacesBackups.filenamesRegex + ); + Assert.equal(matches[1], PlacesBackups.toISODateString(new Date())); + Assert.equal(matches[2], count + 1); + Assert.notEqual(matches[3], hash); + + // Clean up + await PlacesUtils.bookmarks.remove(bookmark); + await PlacesBackups.create(0); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_818587_compress-bookmarks-backups.js b/toolkit/components/places/tests/bookmarks/test_818587_compress-bookmarks-backups.js new file mode 100644 index 0000000000..91e0c50f7e --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_818587_compress-bookmarks-backups.js @@ -0,0 +1,61 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function compress_bookmark_backups_test() { + // Check for jsonlz4 extension + let todayFilename = PlacesBackups.getFilenameForDate( + new Date(2014, 4, 15), + true + ); + Assert.equal(todayFilename, "bookmarks-2014-05-15.jsonlz4"); + + await PlacesBackups.create(); + + // Check that a backup for today has been created and the regex works fine for lz4. + Assert.equal((await PlacesBackups.getBackupFiles()).length, 1); + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + Assert.notEqual(mostRecentBackupFile, null); + Assert.ok( + PlacesBackups.filenamesRegex.test(PathUtils.filename(mostRecentBackupFile)) + ); + + // The most recent backup file has to be removed since saveBookmarksToJSONFile + // will otherwise over-write the current backup, since it will be made on the + // same date + await IOUtils.remove(mostRecentBackupFile); + Assert.equal(false, await IOUtils.exists(mostRecentBackupFile)); + + // Check that, if the user created a custom backup out of the default + // backups folder, it gets copied (compressed) into it. + let jsonFile = PathUtils.join(PathUtils.profileDir, "bookmarks.json"); + await PlacesBackups.saveBookmarksToJSONFile(jsonFile); + Assert.equal((await PlacesBackups.getBackupFiles()).length, 1); + + // Check if import works from lz4 compressed json + let url = "http://www.mozilla.org/en-US/"; + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "bookmark", + url, + }); + + // Force create a compressed backup, Remove the bookmark, the restore the backup + await PlacesBackups.create(undefined, true); + let recentBackup = await PlacesBackups.getMostRecentBackup(); + await PlacesUtils.bookmarks.remove(bm); + await BookmarkJSONUtils.importFromFile(recentBackup, { replace: true }); + let root = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.unfiledGuid + ).root; + let node = root.getChild(0); + Assert.equal(node.uri, url); + + root.containerOpen = false; + await PlacesUtils.bookmarks.eraseEverything(); + + // Cleanup. + await IOUtils.remove(jsonFile); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_818593-store-backup-metadata.js b/toolkit/components/places/tests/bookmarks/test_818593-store-backup-metadata.js new file mode 100644 index 0000000000..6d280e8cad --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_818593-store-backup-metadata.js @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * To confirm that metadata i.e. bookmark count is set and retrieved for + * automatic backups. + */ +add_task(async function test_saveBookmarksToJSONFile_and_create() { + // Add a bookmark + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "Get Firefox!", + url: "http://getfirefox.com/", + }); + + // Test saveBookmarksToJSONFile() + let backupFile = PathUtils.join(PathUtils.tempDir, "bookmarks.json"); + + let nodeCount = await PlacesBackups.saveBookmarksToJSONFile(backupFile, true); + Assert.ok(nodeCount > 0); + Assert.ok(await IOUtils.exists(backupFile)); + + // Ensure the backup would be copied to our backups folder when the original + // backup is saved somewhere else. + let recentBackup = await PlacesBackups.getMostRecentBackup(); + let matches = PathUtils.filename(recentBackup).match( + PlacesBackups.filenamesRegex + ); + Assert.equal(matches[2], nodeCount); + Assert.equal(matches[3].length, 24); + + // Clear all backups in our backups folder. + await PlacesBackups.create(0); + Assert.equal((await PlacesBackups.getBackupFiles()).length, 0); + + // Test create() which saves bookmarks with metadata on the filename. + await PlacesBackups.create(); + Assert.equal((await PlacesBackups.getBackupFiles()).length, 1); + + let mostRecentBackupFile = await PlacesBackups.getMostRecentBackup(); + Assert.notEqual(mostRecentBackupFile, null); + matches = PathUtils.filename(recentBackup).match( + PlacesBackups.filenamesRegex + ); + Assert.equal(matches[2], nodeCount); + Assert.equal(matches[3].length, 24); + + // Cleanup + await IOUtils.remove(backupFile); + await PlacesBackups.create(0); + await PlacesUtils.bookmarks.remove(bookmark); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_992901-backup-unsorted-hierarchy.js b/toolkit/components/places/tests/bookmarks/test_992901-backup-unsorted-hierarchy.js new file mode 100644 index 0000000000..c835a3bd09 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_992901-backup-unsorted-hierarchy.js @@ -0,0 +1,65 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Checks that backups properly include all of the bookmarks if the hierarchy + * in the database is unordered so that a hierarchy is defined before its + * ancestor in the bookmarks table. + */ +add_task(async function () { + let bms = await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + title: "bookmark", + url: "http://mozilla.org", + }, + { + title: "f2", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { + title: "f1", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }, + ], + }); + + let bookmark = bms[0]; + let folder2 = bms[1]; + let folder1 = bms[2]; + bookmark.parentGuid = folder2.guid; + await PlacesUtils.bookmarks.update(bookmark); + + folder2.parentGuid = folder1.guid; + await PlacesUtils.bookmarks.update(folder2); + + // Create a backup. + await PlacesBackups.create(); + + // Remove the bookmarks, then restore the backup. + await PlacesUtils.bookmarks.remove(folder1); + await BookmarkJSONUtils.importFromFile( + await PlacesBackups.getMostRecentBackup(), + { replace: true } + ); + + info("Checking first level"); + let root = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.unfiledGuid + ).root; + let level1 = root.getChild(0); + Assert.equal(level1.title, "f1"); + info("Checking second level"); + PlacesUtils.asContainer(level1).containerOpen = true; + let level2 = level1.getChild(0); + Assert.equal(level2.title, "f2"); + info("Checking bookmark"); + PlacesUtils.asContainer(level2).containerOpen = true; + bookmark = level2.getChild(0); + Assert.equal(bookmark.title, "bookmark"); + level2.containerOpen = false; + level1.containerOpen = false; + root.containerOpen = false; +}); diff --git a/toolkit/components/places/tests/bookmarks/test_997030-bookmarks-html-encode.js b/toolkit/components/places/tests/bookmarks/test_997030-bookmarks-html-encode.js new file mode 100644 index 0000000000..6f3132b275 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_997030-bookmarks-html-encode.js @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Checks that we don't encodeURI twice when creating bookmarks.html. + */ +add_task(async function () { + let url = + "http://bt.ktxp.com/search.php?keyword=%E5%A6%84%E6%83%B3%E5%AD%A6%E7%94%9F%E4%BC%9A"; + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "bookmark", + url, + }); + + let file = PathUtils.join( + PathUtils.profileDir, + "bookmarks.exported.997030.html" + ); + await IOUtils.remove(file, { ignoreAbsent: true }); + await BookmarkHTMLUtils.exportToFile(file); + + // Remove the bookmarks, then restore the backup. + await PlacesUtils.bookmarks.remove(bm); + await BookmarkHTMLUtils.importFromFile(file, { replace: true }); + + info("Checking first level"); + let root = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.unfiledGuid + ).root; + let node = root.getChild(0); + Assert.equal(node.uri, url); + + root.containerOpen = false; + await PlacesUtils.bookmarks.eraseEverything(); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_async_observers.js b/toolkit/components/places/tests/bookmarks/test_async_observers.js new file mode 100644 index 0000000000..504f489339 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_async_observers.js @@ -0,0 +1,71 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* This test checks that bookmarks service is correctly forwarding async + * events like visit or favicon additions. */ + +let gBookmarkGuids = []; + +add_task(async function setup() { + // Add multiple bookmarks to the same uri. + gBookmarkGuids.push( + ( + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://book.ma.rk/", + }) + ).guid + ); + gBookmarkGuids.push( + ( + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://book.ma.rk/", + }) + ).guid + ); + Assert.equal(gBookmarkGuids.length, 2); +}); + +add_task(async function test_add_icon() { + // Add a visit to the bookmark and wait for the observer. + let guids = new Set(gBookmarkGuids); + Assert.equal(guids.size, 2); + let promiseNotifications = PlacesTestUtils.waitForNotification( + "favicon-changed", + events => + events.some( + event => + event.url == "http://book.ma.rk/" && + event.faviconUrl.startsWith("data:image/png;base64") + ) + ); + + PlacesUtils.favicons.setAndFetchFaviconForPage( + NetUtil.newURI("http://book.ma.rk/"), + SMALLPNG_DATA_URI, + true, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + await promiseNotifications; +}); + +add_task(async function test_remove_page() { + // Add a visit to the bookmark and wait for the observer. + let guids = new Set(gBookmarkGuids); + Assert.equal(guids.size, 2); + let promiseNotifications = PlacesTestUtils.waitForNotification( + "page-removed", + events => + events.some( + event => + event.url === "http://book.ma.rk/" && + !event.isRemovedFromStore && + !event.isPartialVisistsRemoval + ) + ); + await PlacesUtils.history.remove("http://book.ma.rk/"); + await promiseNotifications; +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bmindex.js b/toolkit/components/places/tests/bookmarks/test_bmindex.js new file mode 100644 index 0000000000..c79da88282 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bmindex.js @@ -0,0 +1,133 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const NUM_BOOKMARKS = 20; +const NUM_SEPARATORS = 5; +const NUM_FOLDERS = 10; +const NUM_ITEMS = NUM_BOOKMARKS + NUM_SEPARATORS + NUM_FOLDERS; +const MIN_RAND = -5; +const MAX_RAND = 40; + +async function check_contiguous_indexes(bookmarks) { + var indexes = []; + for (let bm of bookmarks) { + let bmIndex = (await PlacesUtils.bookmarks.fetch(bm.guid)).index; + info(`Index: ${bmIndex}\n`); + info("Checking duplicates\n"); + Assert.ok(!indexes.includes(bmIndex)); + info(`Checking out of range, found ${bookmarks.length} items\n`); + Assert.ok(bmIndex >= 0 && bmIndex < bookmarks.length); + indexes.push(bmIndex); + } + info("Checking all valid indexes have been used\n"); + Assert.equal(indexes.length, bookmarks.length); +} + +add_task(async function test_bookmarks_indexing() { + let bookmarks = []; + // Insert bookmarks with random indexes. + for (let i = 0; bookmarks.length < NUM_BOOKMARKS; i++) { + let randIndex = Math.round( + MIN_RAND + Math.random() * (MAX_RAND - MIN_RAND) + ); + try { + let bm = await PlacesUtils.bookmarks.insert({ + index: randIndex, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: `Test bookmark ${i}`, + url: `http://${i}.mozilla.org/`, + }); + if (randIndex < -1) { + do_throw("Creating a bookmark at an invalid index should throw"); + } + bookmarks.push(bm); + } catch (ex) { + if (randIndex >= -1) { + do_throw("Creating a bookmark at a valid index should not throw"); + } + } + } + await check_contiguous_indexes(bookmarks); + + // Insert separators with random indexes. + for (let i = 0; bookmarks.length < NUM_BOOKMARKS + NUM_SEPARATORS; i++) { + let randIndex = Math.round( + MIN_RAND + Math.random() * (MAX_RAND - MIN_RAND) + ); + try { + let bm = await PlacesUtils.bookmarks.insert({ + index: randIndex, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + if (randIndex < -1) { + do_throw("Creating a separator at an invalid index should throw"); + } + bookmarks.push(bm); + } catch (ex) { + if (randIndex >= -1) { + do_throw("Creating a separator at a valid index should not throw"); + } + } + } + await check_contiguous_indexes(bookmarks); + + // Insert folders with random indexes. + for (let i = 0; bookmarks.length < NUM_ITEMS; i++) { + let randIndex = Math.round( + MIN_RAND + Math.random() * (MAX_RAND - MIN_RAND) + ); + try { + let bm = await PlacesUtils.bookmarks.insert({ + index: randIndex, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: `Test folder ${i}`, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + if (randIndex < -1) { + do_throw("Creating a folder at an invalid index should throw"); + } + bookmarks.push(bm); + } catch (ex) { + if (randIndex >= -1) { + do_throw("Creating a folder at a valid index should not throw"); + } + } + } + await check_contiguous_indexes(bookmarks); + + // Execute some random bookmark delete. + for (let i = 0; i < Math.ceil(NUM_ITEMS / 4); i++) { + let bm = bookmarks.splice( + Math.floor(Math.random() * bookmarks.length), + 1 + )[0]; + info(`Removing item with guid ${bm.guid}\n`); + await PlacesUtils.bookmarks.remove(bm); + } + await check_contiguous_indexes(bookmarks); + + // Execute some random bookmark move. This will also try to move it to + // invalid index values. + for (let i = 0; i < Math.ceil(NUM_ITEMS / 4); i++) { + let randIndex = Math.floor(Math.random() * bookmarks.length); + let bm = bookmarks[randIndex]; + let newIndex = Math.round(MIN_RAND + Math.random() * (MAX_RAND - MIN_RAND)); + info(`Moving item with guid ${bm.guid} to index ${newIndex}\n`); + try { + bm.index = newIndex; + await PlacesUtils.bookmarks.update(bm); + if (newIndex < -1) { + do_throw("Moving an item to a negative index should throw\n"); + } + } catch (ex) { + if (newIndex >= -1) { + do_throw("Moving an item to a valid index should not throw\n"); + } + } + } + await check_contiguous_indexes(bookmarks); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmark_observer.js b/toolkit/components/places/tests/bookmarks/test_bookmark_observer.js new file mode 100644 index 0000000000..9ad01cfab9 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmark_observer.js @@ -0,0 +1,937 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Tests that each bookmark event gets the correct input. + +var gUnfiledFolderId; + +var gBookmarksObserver = { + expected: [], + setup(expected) { + this.expected = expected; + this.deferred = PromiseUtils.defer(); + return this.deferred.promise; + }, + + validateEvents(events) { + Assert.greaterOrEqual(this.expected.length, events.length); + for (let event of events) { + let expected = this.expected.shift(); + Assert.equal(expected.eventType, event.type); + let args = expected.args; + for (let i = 0; i < args.length; i++) { + Assert.ok( + args[i].check(event[args[i].name]), + event.type + "(args[" + i + "]: " + args[i].name + ")" + ); + } + } + + if (this.expected.length === 0) { + this.deferred.resolve(); + } + }, + + handlePlacesEvents(events) { + this.validateEvents(events); + }, +}; + +var gBookmarkSkipObserver = { + expected: null, + setup(expected) { + this.expected = expected; + this.deferred = PromiseUtils.defer(); + return this.deferred.promise; + }, + + validateEvents(events) { + events = events.filter(e => !e.isTagging); + Assert.greaterOrEqual(this.expected.length, events.length); + for (let event of events) { + let expectedEventType = this.expected.shift(); + Assert.equal(expectedEventType, event.type); + } + + if (this.expected.length === 0) { + this.deferred.resolve(); + } + }, + + handlePlacesEvents(events) { + this.validateEvents(events); + }, +}; + +add_task(async function setup() { + gUnfiledFolderId = await PlacesUtils.promiseItemId( + PlacesUtils.bookmarks.unfiledGuid + ); + gBookmarksObserver.handlePlacesEvents = + gBookmarksObserver.handlePlacesEvents.bind(gBookmarksObserver); + gBookmarkSkipObserver.handlePlacesEvents = + gBookmarkSkipObserver.handlePlacesEvents.bind(gBookmarkSkipObserver); + PlacesUtils.observers.addListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-tags-changed", + "bookmark-title-changed", + ], + gBookmarksObserver.handlePlacesEvents + ); + PlacesUtils.observers.addListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-tags-changed", + "bookmark-title-changed", + ], + gBookmarkSkipObserver.handlePlacesEvents + ); +}); + +add_task(async function bookmarkItemAdded_bookmark() { + const title = "Bookmark 1"; + let uri = Services.io.newURI("http://1.mozilla.org/"); + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-added"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === gUnfiledFolderId }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v == uri.spec }, + { name: "title", check: v => v === title }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: uri, + title, + }); + await promise; +}); + +add_task(async function bookmarkItemAdded_separator() { + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-added"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === gUnfiledFolderId }, + { name: "index", check: v => v === 1 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v === "" }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + await promise; +}); + +add_task(async function bookmarkItemAdded_folder() { + const title = "Folder 1"; + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-added"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === gUnfiledFolderId }, + { name: "index", check: v => v === 2 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v === title }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + await promise; +}); + +add_task(async function bookmarkTitleChanged() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + const title = "New title"; + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-title-changed"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-title-changed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "title", check: v => v === title }, + { name: "lastModified", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.update({ guid: bm.guid, title }); + await promise; +}); + +add_task(async function bookmarkTagsChanged() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + let uri = bm.url.URI; + const TAG = "tag"; + let promise = Promise.all([ + gBookmarkSkipObserver.setup([ + "bookmark-tags-changed", + "bookmark-tags-changed", + ]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-added", // This is the tag folder. + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === PlacesUtils.tagsFolderId }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v === TAG }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-added", // This is the tag. + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v == uri.spec }, + { name: "title", check: v => v === "" }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-tags-changed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { name: "lastModified", check: v => typeof v == "number" && v > 0 }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + { + name: "isTagging", + check: v => v === false, + }, + ], + }, + { + eventType: "bookmark-removed", // This is the tag. + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v == uri.spec }, + { name: "title", check: v => v == "" }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-tags-changed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { name: "lastModified", check: v => typeof v == "number" && v > 0 }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + { + name: "isTagging", + check: v => v === false, + }, + ], + }, + { + eventType: "bookmark-removed", // This is the tag folder. + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === PlacesUtils.tagsFolderId }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v == TAG }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + PlacesUtils.tagging.tagURI(uri, [TAG]); + PlacesUtils.tagging.untagURI(uri, [TAG]); + await promise; +}); + +add_task(async function bookmarkItemMoved_bookmark() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-moved", "bookmark-moved"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-moved", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "oldIndex", check: v => v === 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "oldParentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + { name: "url", check: v => typeof v == "string" }, + ], + }, + { + eventType: "bookmark-moved", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "oldIndex", check: v => v === 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "oldParentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + { name: "url", check: v => typeof v == "string" }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + index: 0, + }); + await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + await promise; +}); + +add_task(async function bookmarkItemRemoved_bookmark() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + let uri = bm.url.URI; + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-removed"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === gUnfiledFolderId }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v === uri.spec }, + { name: "title", check: v => v == "New title" }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.remove(bm); + await promise; +}); + +add_task(async function bookmarkItemRemoved_separator() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-removed"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v == "" }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.remove(bm); + await promise; +}); + +add_task(async function bookmarkItemRemoved_folder() { + let bm = await PlacesUtils.bookmarks.fetch({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + let promise = Promise.all([ + gBookmarkSkipObserver.setup(["bookmark-removed"]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v == "Folder 1" }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + await PlacesUtils.bookmarks.remove(bm); + await promise; +}); + +add_task(async function bookmarkItemRemoved_folder_recursive() { + const title = "Folder 3"; + const BMTITLE = "Bookmark 1"; + let uri = Services.io.newURI("http://1.mozilla.org/"); + let promise = Promise.all([ + gBookmarkSkipObserver.setup([ + "bookmark-added", + "bookmark-added", + "bookmark-added", + "bookmark-added", + "bookmark-removed", + "bookmark-removed", + "bookmark-removed", + "bookmark-removed", + ]), + gBookmarksObserver.setup([ + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => v === gUnfiledFolderId }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v === title }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v == uri.spec }, + { name: "title", check: v => v === BMTITLE }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 1 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v === title }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-added", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v == uri.spec }, + { name: "title", check: v => v === BMTITLE }, + { name: "dateAdded", check: v => typeof v == "number" && v > 0 }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v === uri.spec }, + { name: "title", check: v => v == BMTITLE }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 1 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v == title }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + { name: "url", check: v => v === uri.spec }, + { name: "title", check: v => v == BMTITLE }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + { + eventType: "bookmark-removed", + args: [ + { name: "id", check: v => typeof v == "number" && v > 0 }, + { name: "parentId", check: v => typeof v == "number" && v > 0 }, + { name: "index", check: v => v === 0 }, + { + name: "itemType", + check: v => v === PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { name: "url", check: v => v === "" }, + { name: "title", check: v => v == title }, + { + name: "guid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "parentGuid", + check: v => typeof v == "string" && PlacesUtils.isValidGuid(v), + }, + { + name: "source", + check: v => + Object.values(PlacesUtils.bookmarks.SOURCES).includes(v), + }, + ], + }, + ]), + ]); + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: uri, + title: BMTITLE, + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + title, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: uri, + title: BMTITLE, + }); + + await PlacesUtils.bookmarks.remove(folder); + await promise; +}); + +add_task(function cleanup() { + PlacesUtils.observers.removeListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-tags-changed", + "bookmark-title-changed", + ], + gBookmarksObserver.handlePlacesEvents + ); + PlacesUtils.observers.removeListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-tags-changed", + "bookmark-title-changed", + ], + gBookmarkSkipObserver.handlePlacesEvents + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_eraseEverything.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_eraseEverything.js new file mode 100644 index 0000000000..55bcea5978 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_eraseEverything.js @@ -0,0 +1,207 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_task(async function test_eraseEverything() { + await PlacesTestUtils.addVisits({ + uri: NetUtil.newURI("http://example.com/"), + }); + await PlacesTestUtils.addVisits({ + uri: NetUtil.newURI("http://mozilla.org/"), + }); + let frecencyForExample = await PlacesTestUtils.getDatabaseValue( + "moz_places", + "frecency", + { url: "http://example.com/" } + ); + let frecencyForMozilla = await PlacesTestUtils.getDatabaseValue( + "moz_places", + "frecency", + { url: "http://mozilla.org/" } + ); + Assert.ok(frecencyForExample > 0); + Assert.ok(frecencyForMozilla > 0); + let unfiledFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(unfiledFolder); + let unfiledBookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + }); + checkBookmarkObject(unfiledBookmark); + let unfiledBookmarkInFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: unfiledFolder.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://mozilla.org/", + }); + checkBookmarkObject(unfiledBookmarkInFolder); + + let menuFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(menuFolder); + let menuBookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + }); + checkBookmarkObject(menuBookmark); + let menuBookmarkInFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: menuFolder.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://mozilla.org/", + }); + checkBookmarkObject(menuBookmarkInFolder); + + let toolbarFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(toolbarFolder); + let toolbarBookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + }); + checkBookmarkObject(toolbarBookmark); + let toolbarBookmarkInFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: toolbarFolder.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://mozilla.org/", + }); + checkBookmarkObject(toolbarBookmarkInFolder); + + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + Assert.ok( + (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://example.com/", + })) > frecencyForExample + ); + Assert.ok( + (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://example.com/", + })) > frecencyForMozilla + ); + + await PlacesUtils.bookmarks.eraseEverything(); + + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + Assert.equal( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://example.com/", + }), + frecencyForExample + ); + Assert.equal( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://example.com/", + }), + frecencyForMozilla + ); +}); + +add_task(async function test_eraseEverything_roots() { + await PlacesUtils.bookmarks.eraseEverything(); + + // Ensure the roots have not been removed. + Assert.ok( + await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.unfiledGuid) + ); + Assert.ok( + await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.toolbarGuid) + ); + Assert.ok(await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.menuGuid)); + Assert.ok(await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.tagsGuid)); + Assert.ok(await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.rootGuid)); +}); + +add_task(async function test_eraseEverything_reparented() { + // Create a folder with 1 bookmark in it... + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let bookmark1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://example.com/", + }); + // ...and a second folder. + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // Reparent the bookmark to the 2nd folder. + bookmark1.parentGuid = folder2.guid; + await PlacesUtils.bookmarks.update(bookmark1); + + // Erase everything. + await PlacesUtils.bookmarks.eraseEverything(); + + // All the above items should no longer be in the GUIDHelper cache. + for (let guid of [folder1.guid, bookmark1.guid, folder2.guid]) { + await Assert.rejects( + PlacesUtils.promiseItemId(guid), + /no item found for the given GUID/ + ); + } +}); + +add_task(async function test_notifications() { + let bms = await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + title: "test", + url: "http://example.com", + }, + { + title: "folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children: [ + { + title: "test2", + url: "http://example.com/2", + }, + ], + }, + ], + }); + + let skipDescendantsObserver = expectPlacesObserverNotifications( + ["bookmark-removed"], + false, + true + ); + let receiveAllObserver = expectPlacesObserverNotifications( + ["bookmark-removed"], + false + ); + + await PlacesUtils.bookmarks.eraseEverything(); + + let expectedNotifications = [ + { + type: "bookmark-removed", + guid: bms[1].guid, + }, + { + type: "bookmark-removed", + guid: bms[0].guid, + }, + ]; + + // If we're skipping descendents, we'll only be notified of the folder. + skipDescendantsObserver.check(expectedNotifications); + + // Note: Items of folders get notified first. + expectedNotifications.unshift({ + type: "bookmark-removed", + guid: bms[2].guid, + }); + + receiveAllObserver.check(expectedNotifications); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_fetch.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_fetch.js new file mode 100644 index 0000000000..d04d3ee5da --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_fetch.js @@ -0,0 +1,614 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +var gAccumulator = { + get callback() { + this.results = []; + return result => this.results.push(result); + }, +}; + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.fetch(), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch(null), + /Input should be a valid object/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guid: "123456789012", index: 0 }), + /The following properties were expected: parentGuid/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({}), + /Unexpected number of conditions provided: 0/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + }), + /Unexpected number of conditions provided: 0/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ + guid: "123456789012", + parentGuid: "012345678901", + index: 0, + }), + /Unexpected number of conditions provided: 2/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ + guid: "123456789012", + url: "http://example.com", + }), + /Unexpected number of conditions provided: 2/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch("test"), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch(123), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guid: "test" }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guid: null }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guid: 123 }), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guidPrefix: "" }), + /Invalid value for property 'guidPrefix'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guidPrefix: null }), + /Invalid value for property 'guidPrefix'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guidPrefix: 123 }), + /Invalid value for property 'guidPrefix'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guidPrefix: "123456789012" }), + /Invalid value for property 'guidPrefix'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ guidPrefix: "@" }), + /Invalid value for property 'guidPrefix'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ parentGuid: "test", index: 0 }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ parentGuid: null, index: 0 }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ parentGuid: 123, index: 0 }), + /Invalid value for property 'parentGuid'/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ parentGuid: "123456789012", index: "0" }), + /Invalid value for property 'index'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ parentGuid: "123456789012", index: null }), + /Invalid value for property 'index'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.fetch({ parentGuid: "123456789012", index: -10 }), + /Invalid value for property 'index'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ url: "http://te st/" }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ url: null }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ url: -10 }), + /Invalid value for property 'url'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.fetch("123456789012", "test"), + /onResult callback must be a valid function/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch("123456789012", {}), + /onResult callback must be a valid function/ + ); +}); + +add_task(async function fetch_nonexistent_guid() { + let bm = await PlacesUtils.bookmarks.fetch( + { guid: "123456789012" }, + gAccumulator.callback + ); + Assert.equal(bm, null); + Assert.equal(gAccumulator.results.length, 0); +}); + +add_task(async function fetch_bookmark() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch(bm1.guid, gAccumulator.callback); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + checkBookmarkObject(gAccumulator.results[0]); + Assert.deepEqual(gAccumulator.results[0], bm1); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.index, 0); + Assert.deepEqual(bm2.dateAdded, bm2.lastModified); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm2.url.href, "http://example.com/"); + Assert.equal(bm2.title, "a bookmark"); + Assert.strictEqual(bm2.childCount, undefined); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_bookmar_empty_title() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch(bm1.guid); + checkBookmarkObject(bm2); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.index, 0); + Assert.strictEqual(bm2.title, ""); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_folder() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + checkBookmarkObject(bm1); + Assert.deepEqual(bm1.dateAdded, bm1.lastModified); + + // Inserting a child updates both the childCount and lastModified of bm1, + // though the bm1 object is static once fetched, thus later we'll manually + // update it. + await PlacesUtils.bookmarks.insert({ + parentGuid: bm1.guid, + url: "https://www.mozilla.org/", + title: "", + }); + + let bm2 = await PlacesUtils.bookmarks.fetch(bm1.guid); + checkBookmarkObject(bm2); + + Assert.equal(bm2.childCount, 1); + bm1.childCount = bm2.childCount; + bm1.lastModified = bm2.lastModified; + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.index, 0); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.equal(bm2.title, "a folder"); + Assert.ok(!("url" in bm2)); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_folder_empty_title() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch(bm1.guid); + checkBookmarkObject(bm2); + + Assert.equal(bm2.childCount, 0); + // Insert doesn't populate childCount (it would always be 0 anyway), so set + // it to be able to just use deepEqual. + bm1.childCount = bm2.childCount; + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.index, 0); + Assert.strictEqual(bm2.title, ""); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_separator() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch(bm1.guid); + checkBookmarkObject(bm2); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.index, 0); + Assert.deepEqual(bm2.dateAdded, bm2.lastModified); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_SEPARATOR); + Assert.ok(!("url" in bm2)); + Assert.strictEqual(bm2.title, ""); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_byguid_prefix() { + const PREFIX = "PREFIX-"; + + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + guid: PlacesUtils.generateGuidWithPrefix(PREFIX), + url: "http://bm1.example.com/", + title: "bookmark 1", + }); + checkBookmarkObject(bm1); + Assert.ok(bm1.guid.startsWith(PREFIX)); + + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + guid: PlacesUtils.generateGuidWithPrefix(PREFIX), + url: "http://bm2.example.com/", + title: "bookmark 2", + }); + checkBookmarkObject(bm2); + Assert.ok(bm2.guid.startsWith(PREFIX)); + + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + guid: PlacesUtils.generateGuidWithPrefix(PREFIX), + title: "a folder", + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: bm3.guid, + url: "https://www.mozilla.org/", + title: "", + }); + checkBookmarkObject(bm3); + Assert.ok(bm3.guid.startsWith(PREFIX)); + + // Bookmark 4 doesn't have the same guid prefix, so it shouldn't be returned in the results. + let bm4 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://bm3.example.com/", + title: "bookmark 4", + }); + checkBookmarkObject(bm4); + Assert.ok(!bm4.guid.startsWith(PREFIX)); + + await PlacesUtils.bookmarks.fetch( + { guidPrefix: PREFIX }, + gAccumulator.callback + ); + + Assert.equal(gAccumulator.results.length, 3); + + // The results are returned by most recent first, so the first bookmark + // inserted is the last one in the returned array. + Assert.deepEqual(bm1, gAccumulator.results[2]); + Assert.deepEqual(bm2, gAccumulator.results[1]); + Assert.equal(gAccumulator.results[0].childCount, 1); + bm3.childCount = gAccumulator.results[0].childCount; + bm3.lastModified = gAccumulator.results[0].lastModified; + Assert.deepEqual(bm3, gAccumulator.results[0]); + + Assert.equal(bm3.childCount, 1); + + await PlacesUtils.bookmarks.remove(bm1); + await PlacesUtils.bookmarks.remove(bm2); + await PlacesUtils.bookmarks.remove(bm3); + await PlacesUtils.bookmarks.remove(bm4); +}); + +add_task(async function fetch_byposition_nonexisting_parentGuid() { + let bm = await PlacesUtils.bookmarks.fetch( + { parentGuid: "123456789012", index: 0 }, + gAccumulator.callback + ); + Assert.equal(bm, null); + Assert.equal(gAccumulator.results.length, 0); +}); + +add_task(async function fetch_byposition_nonexisting_index() { + let bm = await PlacesUtils.bookmarks.fetch( + { parentGuid: PlacesUtils.bookmarks.unfiledGuid, index: 100 }, + gAccumulator.callback + ); + Assert.equal(bm, null); + Assert.equal(gAccumulator.results.length, 0); +}); + +add_task(async function fetch_byposition() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch( + { parentGuid: bm1.parentGuid, index: bm1.index }, + gAccumulator.callback + ); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + checkBookmarkObject(gAccumulator.results[0]); + Assert.deepEqual(gAccumulator.results[0], bm1); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.index, 0); + Assert.deepEqual(bm2.dateAdded, bm2.lastModified); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm2.url.href, "http://example.com/"); + Assert.equal(bm2.title, "a bookmark"); +}); + +add_task(async function fetch_byposition_default_index() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/last", + title: "last child", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch( + { parentGuid: bm1.parentGuid, index: PlacesUtils.bookmarks.DEFAULT_INDEX }, + gAccumulator.callback + ); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + checkBookmarkObject(gAccumulator.results[0]); + Assert.deepEqual(gAccumulator.results[0], bm1); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.index, 1); + Assert.deepEqual(bm2.dateAdded, bm2.lastModified); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm2.url.href, "http://example.com/last"); + Assert.equal(bm2.title, "last child"); + + await PlacesUtils.bookmarks.remove(bm1.guid); +}); + +add_task(async function fetch_byurl_nonexisting() { + let bm = await PlacesUtils.bookmarks.fetch( + { url: "http://nonexisting.com/" }, + gAccumulator.callback + ); + Assert.equal(bm, null); + Assert.equal(gAccumulator.results.length, 0); +}); + +add_task(async function fetch_byurl() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://byurl.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + + // Also ensure that fecth-by-url excludes the tags folder. + PlacesUtils.tagging.tagURI(bm1.url.URI, ["Test Tag"]); + + let bm2 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback + ); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + checkBookmarkObject(gAccumulator.results[0]); + Assert.deepEqual(gAccumulator.results[0], bm1); + + Assert.deepEqual(bm1, bm2); + Assert.equal(bm2.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.deepEqual(bm2.dateAdded, bm2.lastModified); + Assert.equal(bm2.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm2.url.href, "http://byurl.com/"); + Assert.equal(bm2.title, "a bookmark"); + + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://byurl.com/", + title: "a bookmark", + }); + let bm4 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback + ); + checkBookmarkObject(bm4); + Assert.deepEqual(bm3, bm4); + Assert.equal(gAccumulator.results.length, 2); + gAccumulator.results.forEach(checkBookmarkObject); + Assert.deepEqual(gAccumulator.results[0], bm4); + + // After an update the returned bookmark should change. + await PlacesUtils.bookmarks.update({ guid: bm1.guid, title: "new title" }); + let bm5 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback + ); + checkBookmarkObject(bm5); + // Cannot use deepEqual cause lastModified changed. + Assert.equal(bm1.guid, bm5.guid); + Assert.ok(bm5.lastModified > bm1.lastModified); + Assert.equal(gAccumulator.results.length, 2); + gAccumulator.results.forEach(checkBookmarkObject); + Assert.deepEqual(gAccumulator.results[0], bm5); + + // cleanup + PlacesUtils.tagging.untagURI(bm1.url.URI, ["Test Tag"]); +}); + +add_task(async function fetch_concurrent() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://concurrent.url.com/", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback, + { concurrent: true } + ); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + Assert.deepEqual(gAccumulator.results[0], bm1); + Assert.deepEqual(bm1, bm2); + let bm3 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback, + { concurrent: false } + ); + checkBookmarkObject(bm3); + Assert.equal(gAccumulator.results.length, 1); + Assert.deepEqual(gAccumulator.results[0], bm1); + Assert.deepEqual(bm1, bm3); + let bm4 = await PlacesUtils.bookmarks.fetch( + { url: bm1.url }, + gAccumulator.callback, + {} + ); + checkBookmarkObject(bm4); + Assert.equal(gAccumulator.results.length, 1); + Assert.deepEqual(gAccumulator.results[0], bm1); + Assert.deepEqual(bm1, bm4); +}); + +add_task(async function fetch_by_parent() { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + checkBookmarkObject(folder1); + + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://bm1.example.com/", + title: "bookmark 1", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://bm2.example.com/", + title: "bookmark 2", + }); + checkBookmarkObject(bm2); + + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "sub folder", + index: 0, + }); + checkBookmarkObject(bm2); + await PlacesUtils.bookmarks.insert({ + parentGuid: bm3.guid, + url: "http://mozilla.org/", + title: "sub bookmark", + }); + + await PlacesUtils.bookmarks.fetch( + { parentGuid: folder1.guid }, + gAccumulator.callback + ); + + Assert.equal(gAccumulator.results.length, 3); + + Assert.equal(gAccumulator.results[0].childCount, 1); + bm3.childCount = gAccumulator.results[0].childCount; + bm3.lastModified = gAccumulator.results[0].lastModified; + Assert.deepEqual(bm3, gAccumulator.results[0]); + Assert.equal(bm1.url.href, gAccumulator.results[1].url.href); + Assert.equal(bm2.url.href, gAccumulator.results[2].url.href); + + await PlacesUtils.bookmarks.remove(folder1); +}); + +add_task(async function fetch_with_bookmark_path() { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Parent", + }); + checkBookmarkObject(folder1); + + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://bookmarkpath.example.com/", + title: "Child Bookmark", + }); + checkBookmarkObject(bm1); + + let bm2 = await PlacesUtils.bookmarks.fetch( + { guid: bm1.guid }, + gAccumulator.callback, + { includePath: true } + ); + checkBookmarkObject(bm2); + Assert.equal(gAccumulator.results.length, 1); + Assert.equal(bm2.path.length, 2); + Assert.equal(bm2.path[0].guid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm2.path[0].title, "unfiled"); + Assert.equal(bm2.path[1].guid, folder1.guid); + Assert.equal(bm2.path[1].title, folder1.title); + + await PlacesUtils.bookmarks.remove(folder1); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_getRecent.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_getRecent.js new file mode 100644 index 0000000000..55cddb8820 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_getRecent.js @@ -0,0 +1,117 @@ +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.getRecent(), + /numberOfItems argument is required/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.getRecent("abc"), + /numberOfItems argument must be an integer/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.getRecent(1.2), + /numberOfItems argument must be an integer/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.getRecent(0), + /numberOfItems argument must be greater than zero/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.getRecent(-1), + /numberOfItems argument must be greater than zero/ + ); +}); + +add_task(async function getRecent_returns_recent_bookmarks() { + await PlacesUtils.bookmarks.eraseEverything(); + + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/path", + title: "another bookmark", + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.net/", + title: "another bookmark", + }); + let bm4 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.net/path", + title: "yet another bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + checkBookmarkObject(bm3); + checkBookmarkObject(bm4); + + // Add a tag to the most recent url to prove it doesn't get returned. + PlacesUtils.tagging.tagURI(uri(bm4.url), ["Test Tag"]); + + // Add a separator. + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + + // Add a query bookmark. + let queryURL = `place:parent=${PlacesUtils.bookmarks.menuGuid}&queryType=1`; + let bm5 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: queryURL, + title: "a test query", + }); + checkBookmarkObject(bm5); + + // Verify that getRecent only returns actual bookmarks. + let results = await PlacesUtils.bookmarks.getRecent(100); + Assert.equal( + results.length, + 4, + "The expected number of bookmarks was returned." + ); + checkBookmarkObject(results[0]); + Assert.deepEqual( + bm4, + results[0], + "The first result is the expected bookmark." + ); + checkBookmarkObject(results[1]); + Assert.deepEqual( + bm3, + results[1], + "The second result is the expected bookmark." + ); + checkBookmarkObject(results[2]); + Assert.deepEqual( + bm2, + results[2], + "The third result is the expected bookmark." + ); + checkBookmarkObject(results[3]); + Assert.deepEqual( + bm1, + results[3], + "The fourth result is the expected bookmark." + ); + + // Verify that getRecent utilizes the numberOfItems argument. + results = await PlacesUtils.bookmarks.getRecent(1); + Assert.equal( + results.length, + 1, + "The expected number of bookmarks was returned." + ); + checkBookmarkObject(results[0]); + Assert.deepEqual( + bm4, + results[0], + "The first result is the expected bookmark." + ); + + await PlacesUtils.bookmarks.eraseEverything(); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_insert.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_insert.js new file mode 100644 index 0000000000..df99a5fe3d --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_insert.js @@ -0,0 +1,432 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.insert(), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert(null), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({}), + /The following properties were expected/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ guid: "test" }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ guid: null }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ guid: 123 }), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ parentGuid: "test" }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ parentGuid: null }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ parentGuid: 123 }), + /Invalid value for property 'parentGuid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ index: "1" }), + /Invalid value for property 'index'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ index: -10 }), + /Invalid value for property 'index'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ dateAdded: -10 }), + /Invalid value for property 'dateAdded'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ dateAdded: "today" }), + /Invalid value for property 'dateAdded'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ dateAdded: Date.now() }), + /Invalid value for property 'dateAdded'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ dateAdded: new Date(NaN) }), + /Invalid value for property 'dateAdded'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ lastModified: -10 }), + /Invalid value for property 'lastModified'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ lastModified: "today" }), + /Invalid value for property 'lastModified'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ lastModified: Date.now() }), + /Invalid value for property 'lastModified'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ lastModified: new Date(NaN) }), + /Invalid value for property 'lastModified'/ + ); + + let past = new Date(Date.now() - 86400000); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ lastModified: past }), + /Invalid value for property 'lastModified'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.insert({ type: -1 }), + /Invalid value for property 'type'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ type: 100 }), + /Invalid value for property 'type'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.insert({ type: "bookmark" }), + /Invalid value for property 'type'/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + title: -1, + }), + /Invalid value for property 'title'/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: 10, + }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://te st", + }), + /Invalid value for property 'url'/ + ); + let longurl = "http://www.example.com/"; + for (let i = 0; i < 65536; i++) { + longurl += "a"; + } + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: longurl, + }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: NetUtil.newURI(longurl), + }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "te st", + }), + /Invalid value for property 'url'/ + ); +}); + +add_task(async function invalid_properties_for_bookmark_type() { + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + url: "http://www.moz.com/", + }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + url: "http://www.moz.com/", + }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + title: "test", + }), + /Invalid value for property 'title'/ + ); +}); + +add_task(async function test_insert_into_root_throws() { + const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" + ); + const sandbox = sinon.createSandbox(); + sandbox.stub(PlacesUtils, "isInAutomation").get(() => false); + registerCleanupFunction(() => sandbox.restore()); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + url: "http://example.com", + }), + /Invalid value for property 'parentGuid'/, + "Should throw when inserting a bookmark into the root." + ); + Assert.throws( + () => + PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }), + /Invalid value for property 'parentGuid'/, + "Should throw when inserting a folder into the root." + ); + sandbox.restore(); +}); + +add_task(async function long_title_trim() { + let longtitle = "a"; + for (let i = 0; i < 4096; i++) { + longtitle += "a"; + } + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: longtitle, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 0); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.equal(bm.title.length, 4096, "title should have been trimmed"); + Assert.ok(!("url" in bm), "url should not be set"); +}); + +add_task(async function create_separator() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 1); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_SEPARATOR); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function create_separator_w_title_fail() { + try { + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + title: "a separator", + }); + Assert.ok(false, "Trying to set title for a separator should reject"); + } catch (ex) {} +}); + +add_task(async function create_separator_invalid_parent_fail() { + try { + await PlacesUtils.bookmarks.insert({ + parentGuid: "123456789012", + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + title: "a separator", + }); + Assert.ok( + false, + "Trying to create an item in a non existing parent reject" + ); + } catch (ex) {} +}); + +add_task(async function create_separator_given_guid() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + guid: "123456789012", + }); + checkBookmarkObject(bm); + Assert.equal(bm.guid, "123456789012"); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 2); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_SEPARATOR); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function create_item_given_guid_no_type_fail() { + try { + await PlacesUtils.bookmarks.insert({ parentGuid: "123456789012" }); + Assert.ok( + false, + "Trying to create an item with a given guid but no type should reject" + ); + } catch (ex) {} +}); + +add_task(async function create_separator_big_index() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + index: 9999, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 3); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_SEPARATOR); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function create_separator_given_dateAdded() { + let time = new Date(); + let past = new Date(time - 86400000); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + dateAdded: past, + }); + checkBookmarkObject(bm); + Assert.equal(bm.dateAdded, past); + Assert.equal(bm.lastModified, past); +}); + +add_task(async function create_folder() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.strictEqual(bm.title, ""); + + // And then create a nested folder. + let parentGuid = bm.guid; + bm = await PlacesUtils.bookmarks.insert({ + parentGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, parentGuid); + Assert.equal(bm.index, 0); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.strictEqual(bm.title, "a folder"); +}); + +add_task(async function create_bookmark() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let parentGuid = bm.guid; + + bm = await PlacesUtils.bookmarks.insert({ + parentGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, parentGuid); + Assert.equal(bm.index, 0); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm.url.href, "http://example.com/"); + Assert.equal(bm.title, "a bookmark"); + + // Check parent lastModified. + let parent = await PlacesUtils.bookmarks.fetch({ guid: bm.parentGuid }); + Assert.deepEqual(parent.lastModified, bm.dateAdded); + + bm = await PlacesUtils.bookmarks.insert({ + parentGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: new URL("http://example.com/"), + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, parentGuid); + Assert.equal(bm.index, 1); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm.url.href, "http://example.com/"); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function create_bookmark_frecency() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm); + + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + Assert.greater( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: bm.url, + }), + 0, + "Check frecency has been updated" + ); +}); + +add_task(async function create_bookmark_without_type() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm.url.href, "http://example.com/"); + Assert.equal(bm.title, "a bookmark"); +}); + +add_task(async function test_url_with_apices() { + // Apices may confuse code and cause injection if mishandled. + const url = `javascript:alert("%s");alert('%s');`; + await PlacesUtils.bookmarks.insert({ + url, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + // Just a sanity check, this should not throw. + await PlacesUtils.history.remove(url); + let bm = await PlacesUtils.bookmarks.fetch({ url }); + await PlacesUtils.bookmarks.remove(bm); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_insertTree.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_insertTree.js new file mode 100644 index 0000000000..0fad7d4b31 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_insertTree.js @@ -0,0 +1,590 @@ +add_task(async function invalid_input_rejects() { + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(), + /Should be provided a valid tree object./ + ); + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(null), + /Should be provided a valid tree object./ + ); + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree("foo"), + /Should be provided a valid tree object./ + ); + + // All subsequent tests pass a valid parent guid. + let guid = PlacesUtils.bookmarks.unfiledGuid; + + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree({ guid, children: [] }), + /Should have a non-zero number of children to insert./ + ); + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree({ guid }), + /Should have a non-zero number of children to insert./ + ); + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree({ children: [{}], guid }), + /The following properties were expected: url/ + ); + + // Reuse another variable to make this easier to read: + let tree = { guid, children: [{ guid: "test" }] }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'guid'/ + ); + tree.children = [{ guid: null }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'guid'/ + ); + tree.children = [{ guid: 123 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'guid'/ + ); + + tree.children = [{ dateAdded: -10 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'dateAdded'/ + ); + tree.children = [{ dateAdded: "today" }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'dateAdded'/ + ); + tree.children = [{ dateAdded: Date.now() }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'dateAdded'/ + ); + tree.children = [{ dateAdded: new Date(NaN) }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'dateAdded'/ + ); + + tree.children = [{ lastModified: -10 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'lastModified'/ + ); + tree.children = [{ lastModified: "today" }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'lastModified'/ + ); + tree.children = [{ lastModified: Date.now() }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'lastModified'/ + ); + + let time = Date.now(); + let future = new Date(time + 86400000); + tree.children = [{ dateAdded: future, lastModified: time }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'dateAdded'/ + ); + let past = new Date(time - 86400000); + tree.children = [{ lastModified: past }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'lastModified'/ + ); + + tree.children = [{ type: -1 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'type'/ + ); + tree.children = [{ type: 100 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'type'/ + ); + tree.children = [{ type: "bookmark" }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'type'/ + ); + + tree.children = [{ type: PlacesUtils.bookmarks.TYPE_BOOKMARK, title: -1 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'title'/ + ); + + tree.children = [{ type: PlacesUtils.bookmarks.TYPE_BOOKMARK, url: 10 }]; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(tree), + /Invalid value for property 'url'/ + ); + + let treeWithBrokenURL = { + children: [ + { type: PlacesUtils.bookmarks.TYPE_BOOKMARK, url: "http://te st" }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(treeWithBrokenURL), + /Invalid value for property 'url'/ + ); + let longurl = "http://www.example.com/" + "a".repeat(65536); + let treeWithLongURL = { + children: [{ type: PlacesUtils.bookmarks.TYPE_BOOKMARK, url: longurl }], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(treeWithLongURL), + /Invalid value for property 'url'/ + ); + let treeWithLongURI = { + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: NetUtil.newURI(longurl), + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(treeWithLongURI), + /Invalid value for property 'url'/ + ); + let treeWithOtherBrokenURL = { + children: [{ type: PlacesUtils.bookmarks.TYPE_BOOKMARK, url: "te st" }], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(treeWithOtherBrokenURL), + /Invalid value for property 'url'/ + ); +}); + +add_task(async function invalid_properties_for_bookmark_type() { + let folderWithURL = { + children: [ + { type: PlacesUtils.bookmarks.TYPE_FOLDER, url: "http://www.moz.com/" }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(folderWithURL), + /Invalid value for property 'url'/ + ); + let separatorWithURL = { + children: [ + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + url: "http://www.moz.com/", + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(separatorWithURL), + /Invalid value for property 'url'/ + ); + let separatorWithTitle = { + children: [{ type: PlacesUtils.bookmarks.TYPE_SEPARATOR, title: "test" }], + guid: PlacesUtils.bookmarks.unfiledGuid, + }; + await Assert.throws( + () => PlacesUtils.bookmarks.insertTree(separatorWithTitle), + /Invalid value for property 'title'/ + ); +}); + +add_task(async function create_separator() { + let [bm] = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 0); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_SEPARATOR); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function create_plain_bm() { + let [bm] = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + url: "http://www.example.com/", + title: "Test", + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 1); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(bm.title, "Test"); + Assert.equal(bm.url.href, "http://www.example.com/"); +}); + +add_task(async function create_folder() { + let [bm] = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Test", + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + Assert.equal(bm.index, 2); + Assert.equal(bm.dateAdded, bm.lastModified); + Assert.equal(bm.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.equal(bm.title, "Test"); +}); + +add_task(async function create_in_tags() { + await Assert.throws( + () => + PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://www.example.com/", + title: "Test adding a tag", + }, + ], + guid: PlacesUtils.bookmarks.tagsGuid, + }), + /Can't use insertTree to insert tags/ + ); + let guidForTag = ( + await PlacesUtils.bookmarks.insert({ + title: "test-tag", + url: "http://www.unused.com/", + parentGuid: PlacesUtils.bookmarks.tagsGuid, + }) + ).guid; + await Assert.rejects( + PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://www.example.com/", + title: "Test adding an item to a tag", + }, + ], + guid: guidForTag, + }), + /Can't use insertTree to insert tags/ + ); + await PlacesUtils.bookmarks.remove(guidForTag); + await PlacesTestUtils.promiseAsyncUpdates(); +}); + +add_task(async function insert_into_root() { + await Assert.throws( + () => + PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://www.example.com/", + title: "Test inserting into root", + }, + ], + guid: PlacesUtils.bookmarks.rootGuid, + }), + /Can't insert into the root/ + ); +}); + +add_task(async function tree_where_separator_or_folder_has_kids() { + await Assert.throws( + () => + PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://www.example.com/", + title: "Test inserting into separator", + }, + ], + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }), + /Invalid value for property 'children'/ + ); + + await Assert.throws( + () => + PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + children: [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://www.example.com/", + title: "Test inserting into separator", + }, + ], + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }), + /Invalid value for property 'children'/ + ); +}); + +add_task(async function create_hierarchy() { + let obsInvoked = 0; + let listener = events => { + for (let event of events) { + obsInvoked++; + Assert.greater(event.id, 0, "Should have a valid itemId"); + } + }; + PlacesUtils.observers.addListener(["bookmark-added"], listener); + let bms = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Root item", + children: [ + { + url: "http://www.example.com/1", + title: "BM 1", + }, + { + url: "http://www.example.com/2", + title: "BM 2", + }, + { + title: "Sub", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children: [ + { + title: "Sub BM 1", + url: "http://www.example.com/sub/1", + }, + { + title: "Sub BM 2", + url: "http://www.example.com/sub/2", + }, + ], + }, + ], + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + PlacesUtils.observers.removeListener(["bookmark-added"], listener); + let parentFolder = null, + subFolder = null; + let prevBM = null; + for (let bm of bms) { + checkBookmarkObject(bm); + if (prevBM && prevBM.parentGuid == bm.parentGuid) { + Assert.equal(prevBM.index + 1, bm.index, "Indices should be subsequent"); + Assert.equal( + (await PlacesUtils.bookmarks.fetch(bm.guid)).index, + bm.index, + "Index reflects inserted index" + ); + } + prevBM = bm; + if (bm.type == PlacesUtils.bookmarks.TYPE_BOOKMARK) { + Assert.greater( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: bm.url, + }), + 0, + "Check frecency has been updated for bookmark " + bm.url + ); + } + if (bm.title == "Root item") { + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + parentFolder = bm; + } else if (!bm.title.startsWith("Sub BM")) { + Assert.equal(bm.parentGuid, parentFolder.guid); + if (bm.type == PlacesUtils.bookmarks.TYPE_FOLDER) { + subFolder = bm; + } + } else { + Assert.equal(bm.parentGuid, subFolder.guid); + } + } + Assert.equal(obsInvoked, bms.length); + Assert.equal(obsInvoked, 6); +}); + +add_task(async function insert_many_non_nested() { + let obsInvoked = 0; + let listener = events => { + for (let event of events) { + obsInvoked++; + Assert.greater(event.id, 0, "Should have a valid itemId"); + } + }; + PlacesUtils.observers.addListener(["bookmark-added"], listener); + let bms = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + url: "http://www.example.com/1", + title: "Item 1", + }, + { + url: "http://www.example.com/2", + title: "Item 2", + }, + { + url: "http://www.example.com/3", + title: "Item 3", + }, + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + { + title: "Item 4", + url: "http://www.example.com/4", + }, + { + title: "Item 5", + url: "http://www.example.com/5", + }, + ], + guid: PlacesUtils.bookmarks.unfiledGuid, + }); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + PlacesUtils.observers.removeListener(["bookmark-added"], listener); + let startIndex = -1; + for (let bm of bms) { + checkBookmarkObject(bm); + if (startIndex == -1) { + startIndex = bm.index; + } else { + Assert.equal(++startIndex, bm.index, "Indices should be subsequent"); + } + Assert.equal( + (await PlacesUtils.bookmarks.fetch(bm.guid)).index, + bm.index, + "Index reflects inserted index" + ); + if (bm.type == PlacesUtils.bookmarks.TYPE_BOOKMARK) { + Assert.greater( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: bm.url, + }), + 0, + "Check frecency has been updated for bookmark " + bm.url + ); + } + Assert.equal(bm.parentGuid, PlacesUtils.bookmarks.unfiledGuid); + } + Assert.equal(obsInvoked, bms.length); + Assert.equal(obsInvoked, 6); +}); + +add_task(async function create_in_folder() { + let mozFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Mozilla", + }); + + let notifications = []; + let listener = events => { + for (let event of events) { + notifications.push({ + itemId: event.id, + parentId: event.parentId, + index: event.index, + title: event.title, + guid: event.guid, + parentGuid: event.parentGuid, + }); + } + }; + PlacesUtils.observers.addListener(["bookmark-added"], listener); + + let bms = await PlacesUtils.bookmarks.insertTree({ + children: [ + { + url: "http://getfirefox.com", + title: "Get Firefox!", + }, + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Community", + children: [ + { + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }, + { + url: "https://www.seamonkey-project.org", + title: "SeaMonkey", + }, + ], + }, + ], + guid: mozFolder.guid, + }); + await PlacesTestUtils.promiseAsyncUpdates(); + + PlacesUtils.observers.removeListener(["bookmark-added"], listener); + + let mozFolderId = await PlacesUtils.promiseItemId(mozFolder.guid); + let commFolderId = await PlacesUtils.promiseItemId(bms[1].guid); + deepEqual(notifications, [ + { + itemId: await PlacesUtils.promiseItemId(bms[0].guid), + parentId: mozFolderId, + index: 0, + title: "Get Firefox!", + guid: bms[0].guid, + parentGuid: mozFolder.guid, + }, + { + itemId: commFolderId, + parentId: mozFolderId, + index: 1, + title: "Community", + guid: bms[1].guid, + parentGuid: mozFolder.guid, + }, + { + itemId: await PlacesUtils.promiseItemId(bms[2].guid), + parentId: commFolderId, + index: 0, + title: "Get Thunderbird!", + guid: bms[2].guid, + parentGuid: bms[1].guid, + }, + { + itemId: await PlacesUtils.promiseItemId(bms[3].guid), + parentId: commFolderId, + index: 1, + title: "SeaMonkey", + guid: bms[3].guid, + parentGuid: bms[1].guid, + }, + ]); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_moveToFolder.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_moveToFolder.js new file mode 100644 index 0000000000..7cc3eb0916 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_moveToFolder.js @@ -0,0 +1,723 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +function ensurePosition(info, parentGuid, index) { + print(`Checking ${info.guid}`); + checkBookmarkObject(info); + Assert.equal( + info.parentGuid, + parentGuid, + "Should be in the correct parent folder" + ); + Assert.equal(info.index, index, "Should have the correct index"); +} + +function insertChildren(folder, items) { + if (!items.length) { + return []; + } + + let children = []; + for (let i = 0; i < items.length; i++) { + if (items[i].type === TYPE_BOOKMARK) { + children.push({ + title: `${i}`, + url: "http://example.com", + }); + } else { + throw new Error(`Type ${items[i].type} is not supported.`); + } + } + return PlacesUtils.bookmarks.insertTree({ + guid: folder.guid, + children, + }); +} + +async function dumpFolderChildren( + folder, + details, + folderA, + folderB, + originalAChildren, + originalBChildren +) { + info(`${folder} Details:`); + info(`Input: ${JSON.stringify(details.initial[folder])}`); + info(`Expected: ${JSON.stringify(details.expected[folder])}`); + info("Index\tOriginal\tExpected\tResult"); + + let originalChildren; + let folderGuid; + if (folder == "folderA") { + originalChildren = originalAChildren; + folderGuid = folderA.guid; + } else { + originalChildren = originalBChildren; + folderGuid = folderB.guid; + } + + let tree = await PlacesUtils.promiseBookmarksTree(folderGuid); + let childrenCount = tree.children ? tree.children.length : 0; + for (let i = 0; i < originalChildren.length || i < childrenCount; i++) { + let originalGuid = + i < originalChildren.length ? originalChildren[i].guid : " "; + let resultGuid = i < childrenCount ? tree.children[i].guid : " "; + let expectedGuid = " "; + if (i < details.expected[folder].length) { + let expected = details.expected[folder][i]; + expectedGuid = + expected.folder == "a" + ? originalAChildren[expected.originalIndex].guid + : originalBChildren[expected.originalIndex].guid; + } + info(`${i}\t${originalGuid}\t${expectedGuid}\t${resultGuid}\n`); + } +} + +async function checkExpectedResults( + details, + folder, + folderGuid, + lastModified, + movedItems, + folderAChildren, + folderBChildren +) { + let expectedResults = details.expected[folder]; + for (let i = 0; i < expectedResults.length; i++) { + let expectedDetails = expectedResults[i]; + let originalItem = + expectedDetails.folder == "a" + ? folderAChildren[expectedDetails.originalIndex] + : folderBChildren[expectedDetails.originalIndex]; + + // Check the item got updated correctly in the database. + let updatedItem = await PlacesUtils.bookmarks.fetch(originalItem.guid); + + ensurePosition(updatedItem, folderGuid, i); + Assert.greaterOrEqual( + updatedItem.lastModified.getTime(), + lastModified.getTime(), + "Last modified should be later or equal to before" + ); + } + + if (details.expected.skipResultIndexChecks) { + return; + } + + // Check the items returned from the actual move() call are correct. + let index = 0; + for (let item of details.initial[folder]) { + if (!("targetFolder" in item)) { + // We weren't moving this item, so skip it and continue. + continue; + } + + let movedItem = movedItems[index]; + let updatedItem = await PlacesUtils.bookmarks.fetch(movedItem.guid); + + ensurePosition(movedItem, updatedItem.parentGuid, updatedItem.index); + + index++; + } +} + +async function checkLastModifiedForFolders(details, folder, movedItems) { + // For the tests, the moves always come from folderA. + if ( + details.initial.folderA.some( + item => "targetFolder" in item && item.targetFolder == folder.title + ) + ) { + let updatedFolder = await PlacesUtils.bookmarks.fetch(folder.guid); + + Assert.greaterOrEqual( + updatedFolder.lastModified.getTime(), + folder.lastModified.getTime(), + "Should have updated the folder's last modified time." + ); + print(JSON.stringify(movedItems[0])); + Assert.deepEqual( + updatedFolder.lastModified, + movedItems[0].lastModified, + "Should have the same last modified as the moved items." + ); + } +} + +async function testMoveToFolder(details) { + await PlacesUtils.bookmarks.eraseEverything(); + + // Always create two folders by default. + let [folderA, folderB] = await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + title: "a", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }, + { + title: "b", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }, + ], + }); + + checkBookmarkObject(folderA); + let folderAChildren = await insertChildren(folderA, details.initial.folderA); + checkBookmarkObject(folderB); + let folderBChildren = await insertChildren(folderB, details.initial.folderB); + + const originalAChildren = folderAChildren.map(child => { + return { ...child }; + }); + const originalBChildren = folderBChildren.map(child => { + return { ...child }; + }); + + let lastModified; + if (folderAChildren.length) { + lastModified = folderAChildren[0].lastModified; + } else if (folderBChildren.length) { + lastModified = folderBChildren[0].lastModified; + } else { + throw new Error("No children added, can't determine lastModified"); + } + + // Work out which children to move and to where. + let childrenToUpdate = []; + for (let i = 0; i < details.initial.folderA.length; i++) { + if ("move" in details.initial.folderA[i]) { + childrenToUpdate.push(folderAChildren[i].guid); + } + } + + let observer; + if (details.notifications) { + observer = expectPlacesObserverNotifications(["bookmark-moved"]); + } + + let movedItems = await PlacesUtils.bookmarks.moveToFolder( + childrenToUpdate, + details.targetFolder == "a" ? folderA.guid : folderB.guid, + details.targetIndex + ); + + await dumpFolderChildren( + "folderA", + details, + folderA, + folderB, + originalAChildren, + originalBChildren + ); + await dumpFolderChildren( + "folderB", + details, + folderA, + folderB, + originalAChildren, + originalBChildren + ); + + Assert.equal(movedItems.length, childrenToUpdate.length); + await checkExpectedResults( + details, + "folderA", + folderA.guid, + lastModified, + movedItems, + folderAChildren, + folderBChildren + ); + await checkExpectedResults( + details, + "folderB", + folderB.guid, + lastModified, + movedItems, + folderAChildren, + folderBChildren + ); + + if (details.notifications) { + let expectedNotifications = []; + + for (let notification of details.notifications) { + let origItem = + notification.originalFolder == "folderA" + ? originalAChildren[notification.originalIndex] + : originalBChildren[notification.originalIndex]; + let newFolder = notification.newFolder == "folderA" ? folderA : folderB; + + expectedNotifications.push({ + type: "bookmark-moved", + id: await PlacesUtils.promiseItemId(origItem.guid), + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: origItem.url, + guid: origItem.guid, + parentGuid: newFolder.guid, + source: PlacesUtils.bookmarks.SOURCES.DEFAULT, + index: notification.newIndex, + oldParentGuid: origItem.parentGuid, + oldIndex: notification.originalIndex, + isTagging: false, + }); + } + observer.check(expectedNotifications); + } + + await checkLastModifiedForFolders(details, folderA, movedItems); + await checkLastModifiedForFolders(details, folderB, movedItems); +} + +const TYPE_BOOKMARK = PlacesUtils.bookmarks.TYPE_BOOKMARK; + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder(), + /guids should be an array/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder({}), + /guids should be an array/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder([]), + /guids should be an array of at least one item/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder(["test"]), + /Expected only valid GUIDs to be passed/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder([null]), + /Expected only valid GUIDs to be passed/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder([123]), + /Expected only valid GUIDs to be passed/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.moveToFolder(["123456789012"], 123), + /Error: parentGuid should be a valid GUID/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.moveToFolder( + ["123456789012"], + PlacesUtils.bookmarks.rootGuid + ), + /Cannot move bookmarks into root/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.moveToFolder(["123456789012"], "123456789012", -2), + /index should be a number greater than/ + ); + Assert.throws( + () => + PlacesUtils.bookmarks.moveToFolder( + ["123456789012"], + "123456789012", + "sdffd" + ), + /index should be a number greater than/ + ); +}); + +add_task(async function test_move_nonexisting_bookmark_rejects() { + await Assert.rejects( + PlacesUtils.bookmarks.moveToFolder(["123456789012"], "123456789012", -1), + /No bookmarks found for the provided GUID/, + "Should reject when moving a non-existing bookmark" + ); +}); + +add_task(async function test_move_folder_into_descendant_rejects() { + let parent = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let descendant = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + await Assert.rejects( + PlacesUtils.bookmarks.moveToFolder([parent.guid], parent.guid, 0), + /Cannot insert a folder into itself or one of its descendants/, + "Should reject when moving a folder into itself" + ); + + await Assert.rejects( + PlacesUtils.bookmarks.moveToFolder([parent.guid], descendant.guid, 0), + /Cannot insert a folder into itself or one of its descendants/, + "Should reject when moving a folder into a descendant" + ); +}); + +add_task(async function test_move_from_differnt_with_no_target_rejects() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + await Assert.rejects( + PlacesUtils.bookmarks.moveToFolder([bm1.guid, bm2.guid], null, -1), + /All bookmarks should be in the same folder if no parent is specified/, + "Should reject when moving bookmarks from different folders with no target folder" + ); +}); + +add_task(async function test_move_append_same_folder() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + ], + folderB: [], + }, + targetFolder: "a", + targetIndex: -1, + expected: { + folderA: [ + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 2 }, + { folder: "a", originalIndex: 1 }, + ], + folderB: [], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 1, + newFolder: "folderA", + newIndex: 2, + }, + ], + }); +}); + +add_task(async function test_move_append_multiple_same_folder() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + ], + folderB: [], + }, + targetFolder: "a", + targetIndex: -1, + expected: { + folderA: [ + { folder: "a", originalIndex: 3 }, + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 1 }, + { folder: "a", originalIndex: 2 }, + ], + folderB: [], + }, + // These are all inserted at position 3 as that's what the views require + // to be notified, to ensure the new items are displayed in their correct + // positions. + notifications: [ + { + originalFolder: "folderA", + originalIndex: 0, + newFolder: "folderA", + newIndex: 3, + }, + { + originalFolder: "folderA", + originalIndex: 1, + newFolder: "folderA", + newIndex: 3, + }, + { + originalFolder: "folderA", + originalIndex: 2, + newFolder: "folderA", + newIndex: 3, + }, + ], + }); +}); + +add_task(async function test_move_append_multiple_new_folder() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + ], + folderB: [], + }, + targetFolder: "b", + targetIndex: -1, + expected: { + folderA: [], + folderB: [ + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 1 }, + { folder: "a", originalIndex: 2 }, + ], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 0, + newFolder: "folderB", + newIndex: 0, + }, + { + originalFolder: "folderA", + originalIndex: 1, + newFolder: "folderB", + newIndex: 1, + }, + { + originalFolder: "folderA", + originalIndex: 2, + newFolder: "folderB", + newIndex: 2, + }, + ], + }); +}); + +add_task(async function test_move_append_multiple_new_folder_with_existing() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK, move: true }, + ], + folderB: [{ type: TYPE_BOOKMARK }, { type: TYPE_BOOKMARK }], + }, + targetFolder: "b", + targetIndex: -1, + expected: { + folderA: [], + folderB: [ + { folder: "b", originalIndex: 0 }, + { folder: "b", originalIndex: 1 }, + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 1 }, + { folder: "a", originalIndex: 2 }, + ], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 0, + newFolder: "folderB", + newIndex: 2, + }, + { + originalFolder: "folderA", + originalIndex: 1, + newFolder: "folderB", + newIndex: 3, + }, + { + originalFolder: "folderA", + originalIndex: 2, + newFolder: "folderB", + newIndex: 4, + }, + ], + }); +}); + +add_task(async function test_move_insert_same_folder_up() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK, move: true }, + ], + folderB: [], + }, + targetFolder: "a", + targetIndex: 0, + expected: { + folderA: [ + { folder: "a", originalIndex: 2 }, + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 1 }, + ], + folderB: [], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 2, + newFolder: "folderA", + newIndex: 0, + }, + ], + }); +}); + +add_task(async function test_move_insert_same_folder_down() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK }, + ], + folderB: [], + }, + targetFolder: "a", + targetIndex: 2, + expected: { + folderA: [ + { folder: "a", originalIndex: 1 }, + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 2 }, + ], + folderB: [], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 0, + newFolder: "folderA", + newIndex: 1, + }, + ], + }); +}); + +add_task( + async function test_move_insert_multiple_same_folder_split_locations() { + await testMoveToFolder({ + initial: { + folderA: [ + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK, move: true }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK }, + { type: TYPE_BOOKMARK, move: true }, + ], + folderB: [], + }, + targetFolder: "a", + targetIndex: 2, + expected: { + folderA: [ + { folder: "a", originalIndex: 1 }, + { folder: "a", originalIndex: 0 }, + { folder: "a", originalIndex: 3 }, + { folder: "a", originalIndex: 6 }, + { folder: "a", originalIndex: 9 }, + { folder: "a", originalIndex: 2 }, + { folder: "a", originalIndex: 4 }, + { folder: "a", originalIndex: 5 }, + { folder: "a", originalIndex: 7 }, + { folder: "a", originalIndex: 8 }, + ], + folderB: [], + }, + notifications: [ + { + originalFolder: "folderA", + originalIndex: 0, + newFolder: "folderA", + newIndex: 1, + }, + { + originalFolder: "folderA", + originalIndex: 3, + newFolder: "folderA", + newIndex: 2, + }, + { + originalFolder: "folderA", + originalIndex: 6, + newFolder: "folderA", + newIndex: 3, + }, + { + originalFolder: "folderA", + originalIndex: 9, + newFolder: "folderA", + newIndex: 4, + }, + ], + }); + } +); + +add_task(async function test_move_folder_with_descendant() { + let parent = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + url: "http://example.com/", + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + }); + let descendant = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(descendant.index, 1); + let lastModified = bm.lastModified; + + // This is moving to a nonexisting index by purpose, it will be appended. + [bm] = await PlacesUtils.bookmarks.moveToFolder( + [bm.guid], + descendant.guid, + 1 + ); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, descendant.guid); + Assert.equal(bm.index, 0); + Assert.ok(bm.lastModified >= lastModified); + + parent = await PlacesUtils.bookmarks.fetch(parent.guid); + descendant = await PlacesUtils.bookmarks.fetch(descendant.guid); + Assert.deepEqual(parent.lastModified, bm.lastModified); + Assert.deepEqual(descendant.lastModified, bm.lastModified); + Assert.equal(descendant.index, 0); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal(bm.parentGuid, descendant.guid); + Assert.equal(bm.index, 0); + + [bm] = await PlacesUtils.bookmarks.moveToFolder([bm.guid], parent.guid, 0); + Assert.equal(bm.parentGuid, parent.guid); + Assert.equal(bm.index, 0); + + descendant = await PlacesUtils.bookmarks.fetch(descendant.guid); + Assert.equal(descendant.index, 1); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_notifications.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_notifications.js new file mode 100644 index 0000000000..08afb6001d --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_notifications.js @@ -0,0 +1,1135 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.defineESModuleGetters(this, { + Preferences: "resource://gre/modules/Preferences.sys.mjs", +}); + +add_task(async function insert_separator_notification() { + let observer = expectPlacesObserverNotifications(["bookmark-added"]); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + observer.check([ + { + type: "bookmark-added", + id: itemId, + itemType: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentId, + index: bm.index, + url: bm.url, + title: bm.title, + dateAdded: bm.dateAdded, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function insert_folder_notification() { + let observer = expectPlacesObserverNotifications(["bookmark-added"]); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "a folder", + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + observer.check([ + { + type: "bookmark-added", + id: itemId, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + parentId, + index: bm.index, + url: bm.url, + title: bm.title, + dateAdded: bm.dateAdded, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function insert_folder_notitle_notification() { + let observer = expectPlacesObserverNotifications(["bookmark-added"]); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + strictEqual(bm.title, "", "Should return empty string for untitled folder"); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + observer.check([ + { + type: "bookmark-added", + id: itemId, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + parentId, + index: bm.index, + url: bm.url, + title: bm.title, + dateAdded: bm.dateAdded, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function insert_bookmark_notification() { + let observer = expectPlacesObserverNotifications(["bookmark-added"]); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://example.com/"), + title: "a bookmark", + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + observer.check([ + { + type: "bookmark-added", + id: itemId, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentId, + index: bm.index, + url: bm.url, + title: bm.title, + dateAdded: bm.dateAdded, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function insert_bookmark_notitle_notification() { + let observer = expectPlacesObserverNotifications(["bookmark-added"]); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://example.com/"), + }); + strictEqual(bm.title, "", "Should return empty string for untitled bookmark"); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + observer.check([ + { + type: "bookmark-added", + id: itemId, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentId, + index: bm.index, + url: bm.url, + title: bm.title, + dateAdded: bm.dateAdded, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function insert_bookmark_tag_notification() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://tag.example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + let tagFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "tag", + }); + const observer = expectPlacesObserverNotifications([ + "bookmark-added", + "bookmark-tags-changed", + ]); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tagFolder.guid, + url: new URL("http://tag.example.com/"), + }); + let tagId = await PlacesUtils.promiseItemId(tag.guid); + let tagParentId = await PlacesUtils.promiseItemId(tag.parentGuid); + + observer.check([ + { + type: "bookmark-added", + id: tagId, + parentId: tagParentId, + index: tag.index, + itemType: tag.type, + url: tag.url, + title: "", + dateAdded: tag.dateAdded, + guid: tag.guid, + parentGuid: tag.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: true, + }, + { + type: "bookmark-tags-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + tags: ["tag"], + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function update_bookmark_lastModified() { + let timerPrecision = Preferences.get("privacy.reduceTimerPrecision"); + Preferences.set("privacy.reduceTimerPrecision", false); + + registerCleanupFunction(function () { + Preferences.set("privacy.reduceTimerPrecision", timerPrecision); + }); + + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://lastmod.example.com/"), + }); + const observer = expectPlacesObserverNotifications(["bookmark-time-changed"]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + lastModified: new Date(), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + observer.check([ + { + type: "bookmark-time-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + dateAdded: bm.dateAdded, + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function update_bookmark_title() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://title.example.com/"), + }); + const observer = expectPlacesObserverNotifications([ + "bookmark-title-changed", + ]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + title: "new title", + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + observer.check([ + { + type: "bookmark-title-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + title: bm.title, + guid: bm.guid, + parentGuid: bm.parentGuid, + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function update_bookmark_uri() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://url.example.com/"), + }); + const observer = expectPlacesObserverNotifications(["bookmark-url-changed"]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + url: "http://mozilla.org/", + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + observer.check([ + { + type: "bookmark-url-changed", + id: itemId, + itemType: bm.type, + url: bm.url.href, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + lastModified: bm.lastModified, + }, + ]); +}); + +add_task(async function update_move_same_folder() { + // Ensure there are at least two items in place (others test do so for us, + // but we don't have to depend on that). + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://move.example.com/"), + }); + let bmItemId = await PlacesUtils.promiseItemId(bm.guid); + let bmOldIndex = bm.index; + + let observer = expectPlacesObserverNotifications(["bookmark-moved"]); + + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: 0, + }); + Assert.equal(bm.index, 0); + observer.check([ + { + type: "bookmark-moved", + id: bmItemId, + itemType: bm.type, + url: "http://move.example.com/", + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + index: bm.index, + oldParentGuid: bm.parentGuid, + oldIndex: bmOldIndex, + isTagging: false, + }, + ]); + + // Test that we get the right index for DEFAULT_INDEX input. + bmOldIndex = 0; + observer = expectPlacesObserverNotifications(["bookmark-moved"]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + Assert.ok(bm.index > 0); + observer.check([ + { + type: "bookmark-moved", + id: bmItemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + index: bm.index, + oldParentGuid: bm.parentGuid, + oldIndex: bmOldIndex, + isTagging: false, + }, + ]); +}); + +add_task(async function update_move_different_folder() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://move.example.com/"), + }); + let folder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let bmItemId = await PlacesUtils.promiseItemId(bm.guid); + let bmOldIndex = bm.index; + + const observer = expectPlacesObserverNotifications(["bookmark-moved"]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: folder.guid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + Assert.equal(bm.index, 0); + observer.check([ + { + type: "bookmark-moved", + id: bmItemId, + itemType: bm.type, + url: "http://move.example.com/", + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + index: bm.index, + oldParentGuid: PlacesUtils.bookmarks.unfiledGuid, + oldIndex: bmOldIndex, + isTagging: false, + }, + ]); +}); + +add_task(async function update_move_tag_folder() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://move.example.com/"), + }); + let folder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "tag", + }); + let bmItemId = await PlacesUtils.promiseItemId(bm.guid); + let bmOldIndex = bm.index; + + const observer = expectPlacesObserverNotifications(["bookmark-moved"]); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: folder.guid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + Assert.equal(bm.index, 0); + observer.check([ + { + type: "bookmark-moved", + id: bmItemId, + itemType: bm.type, + url: "http://move.example.com/", + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + index: bm.index, + oldParentGuid: PlacesUtils.bookmarks.unfiledGuid, + oldIndex: bmOldIndex, + isTagging: true, + }, + ]); +}); + +add_task(async function remove_bookmark() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://remove.example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.remove(bm.guid); + observer.check([ + { + type: "bookmark-removed", + id: itemId, + parentId, + index: bm.index, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + ]); +}); + +add_task(async function remove_multiple_bookmarks() { + let bm1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://remove.example.com/", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://remove1.example.com/", + }); + let itemId1 = await PlacesUtils.promiseItemId(bm1.guid); + let parentId1 = await PlacesUtils.promiseItemId(bm1.parentGuid); + let itemId2 = await PlacesUtils.promiseItemId(bm2.guid); + let parentId2 = await PlacesUtils.promiseItemId(bm2.parentGuid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.remove([bm1, bm2]); + observer.check([ + { + type: "bookmark-removed", + id: itemId1, + parentId: parentId1, + index: bm1.index, + url: bm1.url, + guid: bm1.guid, + parentGuid: bm1.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: itemId2, + parentId: parentId2, + index: bm2.index - 1, + url: bm2.url, + guid: bm2.guid, + parentGuid: bm2.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + ]); +}); + +add_task(async function remove_folder() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.remove(bm.guid); + observer.check([ + { + type: "bookmark-removed", + id: itemId, + parentId, + index: bm.index, + url: null, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + ]); +}); + +add_task(async function remove_bookmark_tag_notification() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://untag.example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + let tagFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "tag", + }); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tagFolder.guid, + url: new URL("http://untag.example.com/"), + }); + let tagId = await PlacesUtils.promiseItemId(tag.guid); + let tagParentId = await PlacesUtils.promiseItemId(tag.parentGuid); + + const observer = expectPlacesObserverNotifications([ + "bookmark-removed", + "bookmark-tags-changed", + ]); + await PlacesUtils.bookmarks.remove(tag.guid); + + observer.check([ + { + type: "bookmark-removed", + id: tagId, + parentId: tagParentId, + index: tag.index, + url: tag.url, + guid: tag.guid, + parentGuid: tag.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: true, + }, + { + type: "bookmark-tags-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + tags: [], + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function rename_bookmark_tag_notification() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL("http://renametag.example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + let tagFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "tag", + }); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tagFolder.guid, + url: new URL("http://renametag.example.com/"), + }); + let tagParentId = await PlacesUtils.promiseItemId(tag.parentGuid); + + const observer = expectPlacesObserverNotifications([ + "bookmark-title-changed", + "bookmark-tags-changed", + ]); + tagFolder = await PlacesUtils.bookmarks.update({ + guid: tagFolder.guid, + title: "renamed", + }); + + observer.check([ + { + type: "bookmark-title-changed", + id: tagParentId, + title: "renamed", + guid: tagFolder.guid, + url: "", + lastModified: tagFolder.lastModified, + parentGuid: tagFolder.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: true, + }, + { + type: "bookmark-tags-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + tags: ["renamed"], + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); + +add_task(async function remove_folder_notification() { + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let folder1Id = await PlacesUtils.promiseItemId(folder1.guid); + let folder1ParentId = await PlacesUtils.promiseItemId(folder1.parentGuid); + + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder1.guid, + url: new URL("http://example.com/"), + }); + let bmItemId = await PlacesUtils.promiseItemId(bm.guid); + + let folder2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: folder1.guid, + }); + let folder2Id = await PlacesUtils.promiseItemId(folder2.guid); + + let bm2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder2.guid, + url: new URL("http://example.com/"), + }); + let bm2ItemId = await PlacesUtils.promiseItemId(bm2.guid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.remove(folder1.guid); + + observer.check([ + { + type: "bookmark-removed", + id: bm2ItemId, + parentId: folder2Id, + index: bm2.index, + url: bm2.url, + guid: bm2.guid, + parentGuid: bm2.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder2Id, + parentId: folder1Id, + index: folder2.index, + url: null, + guid: folder2.guid, + parentGuid: folder2.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + { + type: "bookmark-removed", + id: bmItemId, + parentId: folder1Id, + index: bm.index, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder1Id, + parentId: folder1ParentId, + index: folder1.index, + url: null, + guid: folder1.guid, + parentGuid: folder1.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + ]); +}); + +add_task(async function multiple_tags() { + const BOOKMARK_URL = "http://multipletags.example.com/"; + const TAG_NAMES = ["tag1", "tag2", "tag3", "tag4", "tag5", "tag6"]; + + const bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: new URL(BOOKMARK_URL), + }); + const itemId = await PlacesUtils.promiseItemId(bm.guid); + + info("Register all tags"); + const tagFolders = await Promise.all( + TAG_NAMES.map(tagName => + PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: tagName, + }) + ) + ); + + info("Test adding tags to bookmark"); + for (let i = 0; i < tagFolders.length; i++) { + const tagFolder = tagFolders[i]; + const expectedTagNames = TAG_NAMES.slice(0, i + 1); + + const observer = expectPlacesObserverNotifications([ + "bookmark-tags-changed", + ]); + + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tagFolder.guid, + url: new URL(BOOKMARK_URL), + }); + + observer.check([ + { + type: "bookmark-tags-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + tags: expectedTagNames, + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + } + + info("Test removing tags from bookmark"); + for (const removedLength of [1, 2, 3]) { + const removedTags = tagFolders.splice(0, removedLength); + + const observer = expectPlacesObserverNotifications([ + "bookmark-tags-changed", + ]); + + // We can remove multiple tags at one time. + await PlacesUtils.bookmarks.remove(removedTags); + + const expectedResults = []; + + for (let i = 0; i < removedLength; i++) { + TAG_NAMES.splice(0, 1); + const expectedTagNames = [...TAG_NAMES]; + + expectedResults.push({ + type: "bookmark-tags-changed", + id: itemId, + itemType: bm.type, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + tags: expectedTagNames, + lastModified: bm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }); + } + + observer.check(expectedResults); + } +}); + +add_task(async function eraseEverything_notification() { + // Let's start from a clean situation. + await PlacesUtils.bookmarks.eraseEverything(); + + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let folder1Id = await PlacesUtils.promiseItemId(folder1.guid); + let folder1ParentId = await PlacesUtils.promiseItemId(folder1.parentGuid); + + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder1.guid, + url: new URL("http://example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + + let folder2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let folder2Id = await PlacesUtils.promiseItemId(folder2.guid); + let folder2ParentId = await PlacesUtils.promiseItemId(folder2.parentGuid); + + let toolbarBm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: new URL("http://example.com/"), + }); + let toolbarBmId = await PlacesUtils.promiseItemId(toolbarBm.guid); + let toolbarBmParentId = await PlacesUtils.promiseItemId(toolbarBm.parentGuid); + + let menuBm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: new URL("http://example.com/"), + }); + let menuBmId = await PlacesUtils.promiseItemId(menuBm.guid); + let menuBmParentId = await PlacesUtils.promiseItemId(menuBm.parentGuid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.eraseEverything(); + + // Bookmarks should always be notified before their parents. + observer.check([ + { + type: "bookmark-removed", + id: itemId, + parentId, + index: bm.index, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder2Id, + parentId: folder2ParentId, + index: folder2.index, + url: null, + guid: folder2.guid, + parentGuid: folder2.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder1Id, + parentId: folder1ParentId, + index: folder1.index, + url: null, + guid: folder1.guid, + parentGuid: folder1.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + { + type: "bookmark-removed", + id: menuBmId, + parentId: menuBmParentId, + index: menuBm.index, + url: menuBm.url, + guid: menuBm.guid, + parentGuid: menuBm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: toolbarBmId, + parentId: toolbarBmParentId, + index: toolbarBm.index, + url: toolbarBm.url, + guid: toolbarBm.guid, + parentGuid: toolbarBm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + ]); +}); + +add_task(async function eraseEverything_reparented_notification() { + // Let's start from a clean situation. + await PlacesUtils.bookmarks.eraseEverything(); + + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let folder1Id = await PlacesUtils.promiseItemId(folder1.guid); + let folder1ParentId = await PlacesUtils.promiseItemId(folder1.parentGuid); + + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder1.guid, + url: new URL("http://example.com/"), + }); + let itemId = await PlacesUtils.promiseItemId(bm.guid); + + let folder2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let folder2Id = await PlacesUtils.promiseItemId(folder2.guid); + let folder2ParentId = await PlacesUtils.promiseItemId(folder2.parentGuid); + + bm.parentGuid = folder2.guid; + bm = await PlacesUtils.bookmarks.update(bm); + let parentId = await PlacesUtils.promiseItemId(bm.parentGuid); + + let observer = expectPlacesObserverNotifications(["bookmark-removed"]); + await PlacesUtils.bookmarks.eraseEverything(); + + // Bookmarks should always be notified before their parents. + observer.check([ + { + type: "bookmark-removed", + id: itemId, + parentId, + index: bm.index, + url: bm.url, + guid: bm.guid, + parentGuid: bm.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder2Id, + parentId: folder2ParentId, + index: folder2.index, + url: null, + guid: folder2.guid, + parentGuid: folder2.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + { + type: "bookmark-removed", + id: folder1Id, + parentId: folder1ParentId, + index: folder1.index, + url: null, + guid: folder1.guid, + parentGuid: folder1.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + isTagging: false, + }, + ]); +}); + +add_task(async function reorder_notification() { + let bookmarks = [ + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example1.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example2.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example3.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + ]; + let sorted = []; + for (let bm of bookmarks) { + sorted.push(await PlacesUtils.bookmarks.insert(bm)); + } + + // Randomly reorder the array. + sorted.sort(() => 0.5 - Math.random()); + // Ensure there's at least one item out of place, since random does not + // necessarily mean they are unordered. + if (sorted[0].url == bookmarks[0].url) { + sorted.push(sorted.shift()); + } + + const observer = expectPlacesObserverNotifications(["bookmark-moved"]); + await PlacesUtils.bookmarks.reorder( + PlacesUtils.bookmarks.unfiledGuid, + sorted.map(bm => bm.guid) + ); + + let expectedNotifications = []; + for (let i = 0; i < sorted.length; ++i) { + let child = sorted[i]; + let childId = await PlacesUtils.promiseItemId(child.guid); + expectedNotifications.push({ + type: "bookmark-moved", + id: childId, + itemType: child.type, + url: child.url || "", + guid: child.guid, + parentGuid: child.parentGuid, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + index: i, + oldParentGuid: child.parentGuid, + oldIndex: child.index, + isTagging: false, + }); + } + + observer.check(expectedNotifications); +}); + +add_task(async function update_notitle_notification() { + let toolbarBmURI = Services.io.newURI("https://example.com"); + let toolbarItemId = await PlacesUtils.promiseItemId( + PlacesUtils.bookmarks.toolbarGuid + ); + let toolbarBmId = PlacesUtils.bookmarks.insertBookmark( + toolbarItemId, + toolbarBmURI, + 0, + "Bookmark" + ); + let toolbarBmGuid = await PlacesUtils.promiseItemGuid(toolbarBmId); + + let menuFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + index: 0, + title: "Folder", + }); + let menuFolderId = await PlacesUtils.promiseItemId(menuFolder.guid); + + const observer = expectPlacesObserverNotifications([ + "bookmark-title-changed", + ]); + + PlacesUtils.bookmarks.setItemTitle(toolbarBmId, null); + strictEqual( + PlacesUtils.bookmarks.getItemTitle(toolbarBmId), + "", + "Legacy API should return empty string for untitled bookmark" + ); + + let updatedMenuBm = await PlacesUtils.bookmarks.update({ + guid: menuFolder.guid, + title: null, + }); + strictEqual( + updatedMenuBm.title, + "", + "Async API should return empty string for untitled bookmark" + ); + + let toolbarBmModified = await PlacesUtils.bookmarks.fetch(toolbarBmGuid); + observer.check([ + { + type: "bookmark-title-changed", + id: toolbarBmId, + itemType: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: toolbarBmURI.spec, + title: "", + guid: toolbarBmGuid, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + lastModified: toolbarBmModified.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-title-changed", + id: menuFolderId, + itemType: PlacesUtils.bookmarks.TYPE_FOLDER, + url: "", + title: "", + guid: menuFolder.guid, + parentGuid: PlacesUtils.bookmarks.menuGuid, + lastModified: updatedMenuBm.lastModified, + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_remove.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_remove.js new file mode 100644 index 0000000000..01bb591e3c --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_remove.js @@ -0,0 +1,465 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const UNVISITED_BOOKMARK_BONUS = 140; + +function promiseRankingChanged() { + return PlacesTestUtils.waitForNotification("pages-rank-changed"); +} + +add_task(async function setup() { + Services.prefs.setIntPref( + "places.frecency.unvisitedBookmarkBonus", + UNVISITED_BOOKMARK_BONUS + ); +}); + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.remove(), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove(null), + /Input should be a valid object/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.remove("test"), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove(123), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.remove({ guid: "test" }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ guid: null }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ guid: 123 }), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.remove({ parentGuid: "test" }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ parentGuid: null }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ parentGuid: 123 }), + /Invalid value for property 'parentGuid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.remove({ url: "http://te st/" }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ url: null }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.remove({ url: -10 }), + /Invalid value for property 'url'/ + ); +}); + +add_task(async function remove_nonexistent_guid() { + try { + await PlacesUtils.bookmarks.remove({ guid: "123456789012" }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/No bookmarks found for the provided GUID/.test(ex)); + } +}); + +add_task(async function remove_roots_fail() { + let guids = [ + PlacesUtils.bookmarks.rootGuid, + PlacesUtils.bookmarks.unfiledGuid, + PlacesUtils.bookmarks.menuGuid, + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.tagsGuid, + PlacesUtils.bookmarks.mobileGuid, + ]; + for (let guid of guids) { + Assert.throws( + () => PlacesUtils.bookmarks.remove(guid), + /It's not possible to remove Places root folders\./ + ); + } +}); + +add_task(async function remove_bookmark() { + // When removing a bookmark we need to check the frecency. First we confirm + // that there is a normal update when it is inserted. + let promise = promiseRankingChanged(); + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await promise; + + // This second one checks the frecency is changed when we remove the bookmark. + promise = promiseRankingChanged(); + + await PlacesUtils.bookmarks.remove(bm1.guid); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await promise; +}); + +add_task(async function remove_multiple_bookmarks_simple() { + // When removing a bookmark we need to check the frecency. First we confirm + // that there is a normal update when it is inserted. + const promise1 = promiseRankingChanged(); + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + const promise2 = promiseRankingChanged(); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example1.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm2); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await Promise.all([promise1, promise2]); + + // We should get a pages-rank-changed event with the removal of + // multiple bookmarks. + const promise3 = promiseRankingChanged(); + + await PlacesUtils.bookmarks.remove([bm1, bm2]); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await promise3; +}); + +add_task(async function remove_multiple_bookmarks_complex() { + let bms = []; + for (let i = 0; i < 10; i++) { + bms.push( + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: `http://example.com/${i}`, + title: `bookmark ${i}`, + }) + ); + } + + // Remove bookmarks 2 and 3. + let bmsToRemove = bms.slice(2, 4); + let notifiedIndexes = []; + let notificationPromise = PlacesTestUtils.waitForNotification( + "bookmark-removed", + events => { + for (let event of events) { + notifiedIndexes.push({ guid: event.guid, index: event.index }); + } + return notifiedIndexes.length == bmsToRemove.length; + } + ); + await PlacesUtils.bookmarks.remove(bmsToRemove); + await notificationPromise; + + let indexModifier = 0; + for (let i = 0; i < bmsToRemove.length; i++) { + Assert.equal( + notifiedIndexes[i].guid, + bmsToRemove[i].guid, + `Should have been notified of the correct guid for item ${i}` + ); + Assert.equal( + notifiedIndexes[i].index, + bmsToRemove[i].index - indexModifier, + `Should have been notified of the correct index for the item ${i}` + ); + indexModifier++; + } + + let expectedIndex = 0; + for (let bm of [bms[0], bms[1], ...bms.slice(4)]) { + const fetched = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal( + fetched.index, + expectedIndex, + "Should have the correct index after consecutive item removal" + ); + bm.index = fetched.index; + expectedIndex++; + } + + // Remove some more including non-consecutive. + bmsToRemove = [bms[1], bms[5], bms[6], bms[8]]; + notifiedIndexes = []; + notificationPromise = PlacesTestUtils.waitForNotification( + "bookmark-removed", + events => { + for (let event of events) { + notifiedIndexes.push({ guid: event.guid, index: event.index }); + } + return notifiedIndexes.length == bmsToRemove.length; + } + ); + await PlacesUtils.bookmarks.remove(bmsToRemove); + await notificationPromise; + + indexModifier = 0; + for (let i = 0; i < bmsToRemove.length; i++) { + Assert.equal( + notifiedIndexes[i].guid, + bmsToRemove[i].guid, + `Should have been notified of the correct guid for item ${i}` + ); + Assert.equal( + notifiedIndexes[i].index, + bmsToRemove[i].index - indexModifier, + `Should have been notified of the correct index for the item ${i}` + ); + indexModifier++; + } + + expectedIndex = 0; + const expectedRemaining = [bms[0], bms[4], bms[7], bms[9]]; + for (let bm of expectedRemaining) { + const fetched = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal( + fetched.index, + expectedIndex, + "Should have the correct index after non-consecutive item removal" + ); + expectedIndex++; + } + + // Tidy up + await PlacesUtils.bookmarks.remove(expectedRemaining); + await PlacesTestUtils.promiseAsyncUpdates(); +}); + +add_task(async function remove_bookmark_empty_title() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "", + }); + checkBookmarkObject(bm1); + + await PlacesUtils.bookmarks.remove(bm1.guid); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); +}); + +add_task(async function remove_folder() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + checkBookmarkObject(bm1); + + await PlacesUtils.bookmarks.remove(bm1.guid); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); + + // No wait for pages-rank-changed event in this test as the folder doesn't have + // any children that would need updating. +}); + +add_task(async function test_contents_removed() { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "", + }); + + let skipDescendantsObserver = expectPlacesObserverNotifications( + ["bookmark-removed"], + false, + true + ); + let receiveAllObserver = expectPlacesObserverNotifications( + ["bookmark-removed"], + false, + false + ); + const promise = promiseRankingChanged(); + await PlacesUtils.bookmarks.remove(folder1); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(folder1.guid), null); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await promise; + + let expectedNotifications = [ + { + type: "bookmark-removed", + guid: folder1.guid, + }, + ]; + + // If we're skipping descendents, we'll only be notified of the folder. + skipDescendantsObserver.check(expectedNotifications); + + // Note: Items of folders get notified first. + expectedNotifications.unshift({ + type: "bookmark-removed", + guid: bm1.guid, + }); + // If we don't skip descendents, we'll be notified of the folder and the + // bookmark. + receiveAllObserver.check(expectedNotifications); +}); + +add_task(async function test_nested_contents_removed() { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "", + }); + + const promise = promiseRankingChanged(); + await PlacesUtils.bookmarks.remove(folder1); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(folder1.guid), null); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(folder2.guid), null); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + await promise; +}); + +add_task(async function remove_folder_empty_title() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "", + }); + checkBookmarkObject(bm1); + + await PlacesUtils.bookmarks.remove(bm1.guid); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); +}); + +add_task(async function remove_separator() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + checkBookmarkObject(bm1); + + await PlacesUtils.bookmarks.remove(bm1.guid); + Assert.strictEqual(await PlacesUtils.bookmarks.fetch(bm1.guid), null); +}); + +add_task(async function test_nested_content_fails_when_not_allowed() { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a folder", + }); + await Assert.rejects( + PlacesUtils.bookmarks.remove(folder1, { + preventRemovalOfNonEmptyFolders: true, + }), + /Cannot remove a non-empty folder./ + ); +}); + +add_task(async function test_remove_bookmark_with_invalid_url() { + let folder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + title: "folder", + }); + let guid = "invalid_____"; + let folderedGuid = "invalid____2"; + let url = "invalid-uri"; + await PlacesUtils.withConnectionWrapper("test_bookmarks_remove", async db => { + await db.execute( + ` + INSERT INTO moz_places(url, url_hash, title, rev_host, guid) + VALUES (:url, hash(:url), 'Invalid URI', '.', GENERATE_GUID()) + `, + { url } + ); + await db.execute( + `INSERT INTO moz_bookmarks (type, fk, parent, position, guid) + VALUES (:type, + (SELECT id FROM moz_places WHERE url = :url), + (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid), + (SELECT MAX(position) + 1 FROM moz_bookmarks WHERE parent = (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid)), + :guid) + `, + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + guid, + } + ); + await db.execute( + `INSERT INTO moz_bookmarks (type, fk, parent, position, guid) + VALUES (:type, + (SELECT id FROM moz_places WHERE url = :url), + (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid), + (SELECT MAX(position) + 1 FROM moz_bookmarks WHERE parent = (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid)), + :guid) + `, + { + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url, + parentGuid: folder.guid, + guid: folderedGuid, + } + ); + }); + await PlacesUtils.bookmarks.remove(guid); + Assert.strictEqual( + await PlacesUtils.bookmarks.fetch(guid), + null, + "Should not throw and not find the bookmark" + ); + + await PlacesUtils.bookmarks.remove(folder); + Assert.strictEqual( + await PlacesUtils.bookmarks.fetch(folderedGuid), + null, + "Should not throw and not find the bookmark" + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_remove_batch.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_remove_batch.js new file mode 100644 index 0000000000..51a2a4dc2b --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_remove_batch.js @@ -0,0 +1,129 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Test whether do batch removal if multiple bookmarks are removed at once. + +add_task(async function test_remove_multiple_bookmarks() { + info("Test for remove multiple bookmarks at once"); + + info("Insert multiple bookmarks"); + const testBookmarks = [ + { + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/1", + }, + { + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/2", + }, + { + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/3", + }, + ]; + const bookmarks = await Promise.all( + testBookmarks.map(bookmark => PlacesUtils.bookmarks.insert(bookmark)) + ); + Assert.equal( + bookmarks.length, + testBookmarks.length, + "All test data are insterted correctly" + ); + + info("Remove multiple bookmarks"); + const onRemoved = PlacesTestUtils.waitForNotification("bookmark-removed"); + await PlacesUtils.bookmarks.remove(bookmarks); + const events = await onRemoved; + + info("Check whether all bookmark-removed events called at at once or not"); + assertBookmarkRemovedEvents(events, bookmarks); +}); + +add_task(async function test_remove_folder_with_bookmarks() { + info("Test for remove a folder that has multiple bookmarks"); + + info("Insert a folder"); + const testFolder = { + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }; + const folder = await PlacesUtils.bookmarks.insert(testFolder); + Assert.ok(folder, "A folder is inserted correctly"); + + info("Insert multiple bookmarks to inserted folder"); + const testBookmarks = [ + { + parentGuid: folder.guid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/1", + }, + { + parentGuid: folder.guid, + url: "http://example.com/2", + }, + { + parentGuid: folder.guid, + url: "http://example.com/3", + }, + ]; + const bookmarks = await Promise.all( + testBookmarks.map(bookmark => PlacesUtils.bookmarks.insert(bookmark)) + ); + Assert.equal( + bookmarks.length, + testBookmarks.length, + "All test data are insterted correctly" + ); + + info("Remove the inserted folder"); + const notifiedEvents = []; + const onRemoved = PlacesTestUtils.waitForNotification( + "bookmark-removed", + events => { + notifiedEvents.push(events); + return notifiedEvents.length === 2; + } + ); + await PlacesUtils.bookmarks.remove(folder); + await onRemoved; + + info("Check whether all bookmark-removed events called at at once or not"); + const eventsForBookmarks = notifiedEvents[0]; + assertBookmarkRemovedEvents(eventsForBookmarks, bookmarks); + + info("Check whether a bookmark-removed event called for the folder"); + const eventsForFolder = notifiedEvents[1]; + Assert.equal( + eventsForFolder.length, + 1, + "The length of notified events is correct" + ); + Assert.equal( + eventsForFolder[0].guid, + folder.guid, + "The guid of event is correct" + ); +}); + +function assertBookmarkRemovedEvents(events, expectedBookmarks) { + Assert.equal( + events.length, + expectedBookmarks.length, + "The length of notified events is correct" + ); + + for (let i = 0; i < events.length; i++) { + const event = events[i]; + const expectedBookmark = expectedBookmarks[i]; + Assert.equal( + event.guid, + expectedBookmark.guid, + `The guid of events[${i}] is correct` + ); + Assert.equal( + event.url, + expectedBookmark.url, + `The url of events[${i}] is correct` + ); + } +} diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_reorder.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_reorder.js new file mode 100644 index 0000000000..7df909c704 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_reorder.js @@ -0,0 +1,310 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.reorder(), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder(null), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.reorder("test"), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder(123), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.reorder({ guid: "test" }), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012"), + /Must provide a sorted array of children GUIDs./ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", {}), + /Must provide a sorted array of children GUIDs./ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", null), + /Must provide a sorted array of children GUIDs./ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", []), + /Must provide a sorted array of children GUIDs./ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", [null]), + /Invalid GUID found in the sorted children array/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", [""]), + /Invalid GUID found in the sorted children array/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", [{}]), + /Invalid GUID found in the sorted children array/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.reorder("123456789012", ["012345678901", null]), + /Invalid GUID found in the sorted children array/ + ); +}); + +add_task(async function reorder_nonexistent_guid() { + await Assert.rejects( + PlacesUtils.bookmarks.reorder("123456789012", ["012345678901"]), + /No folder found for the provided GUID/, + "Should throw for nonexisting guid" + ); +}); + +add_task(async function reorder() { + let bookmarks = [ + { + url: "http://example1.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + url: "http://example2.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + { + url: "http://example3.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }, + ]; + + let sorted = []; + for (let bm of bookmarks) { + sorted.push(await PlacesUtils.bookmarks.insert(bm)); + } + + // Check the initial append sorting. + Assert.ok( + sorted.every((bm, i) => bm.index == i), + "Initial bookmarks sorting is correct" + ); + + // Apply random sorting and run multiple tests. + for (let t = 0; t < 4; t++) { + sorted.sort(() => 0.5 - Math.random()); + let sortedGuids = sorted.map(child => child.guid); + dump("Expected order: " + sortedGuids.join() + "\n"); + // Add a nonexisting guid to the array, to ensure nothing will break. + sortedGuids.push("123456789012"); + await PlacesUtils.bookmarks.reorder( + PlacesUtils.bookmarks.unfiledGuid, + sortedGuids + ); + for (let i = 0; i < sorted.length; ++i) { + let item = await PlacesUtils.bookmarks.fetch(sorted[i].guid); + Assert.equal(item.index, i); + } + } + + info("Test partial sorting"); + { + // Try a partial sorting by passing 2 entries in same order as they + // currently have. No entries should change order. + let sortedGuids = [sorted[0].guid, sorted[3].guid]; + dump("Expected order: " + sorted.map(b => b.guid).join() + "\n"); + await PlacesUtils.bookmarks.reorder( + PlacesUtils.bookmarks.unfiledGuid, + sortedGuids + ); + for (let i = 0; i < sorted.length; ++i) { + let item = await PlacesUtils.bookmarks.fetch(sorted[i].guid); + Assert.equal(item.index, i); + } + } + + { + // Try a partial sorting by passing 2 entries out of order + // The unspecified entries should be appended and retain the original order + sorted = [sorted[1], sorted[0]].concat(sorted.slice(2)); + let sortedGuids = [sorted[0].guid, sorted[1].guid]; + dump("Expected order: " + sorted.map(b => b.guid).join() + "\n"); + await PlacesUtils.bookmarks.reorder( + PlacesUtils.bookmarks.unfiledGuid, + sortedGuids + ); + for (let i = 0; i < sorted.length; ++i) { + let item = await PlacesUtils.bookmarks.fetch(sorted[i].guid); + Assert.equal(item.index, i); + } + } + // Use triangular numbers to detect skipped position. + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.execute( + `SELECT parent + FROM moz_bookmarks + GROUP BY parent + HAVING (SUM(DISTINCT position + 1) - (count(*) * (count(*) + 1) / 2)) <> 0` + ); + Assert.equal( + rows.length, + 0, + "All the bookmarks should have consistent positions" + ); +}); + +add_task(async function move_and_reorder() { + // Start clean. + await PlacesUtils.bookmarks.eraseEverything(); + + let bm1 = await PlacesUtils.bookmarks.insert({ + url: "http://example1.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let f1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + url: "http://example2.com/", + parentGuid: f1.guid, + }); + let f2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + url: "http://example3.com/", + parentGuid: f2.guid, + }); + let bm4 = await PlacesUtils.bookmarks.insert({ + url: "http://example4.com/", + parentGuid: f2.guid, + }); + let bm5 = await PlacesUtils.bookmarks.insert({ + url: "http://example5.com/", + parentGuid: f2.guid, + }); + + // Invert f2 children. + // This is critical to reproduce the bug, cause it inverts the position + // compared to the natural insertion order. + await PlacesUtils.bookmarks.reorder(f2.guid, [bm5.guid, bm4.guid, bm3.guid]); + + bm1.parentGuid = f1.guid; + bm1.index = 0; + await PlacesUtils.bookmarks.update(bm1); + + bm1 = await PlacesUtils.bookmarks.fetch(bm1.guid); + Assert.equal(bm1.index, 0); + bm2 = await PlacesUtils.bookmarks.fetch(bm2.guid); + Assert.equal(bm2.index, 1); + bm3 = await PlacesUtils.bookmarks.fetch(bm3.guid); + Assert.equal(bm3.index, 2); + bm4 = await PlacesUtils.bookmarks.fetch(bm4.guid); + Assert.equal(bm4.index, 1); + bm5 = await PlacesUtils.bookmarks.fetch(bm5.guid); + Assert.equal(bm5.index, 0); + + // No-op reorder on f1 children. + // Nothing should change. Though, due to bug 1293365 this was causing children + // of other folders to get messed up. + await PlacesUtils.bookmarks.reorder(f1.guid, [bm1.guid, bm2.guid]); + + bm1 = await PlacesUtils.bookmarks.fetch(bm1.guid); + Assert.equal(bm1.index, 0); + bm2 = await PlacesUtils.bookmarks.fetch(bm2.guid); + Assert.equal(bm2.index, 1); + bm3 = await PlacesUtils.bookmarks.fetch(bm3.guid); + Assert.equal(bm3.index, 2); + bm4 = await PlacesUtils.bookmarks.fetch(bm4.guid); + Assert.equal(bm4.index, 1); + bm5 = await PlacesUtils.bookmarks.fetch(bm5.guid); + Assert.equal(bm5.index, 0); +}); + +add_task(async function reorder_empty_folder_invalid_children() { + // Start clean. + await PlacesUtils.bookmarks.eraseEverything(); + + let f1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + // Specifying a child that doesn't exist should cause that to be ignored. + // However, before bug 1333304, doing this on an empty folder threw. + await PlacesUtils.bookmarks.reorder(f1.guid, ["123456789012"]); +}); + +add_task(async function reorder_lastModified() { + // Start clean. + await PlacesUtils.bookmarks.eraseEverything(); + + let lastModified = new Date(Date.now() - 1000); + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.menuGuid, + children: [ + { + guid: "bookmarkAAAA", + url: "http://example.com/a", + title: "A", + dateAdded: lastModified, + lastModified, + }, + { + guid: "bookmarkBBBB", + url: "http://example.com/b", + title: "B", + dateAdded: lastModified, + lastModified, + }, + ], + }); + await PlacesUtils.bookmarks.update({ + guid: PlacesUtils.bookmarks.menuGuid, + lastModified, + }); + + info("Reorder and set explicit last modified time"); + let newLastModified = new Date(lastModified.getTime() + 500); + await PlacesUtils.bookmarks.reorder( + PlacesUtils.bookmarks.menuGuid, + ["bookmarkBBBB", "bookmarkAAAA"], + { lastModified: newLastModified } + ); + for (let guid of [ + PlacesUtils.bookmarks.menuGuid, + "bookmarkAAAA", + "bookmarkBBBB", + ]) { + let info = await PlacesUtils.bookmarks.fetch(guid); + Assert.equal(info.lastModified.getTime(), newLastModified.getTime()); + } + + info("Reorder and set default last modified time"); + await PlacesUtils.bookmarks.reorder(PlacesUtils.bookmarks.menuGuid, [ + "bookmarkAAAA", + "bookmarkBBBB", + ]); + for (let guid of [ + PlacesUtils.bookmarks.menuGuid, + "bookmarkAAAA", + "bookmarkBBBB", + ]) { + let info = await PlacesUtils.bookmarks.fetch(guid); + Assert.greater(info.lastModified.getTime(), newLastModified.getTime()); + } +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_search.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_search.js new file mode 100644 index 0000000000..2b34b2c8f9 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_search.js @@ -0,0 +1,339 @@ +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.search(), + /Query object is required/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.search(null), + /Query object is required/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.search({ title: 50 }), + /Title option must be a string/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.search({ url: { url: "wombat" } }), + /Url option must be a string or a URL object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.search(50), + /Query must be an object or a string/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.search(true), + /Query must be an object or a string/ + ); +}); + +add_task(async function search_bookmark() { + await PlacesUtils.bookmarks.eraseEverything(); + + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/", + title: "another bookmark", + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://menu.org/", + title: "an on-menu bookmark", + }); + let bm4 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://toolbar.org/", + title: "an on-toolbar bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + checkBookmarkObject(bm3); + checkBookmarkObject(bm4); + + // finds a result by query + let results = await PlacesUtils.bookmarks.search("example.com"); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm1, results[0]); + + // finds multiple results + results = await PlacesUtils.bookmarks.search("example"); + Assert.equal(results.length, 2); + checkBookmarkObject(results[0]); + checkBookmarkObject(results[1]); + + // finds menu bookmarks + results = await PlacesUtils.bookmarks.search("an on-menu bookmark"); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm3, results[0]); + + // finds toolbar bookmarks + results = await PlacesUtils.bookmarks.search("an on-toolbar bookmark"); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm4, results[0]); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_bookmark_by_query_object() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/", + title: "another bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + + let results = await PlacesUtils.bookmarks.search({ query: "example.com" }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + + Assert.deepEqual(bm1, results[0]); + + results = await PlacesUtils.bookmarks.search({ query: "example" }); + Assert.equal(results.length, 2); + checkBookmarkObject(results[0]); + checkBookmarkObject(results[1]); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_bookmark_by_url() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/path", + title: "another bookmark", + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/path", + title: "third bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + checkBookmarkObject(bm3); + + // finds the correct result by url + let results = await PlacesUtils.bookmarks.search({ + url: "http://example.com/", + }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm1, results[0]); + + // normalizes the url + results = await PlacesUtils.bookmarks.search({ url: "http:/example.com" }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm1, results[0]); + + // returns multiple matches + results = await PlacesUtils.bookmarks.search({ + url: "http://example.org/path", + }); + Assert.equal(results.length, 2); + + // requires exact match + results = await PlacesUtils.bookmarks.search({ url: "http://example.org/" }); + Assert.equal(results.length, 0); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_bookmark_by_title() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/path", + title: "another bookmark", + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.net/", + title: "another bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + checkBookmarkObject(bm3); + + // finds the correct result by title + let results = await PlacesUtils.bookmarks.search({ title: "a bookmark" }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm1, results[0]); + + // returns multiple matches + results = await PlacesUtils.bookmarks.search({ title: "another bookmark" }); + Assert.equal(results.length, 2); + + // requires exact match + results = await PlacesUtils.bookmarks.search({ title: "bookmark" }); + Assert.equal(results.length, 0); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_bookmark_combinations() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.org/path", + title: "another bookmark", + }); + let bm3 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.net/", + title: "third bookmark", + }); + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + checkBookmarkObject(bm3); + + // finds the correct result if title and url match + let results = await PlacesUtils.bookmarks.search({ + url: "http://example.com/", + title: "a bookmark", + }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm1, results[0]); + + // does not match if query is not matching but url and title match + results = await PlacesUtils.bookmarks.search({ + url: "http://example.com/", + title: "a bookmark", + query: "nonexistent", + }); + Assert.equal(results.length, 0); + + // does not match if one parameter is not matching + results = await PlacesUtils.bookmarks.search({ + url: "http://what.ever", + title: "a bookmark", + }); + Assert.equal(results.length, 0); + + // query only matches if other fields match as well + results = await PlacesUtils.bookmarks.search({ + query: "bookmark", + url: "http://example.net/", + }); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm3, results[0]); + + // non-matching query will also return no results + results = await PlacesUtils.bookmarks.search({ + query: "nonexistent", + url: "http://example.net/", + }); + Assert.equal(results.length, 0); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_folder() { + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "a test folder", + }); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(folder); + checkBookmarkObject(bm); + + // also finds folders + let results = await PlacesUtils.bookmarks.search("a test folder"); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.equal(folder.title, results[0].title); + Assert.equal(folder.type, results[0].type); + Assert.equal(folder.parentGuid, results[0].parentGuid); + + // finds elements in folders + results = await PlacesUtils.bookmarks.search("example.com"); + Assert.equal(results.length, 1); + checkBookmarkObject(results[0]); + Assert.deepEqual(bm, results[0]); + Assert.equal(folder.guid, results[0].parentGuid); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_includes_separators() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + let bm2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + + checkBookmarkObject(bm1); + checkBookmarkObject(bm2); + + let results = await PlacesUtils.bookmarks.search({}); + Assert.ok( + results.findIndex(bookmark => { + return bookmark.guid == bm1.guid; + }) > -1, + "The bookmark was found in the results." + ); + Assert.ok( + results.findIndex(bookmark => { + return bookmark.guid == bm2.guid; + }) > -1, + "The separator was included in the results." + ); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function search_excludes_tags() { + let bm1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + title: "a bookmark", + }); + checkBookmarkObject(bm1); + PlacesUtils.tagging.tagURI(bm1.url.URI, ["Test Tag"]); + + let results = await PlacesUtils.bookmarks.search("example.com"); + // If tags are not being excluded, this would return two results, one representing the tag. + Assert.equal(1, results.length, "A single object was returned from search."); + Assert.deepEqual(bm1, results[0], "The bookmark was returned."); + + results = await PlacesUtils.bookmarks.search("Test Tag"); + Assert.equal(0, results.length, "The tag folder was not returned."); + + await PlacesUtils.bookmarks.eraseEverything(); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarks_update.js b/toolkit/components/places/tests/bookmarks/test_bookmarks_update.js new file mode 100644 index 0000000000..1c9bead831 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarks_update.js @@ -0,0 +1,587 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_task(async function invalid_input_throws() { + Assert.throws( + () => PlacesUtils.bookmarks.update(), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update(null), + /Input should be a valid object/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({}), + /The following properties were expected/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ guid: "test" }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ guid: null }), + /Invalid value for property 'guid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ guid: 123 }), + /Invalid value for property 'guid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ parentGuid: "test" }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ parentGuid: null }), + /Invalid value for property 'parentGuid'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ parentGuid: 123 }), + /Invalid value for property 'parentGuid'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ index: "1" }), + /Invalid value for property 'index'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ index: -10 }), + /Invalid value for property 'index'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ dateAdded: -10 }), + /Invalid value for property 'dateAdded'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ dateAdded: "today" }), + /Invalid value for property 'dateAdded'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ dateAdded: Date.now() }), + /Invalid value for property 'dateAdded'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ lastModified: -10 }), + /Invalid value for property 'lastModified'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ lastModified: "today" }), + /Invalid value for property 'lastModified'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ lastModified: Date.now() }), + /Invalid value for property 'lastModified'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ type: -1 }), + /Invalid value for property 'type'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ type: 100 }), + /Invalid value for property 'type'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ type: "bookmark" }), + /Invalid value for property 'type'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ url: 10 }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ url: "http://te st" }), + /Invalid value for property 'url'/ + ); + let longurl = "http://www.example.com/"; + for (let i = 0; i < 65536; i++) { + longurl += "a"; + } + Assert.throws( + () => PlacesUtils.bookmarks.update({ url: longurl }), + /Invalid value for property 'url'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ url: NetUtil.newURI(longurl) }), + /Invalid value for property 'url'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ url: "te st" }), + /Invalid value for property 'url'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ title: -1 }), + /Invalid value for property 'title'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.update({ title: {} }), + /Invalid value for property 'title'/ + ); + + Assert.throws( + () => PlacesUtils.bookmarks.update({ guid: "123456789012" }), + /Not enough properties to update/ + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.update({ + guid: "123456789012", + parentGuid: "012345678901", + }), + /The following properties were expected: index/ + ); +}); + +add_task(async function move_roots_fail() { + let guids = [ + PlacesUtils.bookmarks.unfiledGuid, + PlacesUtils.bookmarks.menuGuid, + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.tagsGuid, + PlacesUtils.bookmarks.mobileGuid, + ]; + for (let guid of guids) { + await Assert.rejects( + PlacesUtils.bookmarks.update({ + guid, + index: -1, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }), + /It's not possible to move Places root folders\./, + `Should reject when attempting to move ${guid}` + ); + } +}); + +add_task(async function nonexisting_bookmark_throws() { + try { + await PlacesUtils.bookmarks.update({ guid: "123456789012", title: "test" }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/No bookmarks found for the provided GUID/.test(ex)); + } +}); + +add_task(async function invalid_properties_for_existing_bookmark() { + let bm = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + }); + + try { + await PlacesUtils.bookmarks.update({ + guid: bm.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/The bookmark type cannot be changed/.test(ex)); + } + + try { + await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: "123456789012", + index: 1, + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/No bookmarks found for the provided parentGuid/.test(ex)); + } + + let past = new Date(Date.now() - 86400000); + try { + await PlacesUtils.bookmarks.update({ guid: bm.guid, lastModified: past }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/Invalid value for property 'lastModified'/.test(ex)); + } + + let folder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + try { + await PlacesUtils.bookmarks.update({ + guid: folder.guid, + url: "http://example.com/", + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/Invalid value for property 'url'/.test(ex)); + } + + let separator = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + try { + await PlacesUtils.bookmarks.update({ + guid: separator.guid, + url: "http://example.com/", + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/Invalid value for property 'url'/.test(ex)); + } + try { + await PlacesUtils.bookmarks.update({ guid: separator.guid, title: "test" }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok(/Invalid value for property 'title'/.test(ex)); + } +}); + +add_task(async function long_title_trim() { + let longtitle = "a"; + for (let i = 0; i < 4096; i++) { + longtitle += "a"; + } + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "title", + }); + checkBookmarkObject(bm); + + bm = await PlacesUtils.bookmarks.update({ guid: bm.guid, title: longtitle }); + let newTitle = bm.title; + Assert.equal(newTitle.length, 4096, "title should have been trimmed"); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal(bm.title, newTitle); +}); + +add_task(async function update_lastModified() { + let yesterday = new Date(Date.now() - 86400000); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "title", + dateAdded: yesterday, + }); + checkBookmarkObject(bm); + Assert.deepEqual(bm.lastModified, yesterday); + + let time = new Date(); + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + lastModified: time, + }); + checkBookmarkObject(bm); + Assert.deepEqual(bm.lastModified, time); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.deepEqual(bm.lastModified, time); + + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + lastModified: yesterday, + }); + Assert.deepEqual(bm.lastModified, yesterday); + + bm = await PlacesUtils.bookmarks.update({ guid: bm.guid, title: "title2" }); + Assert.ok(bm.lastModified >= time); + + bm = await PlacesUtils.bookmarks.update({ guid: bm.guid, title: "" }); + Assert.strictEqual(bm.title, ""); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.strictEqual(bm.title, ""); +}); + +add_task(async function update_url() { + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + url: "http://example.com/", + title: "title", + }); + checkBookmarkObject(bm); + let lastModified = bm.lastModified; + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + let frecency = await PlacesTestUtils.getDatabaseValue( + "moz_places", + "frecency", + { url: bm.url } + ); + Assert.greater(frecency, 0, "Check frecency has been updated"); + + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + url: "http://mozilla.org/", + }); + checkBookmarkObject(bm); + Assert.ok(bm.lastModified >= lastModified); + Assert.equal(bm.url.href, "http://mozilla.org/"); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal(bm.url.href, "http://mozilla.org/"); + Assert.ok(bm.lastModified >= lastModified); + + await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(); + Assert.equal( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://example.com/", + }), + frecency, + "Check frecency for example.com" + ); + Assert.equal( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: "http://mozilla.org/", + }), + frecency, + "Check frecency for mozilla.org" + ); +}); + +add_task(async function update_index() { + let parent = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let f1 = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(f1.index, 0); + let f2 = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(f2.index, 1); + let f3 = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(f3.index, 2); + let lastModified = f1.lastModified; + + f1 = await PlacesUtils.bookmarks.update({ + guid: f1.guid, + parentGuid: f1.parentGuid, + index: 1, + }); + checkBookmarkObject(f1); + Assert.equal(f1.index, 1); + Assert.ok(f1.lastModified >= lastModified); + + parent = await PlacesUtils.bookmarks.fetch(f1.parentGuid); + Assert.deepEqual(parent.lastModified, f1.lastModified); + + f2 = await PlacesUtils.bookmarks.fetch(f2.guid); + Assert.equal(f2.index, 0); + + f3 = await PlacesUtils.bookmarks.fetch(f3.guid); + Assert.equal(f3.index, 2); + + f3 = await PlacesUtils.bookmarks.update({ guid: f3.guid, index: 0 }); + f1 = await PlacesUtils.bookmarks.fetch(f1.guid); + Assert.equal(f1.index, 2); + + f2 = await PlacesUtils.bookmarks.fetch(f2.guid); + Assert.equal(f2.index, 1); +}); + +add_task(async function update_move_folder_into_descendant_throws() { + let parent = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let descendant = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + try { + await PlacesUtils.bookmarks.update({ + guid: parent.guid, + parentGuid: parent.guid, + index: 0, + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok( + /Cannot insert a folder into itself or one of its descendants/.test(ex) + ); + } + + try { + await PlacesUtils.bookmarks.update({ + guid: parent.guid, + parentGuid: descendant.guid, + index: 0, + }); + Assert.ok(false, "Should have thrown"); + } catch (ex) { + Assert.ok( + /Cannot insert a folder into itself or one of its descendants/.test(ex) + ); + } +}); + +add_task(async function update_move_into_root_folder_rejects() { + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: "http://example.com/", + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + }); + + Assert.throws( + () => + PlacesUtils.bookmarks.update({ + guid: bm.guid, + index: -1, + parentGuid: PlacesUtils.bookmarks.rootGuid, + }), + /Invalid value for property 'parentGuid'/, + "Should reject when attempting to move a bookmark into the root" + ); + + Assert.throws( + () => + PlacesUtils.bookmarks.update({ + guid: folder.guid, + index: -1, + parentGuid: PlacesUtils.bookmarks.rootGuid, + }), + /Invalid value for property 'parentGuid'/, + "Should reject when attempting to move a bookmark into the root" + ); +}); + +add_task(async function update_move() { + let parent = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + let bm = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + url: "http://example.com/", + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + }); + let descendant = await PlacesUtils.bookmarks.insert({ + parentGuid: parent.guid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(descendant.index, 1); + let lastModified = bm.lastModified; + + // This is moving to a nonexisting index by purpose, it will be appended. + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: descendant.guid, + index: 1, + }); + checkBookmarkObject(bm); + Assert.equal(bm.parentGuid, descendant.guid); + Assert.equal(bm.index, 0); + Assert.ok(bm.lastModified >= lastModified); + + parent = await PlacesUtils.bookmarks.fetch(parent.guid); + descendant = await PlacesUtils.bookmarks.fetch(descendant.guid); + Assert.deepEqual(parent.lastModified, bm.lastModified); + Assert.deepEqual(descendant.lastModified, bm.lastModified); + Assert.equal(descendant.index, 0); + + bm = await PlacesUtils.bookmarks.fetch(bm.guid); + Assert.equal(bm.parentGuid, descendant.guid); + Assert.equal(bm.index, 0); + + bm = await PlacesUtils.bookmarks.update({ + guid: bm.guid, + parentGuid: parent.guid, + index: 0, + }); + Assert.equal(bm.parentGuid, parent.guid); + Assert.equal(bm.index, 0); + + descendant = await PlacesUtils.bookmarks.fetch(descendant.guid); + Assert.equal(descendant.index, 1); +}); + +add_task(async function update_move_append() { + let folder_a = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(folder_a); + let folder_b = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + checkBookmarkObject(folder_b); + + /* folder_a: [sep_1, sep_2, sep_3], folder_b: [] */ + let sep_1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder_a.guid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + checkBookmarkObject(sep_1); + let sep_2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder_a.guid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + checkBookmarkObject(sep_2); + let sep_3 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder_a.guid, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }); + checkBookmarkObject(sep_3); + + function ensurePosition(info, parentGuid, index) { + checkBookmarkObject(info); + Assert.equal(info.parentGuid, parentGuid); + Assert.equal(info.index, index); + } + + // folder_a: [sep_2, sep_3, sep_1], folder_b: [] + sep_1.index = PlacesUtils.bookmarks.DEFAULT_INDEX; + // Note sep_1 includes parentGuid even though we're not moving the item to + // another folder + sep_1 = await PlacesUtils.bookmarks.update(sep_1); + ensurePosition(sep_1, folder_a.guid, 2); + sep_2 = await PlacesUtils.bookmarks.fetch(sep_2.guid); + ensurePosition(sep_2, folder_a.guid, 0); + sep_3 = await PlacesUtils.bookmarks.fetch(sep_3.guid); + ensurePosition(sep_3, folder_a.guid, 1); + sep_1 = await PlacesUtils.bookmarks.fetch(sep_1.guid); + ensurePosition(sep_1, folder_a.guid, 2); + + // folder_a: [sep_2, sep_1], folder_b: [sep_3] + sep_3.index = PlacesUtils.bookmarks.DEFAULT_INDEX; + sep_3.parentGuid = folder_b.guid; + sep_3 = await PlacesUtils.bookmarks.update(sep_3); + ensurePosition(sep_3, folder_b.guid, 0); + sep_2 = await PlacesUtils.bookmarks.fetch(sep_2.guid); + ensurePosition(sep_2, folder_a.guid, 0); + sep_1 = await PlacesUtils.bookmarks.fetch(sep_1.guid); + ensurePosition(sep_1, folder_a.guid, 1); + sep_3 = await PlacesUtils.bookmarks.fetch(sep_3.guid); + ensurePosition(sep_3, folder_b.guid, 0); + + // folder_a: [sep_1], folder_b: [sep_3, sep_2] + sep_2.index = Number.MAX_SAFE_INTEGER; + sep_2.parentGuid = folder_b.guid; + sep_2 = await PlacesUtils.bookmarks.update(sep_2); + ensurePosition(sep_2, folder_b.guid, 1); + sep_1 = await PlacesUtils.bookmarks.fetch(sep_1.guid); + ensurePosition(sep_1, folder_a.guid, 0); + sep_3 = await PlacesUtils.bookmarks.fetch(sep_3.guid); + ensurePosition(sep_3, folder_b.guid, 0); + sep_2 = await PlacesUtils.bookmarks.fetch(sep_2.guid); + ensurePosition(sep_2, folder_b.guid, 1); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_bookmarkstree_cache.js b/toolkit/components/places/tests/bookmarks/test_bookmarkstree_cache.js new file mode 100644 index 0000000000..be6b4ad669 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_bookmarkstree_cache.js @@ -0,0 +1,23 @@ +// Bug 1192692 - promiseBookmarksTree caches items without adding observers to +// invalidate the cache. +add_task(async function boookmarks_tree_cache() { + // Note that for this test to be effective, it needs to use the "old" sync + // bookmarks methods - using, eg, PlacesUtils.bookmarks.insert() doesn't + // demonstrate the problem as it indirectly arranges for the observers to + // be added. + let id = PlacesUtils.bookmarks.insertBookmark( + await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.unfiledGuid), + uri("http://example.com"), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "A title" + ); + + await PlacesUtils.promiseBookmarksTree(); + + PlacesUtils.bookmarks.removeItem(id); + + await Assert.rejects( + PlacesUtils.promiseItemGuid(id), + /no item found for the given itemId/ + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_insertTree_fixupOrSkipInvalidEntries.js b/toolkit/components/places/tests/bookmarks/test_insertTree_fixupOrSkipInvalidEntries.js new file mode 100644 index 0000000000..9a676e0ed0 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_insertTree_fixupOrSkipInvalidEntries.js @@ -0,0 +1,114 @@ +function insertTree(tree) { + return PlacesUtils.bookmarks.insertTree(tree, { + fixupOrSkipInvalidEntries: true, + }); +} + +add_task(async function () { + let guid = PlacesUtils.bookmarks.unfiledGuid; + await Assert.throws( + () => insertTree({ guid, children: [] }), + /Should have a non-zero number of children to insert./ + ); + await Assert.throws( + () => insertTree({ guid: "invalid", children: [{}] }), + /The parent guid is not valid/ + ); + + let now = new Date(); + let url = "http://mozilla.com/"; + let obs = { + count: 0, + lastIndex: 0, + handlePlacesEvent(events) { + for (let event of events) { + obs.count++; + let lastIndex = obs.lastIndex; + obs.lastIndex = event.index; + if (event.itemType == PlacesUtils.bookmarks.TYPE_BOOKMARK) { + Assert.equal(event.url, url, "Found the expected url"); + } + Assert.ok( + event.index == 0 || event.index == lastIndex + 1, + "Consecutive indices" + ); + Assert.ok(event.dateAdded >= now, "Found a valid dateAdded"); + Assert.ok(PlacesUtils.isValidGuid(event.guid), "guid is valid"); + } + }, + }; + PlacesUtils.observers.addListener(["bookmark-added"], obs.handlePlacesEvent); + + let tree = { + guid, + children: [ + { + // Should be inserted, and the invalid guid should be replaced. + guid: "test", + url, + }, + { + // Should be skipped, since the url is invalid. + url: "fake_url", + }, + { + // Should be skipped, since the type is invalid. + url, + type: 999, + }, + { + // Should be skipped, since the type is invalid. + type: 999, + children: [ + { + url, + }, + ], + }, + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "test", + children: [ + { + // Should fix lastModified and dateAdded. + url, + lastModified: null, + }, + { + // Should be skipped, since the url is invalid. + url: "fake_url", + dateAdded: null, + }, + { + // Should fix lastModified and dateAdded. + url, + dateAdded: undefined, + }, + { + // Should be skipped since it's a separator with a url + url, + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + }, + { + // Should fix lastModified and dateAdded. + url, + dateAdded: new Date(now - 86400000), + lastModified: new Date(now - 172800000), // less than dateAdded + }, + ], + }, + ], + }; + + let bms = await insertTree(tree); + for (let bm of bms) { + checkBookmarkObject(bm); + } + Assert.equal(bms.length, 5); + Assert.equal(obs.count, bms.length); + + PlacesUtils.observers.removeListener( + ["bookmark-added"], + obs.handlePlacesEvent + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_keywords.js b/toolkit/components/places/tests/bookmarks/test_keywords.js new file mode 100644 index 0000000000..fca9ebf32a --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_keywords.js @@ -0,0 +1,691 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.defineESModuleGetters(this, { + Preferences: "resource://gre/modules/Preferences.sys.mjs", +}); + +const URI1 = "http://test1.mozilla.org/"; +const URI2 = "http://test2.mozilla.org/"; +const URI3 = "http://test3.mozilla.org/"; + +async function check_keyword(aURI, aKeyword) { + if (aKeyword) { + aKeyword = aKeyword.toLowerCase(); + } + + if (aKeyword) { + let uri = await PlacesUtils.keywords.fetch(aKeyword); + Assert.equal(uri.url, aURI); + // Check case insensitivity. + uri = await PlacesUtils.keywords.fetch(aKeyword.toUpperCase()); + Assert.equal(uri.url, aURI); + } else { + let entry = await PlacesUtils.keywords.fetch({ url: aURI }); + if (entry) { + throw new Error(`${aURI.spec} should not have a keyword`); + } + } +} + +async function check_orphans() { + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + `SELECT id FROM moz_keywords k + WHERE NOT EXISTS (SELECT 1 FROM moz_places WHERE id = k.place_id) + ` + ); + Assert.equal(rows.length, 0); +} + +function expectNotifications() { + const observer = { + notifications: [], + _start() { + this._handle = this._handle.bind(this); + PlacesUtils.observers.addListener( + ["bookmark-keyword-changed"], + this._handle + ); + }, + _handle(events) { + for (const event of events) { + this.notifications.push({ + type: event.type, + id: event.id, + itemType: event.itemType, + url: event.url, + guid: event.guid, + parentGuid: event.parentGuid, + keyword: event.keyword, + lastModified: new Date(event.lastModified), + source: event.source, + isTagging: event.isTagging, + }); + } + }, + check(expected) { + PlacesUtils.observers.removeListener( + ["bookmark-keyword-changed"], + this._handle + ); + Assert.deepEqual(this.notifications, expected); + }, + }; + observer._start(); + return observer; +} + +add_task(function test_invalid_input() {}); + +add_task(async function test_addBookmarkAndKeyword() { + let timerPrecision = Preferences.get("privacy.reduceTimerPrecision"); + Preferences.set("privacy.reduceTimerPrecision", false); + + registerCleanupFunction(function () { + Preferences.set("privacy.reduceTimerPrecision", timerPrecision); + }); + + await check_keyword(URI1, null); + let fc = await foreign_count(URI1); + let observer = expectNotifications(); + + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI1, + title: "test", + }); + await PlacesUtils.keywords.insert({ url: URI1, keyword: "keyword" }); + let itemId = await PlacesUtils.promiseItemId(bookmark.guid); + observer.check([ + { + type: "bookmark-keyword-changed", + id: itemId, + itemType: bookmark.type, + url: bookmark.url, + guid: bookmark.guid, + parentGuid: bookmark.parentGuid, + keyword: "keyword", + lastModified: new Date(bookmark.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + await check_keyword(URI1, "keyword"); + Assert.equal(await foreign_count(URI1), fc + 2); // + 1 bookmark + 1 keyword + await check_orphans(); +}); + +add_task(async function test_addBookmarkToURIHavingKeyword() { + // The uri has already a keyword. + await check_keyword(URI1, "keyword"); + let fc = await foreign_count(URI1); + + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI1, + title: "test", + }); + await check_keyword(URI1, "keyword"); + Assert.equal(await foreign_count(URI1), fc + 1); // + 1 bookmark + await PlacesUtils.bookmarks.remove(bookmark); + await check_orphans(); +}); + +add_task(async function test_sameKeywordDifferentURI() { + let timerPrecision = Preferences.get("privacy.reduceTimerPrecision"); + Preferences.set("privacy.reduceTimerPrecision", false); + + registerCleanupFunction(function () { + Preferences.set("privacy.reduceTimerPrecision", timerPrecision); + }); + + let fc1 = await foreign_count(URI1); + let fc2 = await foreign_count(URI2); + let observer = expectNotifications(); + + let bookmark2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI2, + title: "test2", + }); + await check_keyword(URI1, "keyword"); + await check_keyword(URI2, null); + + await PlacesUtils.keywords.insert({ url: URI2, keyword: "kEyWoRd" }); + + let bookmark1 = await PlacesUtils.bookmarks.fetch({ url: URI1 }); + observer.check([ + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmark1.guid), + itemType: bookmark1.type, + url: bookmark1.url, + guid: bookmark1.guid, + parentGuid: bookmark1.parentGuid, + keyword: "", + lastModified: new Date(bookmark1.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmark2.guid), + itemType: bookmark2.type, + url: bookmark2.url, + guid: bookmark2.guid, + parentGuid: bookmark2.parentGuid, + keyword: "keyword", + lastModified: new Date(bookmark2.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + + // The keyword should have been "moved" to the new URI. + await check_keyword(URI1, null); + Assert.equal(await foreign_count(URI1), fc1 - 1); // - 1 keyword + await check_keyword(URI2, "keyword"); + Assert.equal(await foreign_count(URI2), fc2 + 2); // + 1 bookmark + 1 keyword + await check_orphans(); +}); + +add_task(async function test_sameURIDifferentKeyword() { + let fc = await foreign_count(URI2); + let observer = expectNotifications(); + + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI2, + title: "test2", + }); + await check_keyword(URI2, "keyword"); + + await PlacesUtils.keywords.insert({ url: URI2, keyword: "keyword2" }); + let bookmarks = []; + await PlacesUtils.bookmarks.fetch({ url: URI2 }, bm => bookmarks.push(bm)); + observer.check([ + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmarks[0].guid), + itemType: bookmarks[0].type, + url: bookmarks[0].url, + guid: bookmarks[0].guid, + parentGuid: bookmarks[0].parentGuid, + keyword: "keyword2", + lastModified: new Date(bookmarks[0].lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmarks[1].guid), + itemType: bookmarks[1].type, + url: bookmarks[1].url, + guid: bookmarks[1].guid, + parentGuid: bookmarks[1].parentGuid, + keyword: "keyword2", + lastModified: new Date(bookmarks[1].lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + await check_keyword(URI2, "keyword2"); + Assert.equal(await foreign_count(URI2), fc + 1); // + 1 bookmark - 1 keyword + 1 keyword + await check_orphans(); +}); + +add_task(async function test_removeBookmarkWithKeyword() { + let fc = await foreign_count(URI2); + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI2, + title: "test", + }); + + // The keyword should not be removed, since there are other bookmarks yet. + await PlacesUtils.bookmarks.remove(bookmark); + + await check_keyword(URI2, "keyword2"); + Assert.equal(await foreign_count(URI2), fc); // + 1 bookmark - 1 bookmark + await check_orphans(); +}); + +add_task(async function test_unsetKeyword() { + let fc = await foreign_count(URI2); + let observer = expectNotifications(); + + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI2, + title: "test", + }); + + // The keyword should be removed from any bookmark. + await PlacesUtils.keywords.remove("keyword2"); + + let bookmarks = []; + await PlacesUtils.bookmarks.fetch({ url: URI2 }, bookmark => + bookmarks.push(bookmark) + ); + Assert.equal(bookmarks.length, 3, "Check number of bookmarks"); + observer.check([ + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmarks[0].guid), + itemType: bookmarks[0].type, + url: bookmarks[0].url, + guid: bookmarks[0].guid, + parentGuid: bookmarks[0].parentGuid, + keyword: "", + lastModified: new Date(bookmarks[0].lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmarks[1].guid), + itemType: bookmarks[1].type, + url: bookmarks[1].url, + guid: bookmarks[1].guid, + parentGuid: bookmarks[1].parentGuid, + keyword: "", + lastModified: new Date(bookmarks[1].lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(bookmarks[2].guid), + itemType: bookmarks[2].type, + url: bookmarks[2].url, + guid: bookmarks[2].guid, + parentGuid: bookmarks[2].parentGuid, + keyword: "", + lastModified: new Date(bookmarks[2].lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + + await check_keyword(URI1, null); + await check_keyword(URI2, null); + Assert.equal(await foreign_count(URI2), fc); // + 1 bookmark - 1 keyword + await check_orphans(); +}); + +add_task(async function test_addRemoveBookmark() { + let fc = await foreign_count(URI3); + let observer = expectNotifications(); + + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + url: URI3, + title: "test3", + }); + let itemId = await PlacesUtils.promiseItemId(bookmark.guid); + await PlacesUtils.keywords.insert({ url: URI3, keyword: "keyword" }); + await PlacesUtils.bookmarks.remove(bookmark); + + observer.check([ + { + type: "bookmark-keyword-changed", + id: itemId, + itemType: bookmark.type, + url: bookmark.url, + guid: bookmark.guid, + parentGuid: bookmark.parentGuid, + keyword: "keyword", + lastModified: new Date(bookmark.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + + await check_keyword(URI3, null); + Assert.equal(await foreign_count(URI3), fc); // +- 1 bookmark +- 1 keyword + await check_orphans(); +}); + +add_task(async function test_reassign() { + // Should move keywords from old URL to new URL. + info("Old URL with keywords; new URL without keywords"); + { + let oldURL = "http://example.com/1/kw"; + let oldBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: oldURL, + }); + await PlacesUtils.keywords.insert({ + url: oldURL, + keyword: "kw1-1", + postData: "a=b", + }); + await PlacesUtils.keywords.insert({ + url: oldURL, + keyword: "kw1-2", + postData: "c=d", + }); + let oldFC = await foreign_count(oldURL); + equal(oldFC, 3); + + let newURL = "http://example.com/2/no-kw"; + let newBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: newURL, + }); + let newFC = await foreign_count(newURL); + equal(newFC, 1); + + let observer = expectNotifications(); + await PlacesUtils.keywords.reassign(oldURL, newURL); + observer.check([ + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(oldBmk.guid), + itemType: oldBmk.type, + url: oldBmk.url, + guid: oldBmk.guid, + parentGuid: oldBmk.parentGuid, + keyword: "", + lastModified: new Date(oldBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "kw1-1", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "kw1-2", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + + await check_keyword(oldURL, null); + await check_keyword(newURL, "kw1-1"); + await check_keyword(newURL, "kw1-2"); + + equal(await foreign_count(oldURL), oldFC - 2); // Removed both keywords. + equal(await foreign_count(newURL), newFC + 2); // Added two keywords. + } + + // Should not remove any keywords from new URL. + info("Old URL without keywords; new URL with keywords"); + { + let oldURL = "http://example.com/3/no-kw"; + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: oldURL, + }); + let oldFC = await foreign_count(oldURL); + equal(oldFC, 1); + + let newURL = "http://example.com/4/kw"; + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: newURL, + }); + await PlacesUtils.keywords.insert({ + url: newURL, + keyword: "kw4-1", + }); + let newFC = await foreign_count(newURL); + equal(newFC, 2); + + let observer = expectNotifications(); + await PlacesUtils.keywords.reassign(oldURL, newURL); + observer.check([]); + + await check_keyword(newURL, "kw4-1"); + + equal(await foreign_count(oldURL), oldFC); + equal(await foreign_count(newURL), newFC); + } + + // Should remove all keywords from new URL, then move keywords from old URL. + info("Old URL with keywords; new URL with keywords"); + { + let oldURL = "http://example.com/8/kw"; + let oldBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: oldURL, + }); + await PlacesUtils.keywords.insert({ + url: oldURL, + keyword: "kw8-1", + postData: "a=b", + }); + await PlacesUtils.keywords.insert({ + url: oldURL, + keyword: "kw8-2", + postData: "c=d", + }); + let oldFC = await foreign_count(oldURL); + equal(oldFC, 3); + + let newURL = "http://example.com/9/kw"; + let newBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: newURL, + }); + await PlacesUtils.keywords.insert({ + url: newURL, + keyword: "kw9-1", + }); + let newFC = await foreign_count(newURL); + equal(newFC, 2); + + let observer = expectNotifications(); + await PlacesUtils.keywords.reassign(oldURL, newURL); + observer.check([ + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(oldBmk.guid), + itemType: oldBmk.type, + url: oldBmk.url, + guid: oldBmk.guid, + parentGuid: oldBmk.parentGuid, + keyword: "", + lastModified: new Date(oldBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "kw8-1", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + { + type: "bookmark-keyword-changed", + id: await PlacesUtils.promiseItemId(newBmk.guid), + itemType: newBmk.type, + url: newBmk.url, + guid: newBmk.guid, + parentGuid: newBmk.parentGuid, + keyword: "kw8-2", + lastModified: new Date(newBmk.lastModified), + source: Ci.nsINavBookmarksService.SOURCE_DEFAULT, + isTagging: false, + }, + ]); + + await check_keyword(oldURL, null); + await check_keyword(newURL, "kw8-1"); + await check_keyword(newURL, "kw8-2"); + + equal(await foreign_count(oldURL), oldFC - 2); // Removed both keywords. + equal(await foreign_count(newURL), newFC + 1); // Removed old keyword; added two keywords. + } + + // Should do nothing. + info("Old URL without keywords; new URL without keywords"); + { + let oldURL = "http://example.com/10/no-kw"; + let oldFC = await foreign_count(oldURL); + + let newURL = "http://example.com/11/no-kw"; + let newFC = await foreign_count(newURL); + + let observer = expectNotifications(); + await PlacesUtils.keywords.reassign(oldURL, newURL); + observer.check([]); + + equal(await foreign_count(oldURL), oldFC); + equal(await foreign_count(newURL), newFC); + } + + await check_orphans(); +}); + +add_task(async function test_invalidation() { + info("Insert bookmarks"); + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Get Firefox!", + url: "http://getfirefox.com", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Get Thunderbird!", + url: "http://getthunderbird.com", + }); + + info("Set keywords for bookmarks"); + await PlacesUtils.keywords.insert({ url: fx.url, keyword: "fx" }); + await PlacesUtils.keywords.insert({ url: tb.url, keyword: "tb" }); + + info("Invalidate cached keywords"); + await PlacesUtils.keywords.invalidateCachedKeywords(); + + info("Change URL of bookmark with keyword"); + let promiseNotification = PlacesTestUtils.waitForNotification( + "bookmark-keyword-changed", + events => + events.some(event => event.guid === fx.guid && event.keyword === "fx") + ); + await PlacesUtils.bookmarks.update({ + guid: fx.guid, + url: "https://www.mozilla.org/firefox", + }); + await promiseNotification; + + let entriesByKeyword = []; + await PlacesUtils.keywords.fetch({ keyword: "fx" }, e => + entriesByKeyword.push(e.url.href) + ); + deepEqual( + entriesByKeyword, + ["https://www.mozilla.org/firefox"], + "Should return new URL for keyword" + ); + + ok( + !(await PlacesUtils.keywords.fetch({ url: "http://getfirefox.com" })), + "Should not return keywords for old URL" + ); + + let entiresByURL = []; + await PlacesUtils.keywords.fetch( + { url: "https://www.mozilla.org/firefox" }, + e => entiresByURL.push(e.keyword) + ); + deepEqual(entiresByURL, ["fx"], "Should return keyword for new URL"); + + info("Invalidate cached keywords"); + await PlacesUtils.keywords.invalidateCachedKeywords(); + + info("Remove bookmark with keyword"); + await PlacesUtils.bookmarks.remove(tb.guid); + + ok( + !(await PlacesUtils.keywords.fetch({ url: "http://getthunderbird.com" })), + "Should not return keywords for removed bookmark URL" + ); + + ok( + !(await PlacesUtils.keywords.fetch({ keyword: "tb" })), + "Should not return URL for removed bookmark keyword" + ); + await check_orphans(); + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function test_eraseAllBookmarks() { + info("Insert bookmarks"); + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Get Firefox!", + url: "http://getfirefox.com", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Get Thunderbird!", + url: "http://getthunderbird.com", + }); + + info("Set keywords for bookmarks"); + await PlacesUtils.keywords.insert({ url: fx.url, keyword: "fx" }); + await PlacesUtils.keywords.insert({ url: tb.url, keyword: "tb" }); + + info("Erase everything"); + await PlacesUtils.bookmarks.eraseEverything(); + + ok( + !(await PlacesUtils.keywords.fetch({ keyword: "fx" })), + "Should remove Firefox keyword" + ); + + ok( + !(await PlacesUtils.keywords.fetch({ keyword: "tb" })), + "Should remove Thunderbird keyword" + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_removeFolderTransaction_reinsert.js b/toolkit/components/places/tests/bookmarks/test_removeFolderTransaction_reinsert.js new file mode 100644 index 0000000000..1c9e55849d --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_removeFolderTransaction_reinsert.js @@ -0,0 +1,115 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ +/** + * This test ensures that reinserting a folder within a transaction gives it + * the same GUID, and passes it to the observers. + */ + +add_task(async function test_removeFolderTransaction_reinsert() { + let folder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Test folder", + }); + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + title: "Get Firefox!", + url: "http://getfirefox.com", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + title: "Get Thunderbird!", + url: "http://getthunderbird.com", + }); + + let notifications = []; + function checkNotifications(expected, message) { + deepEqual(notifications, expected, message); + notifications.length = 0; + } + + let listener = events => { + for (let event of events) { + notifications.push([ + event.type, + event.id, + event.parentId, + event.guid, + event.parentGuid, + ]); + } + }; + PlacesUtils.observers.addListener( + ["bookmark-added", "bookmark-removed"], + listener + ); + PlacesUtils.registerShutdownFunction(function () { + PlacesUtils.observers.removeListener( + ["bookmark-added", "bookmark-removed"], + listener + ); + }); + + let transaction = PlacesTransactions.Remove({ guid: folder.guid }); + + let folderId = await PlacesUtils.promiseItemId(folder.guid); + let fxId = await PlacesUtils.promiseItemId(fx.guid); + let tbId = await PlacesUtils.promiseItemId(tb.guid); + + await transaction.transact(); + let bookmarksMenuItemId = await PlacesUtils.promiseItemId( + PlacesUtils.bookmarks.menuGuid + ); + checkNotifications( + [ + ["bookmark-removed", tbId, folderId, tb.guid, folder.guid], + ["bookmark-removed", fxId, folderId, fx.guid, folder.guid], + [ + "bookmark-removed", + folderId, + bookmarksMenuItemId, + folder.guid, + PlacesUtils.bookmarks.menuGuid, + ], + ], + "Executing transaction should remove folder and its descendants" + ); + + await PlacesTransactions.undo(); + + folderId = await PlacesUtils.promiseItemId(folder.guid); + fxId = await PlacesUtils.promiseItemId(fx.guid); + tbId = await PlacesUtils.promiseItemId(tb.guid); + + checkNotifications( + [ + [ + "bookmark-added", + folderId, + bookmarksMenuItemId, + folder.guid, + PlacesUtils.bookmarks.menuGuid, + ], + ["bookmark-added", fxId, folderId, fx.guid, folder.guid], + ["bookmark-added", tbId, folderId, tb.guid, folder.guid], + ], + "Undo should reinsert folder with different id but same GUID" + ); + + await PlacesTransactions.redo(); + + checkNotifications( + [ + ["bookmark-removed", tbId, folderId, tb.guid, folder.guid], + ["bookmark-removed", fxId, folderId, fx.guid, folder.guid], + [ + "bookmark-removed", + folderId, + bookmarksMenuItemId, + folder.guid, + PlacesUtils.bookmarks.menuGuid, + ], + ], + "Redo should pass the GUID to observer" + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_savedsearches.js b/toolkit/components/places/tests/bookmarks/test_savedsearches.js new file mode 100644 index 0000000000..a10307983d --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_savedsearches.js @@ -0,0 +1,224 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// a search term that matches a default bookmark +const searchTerm = "about"; + +var testRoot; + +add_task(async function setup() { + // create a folder to hold all the tests + // this makes the tests more tolerant of changes to the default bookmarks set + // also, name it using the search term, for testing that containers that match don't show up in query results + testRoot = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: searchTerm, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); +}); + +add_task(async function test_savedsearches_bookmarks() { + // add a bookmark that matches the search term + let bookmark = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: searchTerm, + url: "http://foo.com", + }); + + // create a saved-search that matches a default bookmark + let search = await PlacesUtils.bookmarks.insert({ + parentGuid: testRoot.guid, + title: searchTerm, + url: + "place:terms=" + + searchTerm + + "&excludeQueries=1&expandQueries=1&queryType=1", + }); + + // query for the test root, expandQueries=0 + // the query should show up as a regular bookmark + try { + let options = PlacesUtils.history.getNewQueryOptions(); + options.expandQueries = 0; + let query = PlacesUtils.history.getNewQuery(); + query.setParents([testRoot.guid]); + let result = PlacesUtils.history.executeQuery(query, options); + let rootNode = result.root; + rootNode.containerOpen = true; + let cc = rootNode.childCount; + Assert.equal(cc, 1); + for (let i = 0; i < cc; i++) { + let node = rootNode.getChild(i); + // test that queries have valid itemId + Assert.ok(node.itemId > 0); + // test that the container is closed + node.QueryInterface(Ci.nsINavHistoryContainerResultNode); + Assert.equal(node.containerOpen, false); + } + rootNode.containerOpen = false; + } catch (ex) { + do_throw("expandQueries=0 query error: " + ex); + } + + // bookmark saved search + // query for the test root, expandQueries=1 + // the query should show up as a query container, with 1 child + try { + let options = PlacesUtils.history.getNewQueryOptions(); + options.expandQueries = 1; + let query = PlacesUtils.history.getNewQuery(); + query.setParents([testRoot.guid]); + let result = PlacesUtils.history.executeQuery(query, options); + let rootNode = result.root; + rootNode.containerOpen = true; + let cc = rootNode.childCount; + Assert.equal(cc, 1); + for (let i = 0; i < cc; i++) { + let node = rootNode.getChild(i); + // test that query node type is container when expandQueries=1 + Assert.equal(node.type, node.RESULT_TYPE_QUERY); + // test that queries (as containers) have valid itemId + Assert.ok(node.itemId > 0); + node.QueryInterface(Ci.nsINavHistoryContainerResultNode); + node.containerOpen = true; + + // test that queries have children when excludeItems=1 + // test that query nodes don't show containers (shouldn't have our folder that matches) + // test that queries don't show themselves in query results (shouldn't have our saved search) + Assert.equal(node.childCount, 1); + + // test that bookmark shows in query results + var item = node.getChild(0); + Assert.equal(item.bookmarkGuid, bookmark.guid); + + // XXX - FAILING - test live-update of query results - add a bookmark that matches the query + // var tmpBmId = PlacesUtils.bookmarks.insertBookmark( + // root, uri("http://" + searchTerm + ".com"), + // PlacesUtils.bookmarks.DEFAULT_INDEX, searchTerm + "blah"); + // do_check_eq(query.childCount, 2); + + // XXX - test live-update of query results - delete a bookmark that matches the query + // PlacesUtils.bookmarks.removeItem(tmpBMId); + // do_check_eq(query.childCount, 1); + + // test live-update of query results - add a folder that matches the query + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: searchTerm + "zaa", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + Assert.equal(node.childCount, 1); + // test live-update of query results - add a query that matches the query + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: searchTerm + "blah", + url: "place:terms=foo&excludeQueries=1&expandQueries=1&queryType=1", + }); + Assert.equal(node.childCount, 1); + } + rootNode.containerOpen = false; + } catch (ex) { + do_throw("expandQueries=1 bookmarks query: " + ex); + } + + // delete the bookmark search + await PlacesUtils.bookmarks.remove(search); +}); + +add_task(async function test_savedsearches_history() { + // add a visit that matches the search term + var testURI = uri("http://" + searchTerm + ".com"); + await PlacesTestUtils.addVisits({ uri: testURI, title: searchTerm }); + + // create a saved-search that matches the visit we added + var searchItem = await PlacesUtils.bookmarks.insert({ + parentGuid: testRoot.guid, + title: searchTerm, + url: + "place:terms=" + + searchTerm + + "&excludeQueries=1&expandQueries=1&queryType=0", + }); + + // query for the test root, expandQueries=1 + // the query should show up as a query container, with 1 child + try { + var options = PlacesUtils.history.getNewQueryOptions(); + options.expandQueries = 1; + var query = PlacesUtils.history.getNewQuery(); + query.setParents([testRoot.guid]); + var result = PlacesUtils.history.executeQuery(query, options); + var rootNode = result.root; + rootNode.containerOpen = true; + var cc = rootNode.childCount; + Assert.equal(cc, 1); + for (var i = 0; i < cc; i++) { + var node = rootNode.getChild(i); + // test that query node type is container when expandQueries=1 + Assert.equal(node.type, node.RESULT_TYPE_QUERY); + // test that queries (as containers) have valid itemId + Assert.equal(node.bookmarkGuid, searchItem.guid); + node.QueryInterface(Ci.nsINavHistoryContainerResultNode); + node.containerOpen = true; + + // test that queries have children when excludeItems=1 + // test that query nodes don't show containers (shouldn't have our folder that matches) + // test that queries don't show themselves in query results (shouldn't have our saved search) + Assert.equal(node.childCount, 1); + + // test that history visit shows in query results + var item = node.getChild(0); + Assert.equal(item.type, item.RESULT_TYPE_URI); + Assert.equal(item.itemId, -1); // history visit + Assert.equal(item.uri, testURI.spec); // history visit + + // test live-update of query results - add a history visit that matches the query + await PlacesTestUtils.addVisits({ + uri: uri("http://foo.com"), + title: searchTerm + "blah", + }); + Assert.equal(node.childCount, 2); + + // test live-update of query results - delete a history visit that matches the query + await PlacesUtils.history.remove("http://foo.com"); + Assert.equal(node.childCount, 1); + node.containerOpen = false; + } + + // test live-update of moved queries + let tmpFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: testRoot.guid, + title: "foo", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + searchItem.parentGuid = tmpFolder.guid; + await PlacesUtils.bookmarks.update(searchItem); + var tmpFolderNode = rootNode.getChild(0); + Assert.equal(tmpFolderNode.bookmarkGuid, tmpFolder.guid); + tmpFolderNode.QueryInterface(Ci.nsINavHistoryContainerResultNode); + tmpFolderNode.containerOpen = true; + Assert.equal(tmpFolderNode.childCount, 1); + + // test live-update of renamed queries + searchItem.title = "foo"; + await PlacesUtils.bookmarks.update(searchItem); + Assert.equal(tmpFolderNode.title, "foo"); + + // test live-update of deleted queries + await PlacesUtils.bookmarks.remove(searchItem); + Assert.throws( + () => (tmpFolderNode = rootNode.getChild(1)), + /NS_ERROR_ILLEGAL_VALUE/, + "getting a deleted child should throw" + ); + + tmpFolderNode.containerOpen = false; + rootNode.containerOpen = false; + } catch (ex) { + do_throw("expandQueries=1 bookmarks query: " + ex); + } +}); diff --git a/toolkit/components/places/tests/bookmarks/test_sync_fields.js b/toolkit/components/places/tests/bookmarks/test_sync_fields.js new file mode 100644 index 0000000000..7db76e96e6 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_sync_fields.js @@ -0,0 +1,438 @@ +// Tracks a set of bookmark guids and their syncChangeCounter field and +// provides a simple way for the test to check the correct fields had the +// counter incremented. +class CounterTracker { + constructor() { + this.tracked = new Map(); + } + + async _getCounter(guid) { + let fields = await PlacesTestUtils.fetchBookmarkSyncFields(guid); + if (!fields.length) { + throw new Error(`Item ${guid} does not exist`); + } + return fields[0].syncChangeCounter; + } + + // Call this after creating a new bookmark. + async track(guid, name, expectedInitial = 1) { + if (this.tracked.has(guid)) { + throw new Error(`Already tracking item ${guid}`); + } + let initial = await this._getCounter(guid); + Assert.equal( + initial, + expectedInitial, + `Initial value of item '${name}' is correct` + ); + this.tracked.set(guid, { name, value: expectedInitial }); + } + + // Call this to check *only* the specified IDs had a change increment, and + // that none of the other "tracked" ones did. + async check(...expectedToIncrement) { + info(`Checking counter for items ${JSON.stringify(expectedToIncrement)}`); + for (let [guid, entry] of this.tracked) { + let { name, value } = entry; + let newValue = await this._getCounter(guid); + let desc = `record '${name}' (guid=${guid})`; + if (expectedToIncrement.includes(guid)) { + // Note we don't check specifically for +1, as some changes will + // increment the counter by more than 1 (which is OK). + Assert.ok( + newValue > value, + `${desc} was expected to increment - was ${value}, now ${newValue}` + ); + this.tracked.set(guid, { name, value: newValue }); + } else { + Assert.equal(newValue, value, `${desc} was NOT expected to increment`); + } + } + } +} + +async function checkSyncFields(guid, expected) { + let results = await PlacesTestUtils.fetchBookmarkSyncFields(guid); + if (!results.length) { + throw new Error(`Missing sync fields for ${guid}`); + } + for (let name in expected) { + let expectedValue = expected[name]; + Assert.equal( + results[0][name], + expectedValue, + `field ${name} matches item ${guid}` + ); + } +} + +// Common test cases for sync field changes. +class TestCases { + async run() { + info("Test 1: inserts, updates, tags, and keywords"); + try { + await this.testChanges(); + } finally { + info("Reset sync fields after test 1"); + await PlacesTestUtils.markBookmarksAsSynced(); + } + + if ("moveItem" in this && "reorder" in this) { + info("Test 2: reparenting"); + try { + await this.testReparenting(); + } finally { + info("Reset sync fields after test 2"); + await PlacesTestUtils.markBookmarksAsSynced(); + } + } + + if ("insertSeparator" in this) { + info("Test 3: separators"); + try { + await this.testSeparators(); + } finally { + info("Reset sync fields after test 3"); + await PlacesTestUtils.markBookmarksAsSynced(); + } + } + } + + async testChanges() { + let testUri = NetUtil.newURI("http://test.mozilla.org"); + + let guid = await this.insertBookmark( + PlacesUtils.bookmarks.unfiledGuid, + testUri, + PlacesUtils.bookmarks.DEFAULT_INDEX, + "bookmark title" + ); + info(`Inserted bookmark ${guid}`); + await checkSyncFields(guid, { + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, + syncChangeCounter: 1, + }); + + // Pretend Sync just did whatever it does + await PlacesTestUtils.setBookmarkSyncFields({ + guid, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + info(`Updated sync status of ${guid}`); + await checkSyncFields(guid, { + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + syncChangeCounter: 1, + }); + + // update it - it should increment the change counter + await this.setTitle(guid, "new title"); + info(`Changed title of ${guid}`); + await checkSyncFields(guid, { + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + syncChangeCounter: 2, + }); + + // Tagging a bookmark should update its change counter. + await this.tagURI(testUri, ["test-tag"]); + info(`Tagged bookmark ${guid}`); + await checkSyncFields(guid, { syncChangeCounter: 3 }); + + if ("setKeyword" in this) { + await this.setKeyword(guid, "keyword"); + info(`Set keyword for bookmark ${guid}`); + await checkSyncFields(guid, { syncChangeCounter: 4 }); + } + if ("removeKeyword" in this) { + await this.removeKeyword(guid, "keyword"); + info(`Removed keyword from bookmark ${guid}`); + await checkSyncFields(guid, { syncChangeCounter: 5 }); + } + } + + async testSeparators() { + let insertSyncedBookmark = uri => { + return this.insertBookmark( + PlacesUtils.bookmarks.unfiledGuid, + NetUtil.newURI(uri), + PlacesUtils.bookmarks.DEFAULT_INDEX, + "A bookmark name" + ); + }; + + await insertSyncedBookmark("http://foo.bar"); + let secondBmk = await insertSyncedBookmark("http://bar.foo"); + let sepGuid = await this.insertSeparator( + PlacesUtils.bookmarks.unfiledGuid, + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + await insertSyncedBookmark("http://barbar.foo"); + + info("Move a bookmark around the separator"); + await this.moveItem(secondBmk, PlacesUtils.bookmarks.unfiledGuid, 4); + await checkSyncFields(sepGuid, { syncChangeCounter: 2 }); + + info("Move a separator around directly"); + await this.moveItem(sepGuid, PlacesUtils.bookmarks.unfiledGuid, 0); + await checkSyncFields(sepGuid, { syncChangeCounter: 3 }); + } + + async testReparenting() { + let counterTracker = new CounterTracker(); + + let folder1 = await this.createFolder( + PlacesUtils.bookmarks.unfiledGuid, + "folder1", + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + info(`Created the first folder, guid is ${folder1}`); + + // New folder should have a change recorded. + await counterTracker.track(folder1, "folder 1"); + + // Put a new bookmark in the folder. + let testUri = NetUtil.newURI("http://test2.mozilla.org"); + let child1 = await this.insertBookmark( + folder1, + testUri, + PlacesUtils.bookmarks.DEFAULT_INDEX, + "bookmark 1" + ); + info(`Created a new bookmark into ${folder1}, guid is ${child1}`); + // both the folder and the child should have a change recorded. + await counterTracker.track(child1, "child 1"); + await counterTracker.check(folder1); + + // A new child in the folder at index 0 - even though the existing child + // was bumped down the list, it should *not* have a change recorded. + let child2 = await this.insertBookmark(folder1, testUri, 0, "bookmark 2"); + info( + `Created a second new bookmark into folder ${folder1}, guid is ${child2}` + ); + + await counterTracker.track(child2, "child 2"); + await counterTracker.check(folder1); + + // Move the items within the same folder - this should result in just a + // change for the parent, but for neither of the children. + // child0 is currently at index 0, so move child1 there. + await this.moveItem(child1, folder1, 0); + await counterTracker.check(folder1); + + // Another folder to play with. + let folder2 = await this.createFolder( + PlacesUtils.bookmarks.unfiledGuid, + "folder2", + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + info(`Created a second new folder, guid is ${folder2}`); + await counterTracker.track(folder2, "folder 2"); + // nothing else has changed. + await counterTracker.check(); + + // Move one of the children to the new folder. + info( + `Moving bookmark ${child2} from folder ${folder1} to folder ${folder2}` + ); + await this.moveItem(child2, folder2, PlacesUtils.bookmarks.DEFAULT_INDEX); + // child1 should have no change, everything should have a new change. + await counterTracker.check(folder1, folder2, child2); + + // Move the new folder to another root. + await this.moveItem( + folder2, + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + info(`Moving folder ${folder2} to toolbar`); + await counterTracker.check( + folder2, + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.unfiledGuid + ); + + let child3 = await this.insertBookmark(folder2, testUri, 0, "bookmark 3"); + info(`Prepended child ${child3} to folder ${folder2}`); + await counterTracker.check(folder2, child3); + + // Reordering should only track the parent. + await this.reorder(folder2, [child2, child3]); + info(`Reorder children of ${folder2}`); + await counterTracker.check(folder2); + + // All fields still have a syncStatus of SYNC_STATUS_NEW - so deleting them + // should *not* cause any deleted items to be written. + await this.removeItem(folder1); + Assert.equal((await PlacesTestUtils.fetchSyncTombstones()).length, 0); + + // Set folder2 and child2 to have a state of SYNC_STATUS_NORMAL and deleting + // them will cause both GUIDs to be written to moz_bookmarks_deleted. + await PlacesTestUtils.setBookmarkSyncFields({ + guid: folder2, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + await PlacesTestUtils.setBookmarkSyncFields({ + guid: child2, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + await this.removeItem(folder2); + let tombstones = await PlacesTestUtils.fetchSyncTombstones(); + let tombstoneGuids = sortBy(tombstones, "guid").map(({ guid }) => guid); + Assert.equal(tombstoneGuids.length, 2); + Assert.deepEqual(tombstoneGuids, [folder2, child2].sort(compareAscending)); + } +} + +// Exercises the legacy, synchronous `nsINavBookmarksService` calls implemented +// in C++. +class SyncTestCases extends TestCases { + async createFolder(parentGuid, title, index) { + let parentId = await PlacesUtils.promiseItemId(parentGuid); + let id = PlacesUtils.bookmarks.createFolder(parentId, title, index); + return PlacesUtils.promiseItemGuid(id); + } + + async insertBookmark(parentGuid, uri, index, title) { + let parentId = await PlacesUtils.promiseItemId(parentGuid); + let id = PlacesUtils.bookmarks.insertBookmark(parentId, uri, index, title); + return PlacesUtils.promiseItemGuid(id); + } + + async removeItem(guid) { + let id = await PlacesUtils.promiseItemId(guid); + PlacesUtils.bookmarks.removeItem(id); + } + + async setTitle(guid, title) { + let id = await PlacesUtils.promiseItemId(guid); + PlacesUtils.bookmarks.setItemTitle(id, title); + } + + async tagURI(uri, tags) { + PlacesUtils.tagging.tagURI(uri, tags); + } +} + +async function findTagFolder(tag) { + let db = await PlacesUtils.promiseDBConnection(); + let results = await db.executeCached( + ` + SELECT guid + FROM moz_bookmarks + WHERE type = :type AND + parent = :tagsFolderId AND + title = :tag`, + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + tagsFolderId: PlacesUtils.tagsFolderId, + tag, + } + ); + return results.length ? results[0].getResultByName("guid") : null; +} + +// Exercises the new, async calls implemented in `Bookmarks.jsm`. +class AsyncTestCases extends TestCases { + async createFolder(parentGuid, title, index) { + let item = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid, + title, + index, + }); + return item.guid; + } + + async insertBookmark(parentGuid, uri, index, title) { + let item = await PlacesUtils.bookmarks.insert({ + parentGuid, + url: uri, + index, + title, + }); + return item.guid; + } + + async insertSeparator(parentGuid, index) { + let item = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid, + index, + }); + return item.guid; + } + + async moveItem(guid, newParentGuid, index) { + await PlacesUtils.bookmarks.update({ + guid, + parentGuid: newParentGuid, + index, + }); + } + + async removeItem(guid) { + await PlacesUtils.bookmarks.remove(guid); + } + + async setTitle(guid, title) { + await PlacesUtils.bookmarks.update({ guid, title }); + } + + async setKeyword(guid, keyword) { + let item = await PlacesUtils.bookmarks.fetch(guid); + if (!item) { + throw new Error( + `Cannot set keyword ${keyword} on nonexistent bookmark ${guid}` + ); + } + await PlacesUtils.keywords.insert({ keyword, url: item.url }); + } + + async removeKeyword(guid, keyword) { + let item = await PlacesUtils.bookmarks.fetch(guid); + if (!item) { + throw new Error( + `Cannot remove keyword ${keyword} from nonexistent bookmark ${guid}` + ); + } + let entry = await PlacesUtils.keywords.fetch({ keyword, url: item.url }); + if (!entry) { + throw new Error(`Keyword ${keyword} not set on bookmark ${guid}`); + } + await PlacesUtils.keywords.remove(entry); + } + + // There's no async API for tags, but the `PlacesUtils.bookmarks` methods are + // tag-aware, and should bump the change counters for tagged bookmarks when + // called directly. + async tagURI(uri, tags) { + for (let tag of tags) { + let tagFolderGuid = await findTagFolder(tag); + if (!tagFolderGuid) { + let tagFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: tag, + }); + tagFolderGuid = tagFolder.guid; + } + await PlacesUtils.bookmarks.insert({ + url: uri, + parentGuid: tagFolderGuid, + }); + } + } + + async reorder(parentGuid, childGuids) { + await PlacesUtils.bookmarks.reorder(parentGuid, childGuids); + } +} + +add_task(async function test_sync_api() { + let tests = new SyncTestCases(); + await tests.run(); +}); + +add_task(async function test_async_api() { + let tests = new AsyncTestCases(); + await tests.run(); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_tags.js b/toolkit/components/places/tests/bookmarks/test_tags.js new file mode 100644 index 0000000000..3334d09b7c --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_tags.js @@ -0,0 +1,128 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_task(async function test_fetchTags() { + let tags = await PlacesUtils.bookmarks.fetchTags(); + Assert.deepEqual(tags, []); + + let bm = await PlacesUtils.bookmarks.insert({ + url: "http://page1.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + + PlacesUtils.tagging.tagURI(bm.url.URI, ["1", "2"]); + tags = await PlacesUtils.bookmarks.fetchTags(); + Assert.deepEqual(tags, [ + { name: "1", count: 1 }, + { name: "2", count: 1 }, + ]); + + PlacesUtils.tagging.untagURI(bm.url.URI, ["1"]); + tags = await PlacesUtils.bookmarks.fetchTags(); + Assert.deepEqual(tags, [{ name: "2", count: 1 }]); + + let bm2 = await PlacesUtils.bookmarks.insert({ + url: "http://page2.com/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + PlacesUtils.tagging.tagURI(bm2.url.URI, ["2", "3"]); + tags = await PlacesUtils.bookmarks.fetchTags(); + Assert.deepEqual(tags, [ + { name: "2", count: 2 }, + { name: "3", count: 1 }, + ]); +}); + +add_task(async function test_fetch_by_tags() { + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ tags: "" }), + /Invalid value for property 'tags'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ tags: [] }), + /Invalid value for property 'tags'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ tags: null }), + /Invalid value for property 'tags'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ tags: [""] }), + /Invalid value for property 'tags'/ + ); + Assert.throws( + () => PlacesUtils.bookmarks.fetch({ tags: ["valid", null] }), + /Invalid value for property 'tags'/ + ); + + info("Add bookmarks with tags."); + let bm1 = await PlacesUtils.bookmarks.insert({ + url: "http://bacon.org/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + PlacesUtils.tagging.tagURI(bm1.url.URI, ["egg", "ratafià"]); + let bm2 = await PlacesUtils.bookmarks.insert({ + url: "http://mushroom.org/", + parentGuid: PlacesUtils.bookmarks.unfiledGuid, + }); + PlacesUtils.tagging.tagURI(bm2.url.URI, ["egg"]); + + info("Fetch a single tag."); + let bms = []; + Assert.equal( + (await PlacesUtils.bookmarks.fetch({ tags: ["egg"] }, b => bms.push(b))) + .guid, + bm2.guid, + "Found the expected recent bookmark" + ); + Assert.deepEqual( + bms.map(b => b.guid), + [bm2.guid, bm1.guid], + "Found the expected bookmarks" + ); + + info("Fetch multiple tags."); + bms = []; + Assert.equal( + ( + await PlacesUtils.bookmarks.fetch({ tags: ["egg", "ratafià"] }, b => + bms.push(b) + ) + ).guid, + bm1.guid, + "Found the expected recent bookmark" + ); + Assert.deepEqual( + bms.map(b => b.guid), + [bm1.guid], + "Found the expected bookmarks" + ); + + info("Fetch a nonexisting tag."); + bms = []; + Assert.equal( + await PlacesUtils.bookmarks.fetch({ tags: ["egg", "tomato"] }, b => + bms.push(b) + ), + null, + "Should not find any bookmark" + ); + Assert.deepEqual(bms, [], "Should not find any bookmark"); + + info("Check case insensitive"); + bms = []; + Assert.equal( + ( + await PlacesUtils.bookmarks.fetch({ tags: ["eGg", "raTafiÀ"] }, b => + bms.push(b) + ) + ).guid, + bm1.guid, + "Found the expected recent bookmark" + ); + Assert.deepEqual( + bms.map(b => b.guid), + [bm1.guid], + "Found the expected bookmarks" + ); +}); diff --git a/toolkit/components/places/tests/bookmarks/test_untitled.js b/toolkit/components/places/tests/bookmarks/test_untitled.js new file mode 100644 index 0000000000..6e756d79a6 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/test_untitled.js @@ -0,0 +1,114 @@ +add_task(async function test_untitled_visited_bookmark() { + let fxURI = uri("http://getfirefox.com"); + + await PlacesUtils.history.insert({ + url: fxURI, + title: "Get Firefox!", + visits: [ + { + date: new Date(), + transition: PlacesUtils.history.TRANSITIONS.TYPED, + }, + ], + }); + + let { root: node } = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid + ); + + try { + let fxBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: fxURI, + }); + strictEqual(fxBmk.title, "", "Visited bookmark should not have title"); + + await PlacesTestUtils.promiseAsyncUpdates(); + + let fxBmkId = await PlacesUtils.promiseItemId(fxBmk.guid); + strictEqual( + PlacesUtils.bookmarks.getItemTitle(fxBmkId), + "", + "Should return empty string for untitled visited bookmark" + ); + + let fxBmkNode = node.getChild(0); + equal(fxBmkNode.itemId, fxBmkId, "Visited bookmark ID should match"); + strictEqual( + fxBmkNode.title, + "", + "Visited bookmark node should not have title" + ); + } finally { + node.containerOpen = false; + } + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function test_untitled_unvisited_bookmark() { + let tbURI = uri("http://getthunderbird.com"); + + let { root: node } = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid + ); + + try { + let tbBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: tbURI, + }); + strictEqual(tbBmk.title, "", "Unvisited bookmark should not have title"); + + await PlacesTestUtils.promiseAsyncUpdates(); + + let tbBmkId = await PlacesUtils.promiseItemId(tbBmk.guid); + strictEqual( + PlacesUtils.bookmarks.getItemTitle(tbBmkId), + "", + "Should return empty string for untitled unvisited bookmark" + ); + + let tbBmkNode = node.getChild(0); + equal(tbBmkNode.itemId, tbBmkId, "Unvisited bookmark ID should match"); + strictEqual( + tbBmkNode.title, + "", + "Unvisited bookmark node should not have title" + ); + } finally { + node.containerOpen = false; + } + + await PlacesUtils.bookmarks.eraseEverything(); +}); + +add_task(async function test_untitled_folder() { + let { root: node } = PlacesUtils.getFolderContents( + PlacesUtils.bookmarks.toolbarGuid + ); + + try { + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + await PlacesTestUtils.promiseAsyncUpdates(); + + let folderId = await PlacesUtils.promiseItemId(folder.guid); + strictEqual( + PlacesUtils.bookmarks.getItemTitle(folderId), + "", + "Should return empty string for untitled folder" + ); + + let folderNode = node.getChild(0); + equal(folderNode.itemId, folderId, "Folder ID should match"); + strictEqual(folderNode.title, "", "Folder node should not have title"); + } finally { + node.containerOpen = false; + } + + await PlacesUtils.bookmarks.eraseEverything(); +}); diff --git a/toolkit/components/places/tests/bookmarks/xpcshell.ini b/toolkit/components/places/tests/bookmarks/xpcshell.ini new file mode 100644 index 0000000000..91d3741bb4 --- /dev/null +++ b/toolkit/components/places/tests/bookmarks/xpcshell.ini @@ -0,0 +1,48 @@ +[DEFAULT] +head = head_bookmarks.js +skip-if = toolkit == 'android' +firefox-appdir = browser + +[test_1016953-renaming-uncompressed.js] +[test_1017502-bookmarks_foreign_count.js] +[test_384228.js] +[test_385829.js] +[test_388695.js] +[test_393498.js] +[test_405938_restore_queries.js] +[test_424958-json-quoted-folders.js] +[test_448584.js] +[test_458683.js] +[test_466303-json-remove-backups.js] +[test_477583_json-backup-in-future.js] +[test_818584-discard-duplicate-backups.js] +[test_818587_compress-bookmarks-backups.js] +[test_818593-store-backup-metadata.js] +[test_992901-backup-unsorted-hierarchy.js] +[test_997030-bookmarks-html-encode.js] +[test_1129529.js] +support-files = + bookmarks_long_tag.json +[test_async_observers.js] +[test_bmindex.js] +[test_bookmark_observer.js] +[test_bookmarkstree_cache.js] +[test_bookmarks_eraseEverything.js] +[test_bookmarks_fetch.js] +[test_bookmarks_getRecent.js] +[test_bookmarks_insert.js] +[test_bookmarks_insertTree.js] +[test_bookmarks_notifications.js] +[test_bookmarks_moveToFolder.js] +[test_bookmarks_remove.js] +[test_bookmarks_remove_batch.js] +[test_bookmarks_reorder.js] +[test_bookmarks_search.js] +[test_bookmarks_update.js] +[test_insertTree_fixupOrSkipInvalidEntries.js] +[test_keywords.js] +[test_removeFolderTransaction_reinsert.js] +[test_savedsearches.js] +[test_sync_fields.js] +[test_tags.js] +[test_untitled.js] diff --git a/toolkit/components/places/tests/browser/1601563-1.html b/toolkit/components/places/tests/browser/1601563-1.html new file mode 100644 index 0000000000..4f92778561 --- /dev/null +++ b/toolkit/components/places/tests/browser/1601563-1.html @@ -0,0 +1,20 @@ + +First title + + diff --git a/toolkit/components/places/tests/browser/1601563-2.html b/toolkit/components/places/tests/browser/1601563-2.html new file mode 100644 index 0000000000..b1c000cd5a --- /dev/null +++ b/toolkit/components/places/tests/browser/1601563-2.html @@ -0,0 +1,3 @@ + +Second title +Nothing to see here. diff --git a/toolkit/components/places/tests/browser/399606-history.go-0.html b/toolkit/components/places/tests/browser/399606-history.go-0.html new file mode 100644 index 0000000000..6e36aa23de --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-history.go-0.html @@ -0,0 +1,13 @@ + + +history.go(0) + + + +Testing history.go(0) + + diff --git a/toolkit/components/places/tests/browser/399606-httprefresh.html b/toolkit/components/places/tests/browser/399606-httprefresh.html new file mode 100644 index 0000000000..e43455ee05 --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-httprefresh.html @@ -0,0 +1,8 @@ + + + + +httprefresh + +Testing httprefresh + diff --git a/toolkit/components/places/tests/browser/399606-location.reload.html b/toolkit/components/places/tests/browser/399606-location.reload.html new file mode 100644 index 0000000000..54eefab1c3 --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-location.reload.html @@ -0,0 +1,13 @@ + + +location.reload() + + + +Testing location.reload(); + + diff --git a/toolkit/components/places/tests/browser/399606-location.replace.html b/toolkit/components/places/tests/browser/399606-location.replace.html new file mode 100644 index 0000000000..8a72c96722 --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-location.replace.html @@ -0,0 +1,13 @@ + + +location.replace + + + +Testing location.replace + + diff --git a/toolkit/components/places/tests/browser/399606-window.location.href.html b/toolkit/components/places/tests/browser/399606-window.location.href.html new file mode 100644 index 0000000000..490b08e40c --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-window.location.href.html @@ -0,0 +1,14 @@ + + +window.location.href + + + +Testing window.location.href + + diff --git a/toolkit/components/places/tests/browser/399606-window.location.html b/toolkit/components/places/tests/browser/399606-window.location.html new file mode 100644 index 0000000000..b84366cf8d --- /dev/null +++ b/toolkit/components/places/tests/browser/399606-window.location.html @@ -0,0 +1,14 @@ + + +window.location + + + +Testing window.location + + diff --git a/toolkit/components/places/tests/browser/461710_link_page-2.html b/toolkit/components/places/tests/browser/461710_link_page-2.html new file mode 100644 index 0000000000..726373f83e --- /dev/null +++ b/toolkit/components/places/tests/browser/461710_link_page-2.html @@ -0,0 +1,13 @@ + + + + Link page 2 + + + +

      Link to the second visited page

      + + diff --git a/toolkit/components/places/tests/browser/461710_link_page-3.html b/toolkit/components/places/tests/browser/461710_link_page-3.html new file mode 100644 index 0000000000..d465cf79c6 --- /dev/null +++ b/toolkit/components/places/tests/browser/461710_link_page-3.html @@ -0,0 +1,13 @@ + + + + Link page 3 + + + +

      Link to the third visited page

      + + diff --git a/toolkit/components/places/tests/browser/461710_link_page.html b/toolkit/components/places/tests/browser/461710_link_page.html new file mode 100644 index 0000000000..05189fa41b --- /dev/null +++ b/toolkit/components/places/tests/browser/461710_link_page.html @@ -0,0 +1,13 @@ + + + + Link page + + + +

      Link to the visited page

      + + diff --git a/toolkit/components/places/tests/browser/461710_visited_page.html b/toolkit/components/places/tests/browser/461710_visited_page.html new file mode 100644 index 0000000000..3ff52f69c8 --- /dev/null +++ b/toolkit/components/places/tests/browser/461710_visited_page.html @@ -0,0 +1,9 @@ + + + + Visited page + + +

      This page is marked as visited

      + + diff --git a/toolkit/components/places/tests/browser/begin.html b/toolkit/components/places/tests/browser/begin.html new file mode 100644 index 0000000000..da4c16dd25 --- /dev/null +++ b/toolkit/components/places/tests/browser/begin.html @@ -0,0 +1,10 @@ + + + + + Redirect twice + + diff --git a/toolkit/components/places/tests/browser/browser.ini b/toolkit/components/places/tests/browser/browser.ini new file mode 100644 index 0000000000..33526ebb20 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser.ini @@ -0,0 +1,88 @@ +[DEFAULT] +support-files = + head.js + +[browser_bug399606.js] +https_first_disabled = true +support-files = + 399606-history.go-0.html + 399606-httprefresh.html + 399606-location.reload.html + 399606-location.replace.html + 399606-window.location.html + 399606-window.location.href.html +[browser_bug461710.js] +https_first_disabled = true +support-files = + 461710_link_page-2.html + 461710_link_page-3.html + 461710_link_page.html + 461710_visited_page.html +[browser_bug646422.js] +https_first_disabled = true +[browser_bug680727.js] +https_first_disabled = true +skip-if = verify +[browser_bug1601563.js] +https_first_disabled = true +support-files = + 1601563-1.html + 1601563-2.html +[browser_double_redirect.js] +https_first_disabled = true +support-files = + begin.html + final.html + redirect_once.sjs + redirect_twice.sjs +[browser_favicon_privatebrowsing_perwindowpb.js] +[browser_history_post.js] +https_first_disabled = true +support-files = + history_post.html + history_post.sjs +[browser_notfound.js] +[browser_onvisit_title_null_for_navigation.js] +https_first_disabled = true +skip-if = verify +support-files = + empty_page.html +[browser_redirect.js] +support-files = + redirect.sjs + redirect-target.html +[browser_redirect_self.js] +support-files = + redirect_self.sjs +[browser_multi_redirect_frecency.js] +https_first_disabled = true +support-files = + final.html + redirect_once.sjs + redirect_thrice.sjs + redirect_twice.sjs + redirect_twice_perma.sjs +[browser_settitle.js] +https_first_disabled = true +support-files = + title1.html + title2.html +[browser_visited_notfound.js] +[browser_visituri.js] +https_first_disabled = true +support-files = + begin.html + final.html + redirect_once.sjs + redirect_twice.sjs +[browser_visituri_nohistory.js] +support-files = + begin.html + final.html + favicon-normal16.png + favicon-normal32.png +[browser_visituri_privatebrowsing_perwindowpb.js] +support-files = + begin.html + favicon.html + final.html diff --git a/toolkit/components/places/tests/browser/browser_bug1601563.js b/toolkit/components/places/tests/browser/browser_bug1601563.js new file mode 100644 index 0000000000..41e278ee54 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_bug1601563.js @@ -0,0 +1,40 @@ +const PREFIX = + "http://example.com/tests/toolkit/components/places/tests/browser/1601563"; + +function titleUpdate(pageUrl) { + let lastTitle = null; + return PlacesTestUtils.waitForNotification("page-title-changed", events => { + if (pageUrl != events[0].url) { + return false; + } + lastTitle = events[0].title; + return true; + }).then(() => { + return lastTitle; + }); +} + +add_task(async function () { + registerCleanupFunction(PlacesUtils.history.clear); + const FIRST_URL = PREFIX + "-1.html"; + const SECOND_URL = PREFIX + "-2.html"; + let firstTitlePromise = titleUpdate(FIRST_URL); + let secondTitlePromise = titleUpdate(SECOND_URL); + + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, FIRST_URL); + + let firstTitle = await firstTitlePromise; + is(firstTitle, "First title", "First title should match the page"); + + let secondTitle = await secondTitlePromise; + is(secondTitle, "Second title", "Second title should match the page"); + + let entry = await PlacesUtils.history.fetch(FIRST_URL); + is( + entry.title, + firstTitle, + "Should not override first title with document.open()ed frame" + ); + + await BrowserTestUtils.removeTab(tab); +}); diff --git a/toolkit/components/places/tests/browser/browser_bug399606.js b/toolkit/components/places/tests/browser/browser_bug399606.js new file mode 100644 index 0000000000..f593d68528 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_bug399606.js @@ -0,0 +1,50 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function () { + registerCleanupFunction(PlacesUtils.history.clear); + + const URIS = [ + "http://example.com/tests/toolkit/components/places/tests/browser/399606-window.location.href.html", + "http://example.com/tests/toolkit/components/places/tests/browser/399606-history.go-0.html", + "http://example.com/tests/toolkit/components/places/tests/browser/399606-location.replace.html", + "http://example.com/tests/toolkit/components/places/tests/browser/399606-location.reload.html", + "http://example.com/tests/toolkit/components/places/tests/browser/399606-httprefresh.html", + "http://example.com/tests/toolkit/components/places/tests/browser/399606-window.location.html", + ]; + + // Create and add history observer. + let count = 0; + let expectedURI = null; + function onVisitsListener(aEvents) { + for (let event of aEvents) { + info("Received onVisits: " + event.url); + if (event.url == expectedURI) { + count++; + } + } + } + + async function promiseLoadedThreeTimes(uri) { + count = 0; + expectedURI = uri; + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser); + PlacesObservers.addListener(["page-visited"], onVisitsListener); + BrowserTestUtils.loadURIString(gBrowser, uri); + await BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser, false, uri); + await BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser, false, uri); + await BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser, false, uri); + PlacesObservers.removeListener(["page-visited"], onVisitsListener); + BrowserTestUtils.removeTab(tab); + } + + for (let uri of URIS) { + await promiseLoadedThreeTimes(uri); + is( + count, + 1, + "'page-visited' has been received right number of times for " + uri + ); + } +}); diff --git a/toolkit/components/places/tests/browser/browser_bug461710.js b/toolkit/components/places/tests/browser/browser_bug461710.js new file mode 100644 index 0000000000..6815860929 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_bug461710.js @@ -0,0 +1,89 @@ +const kRed = "rgb(255, 0, 0)"; +const kBlue = "rgb(0, 0, 255)"; + +const prefix = + "http://example.com/tests/toolkit/components/places/tests/browser/461710_"; + +add_task(async function () { + registerCleanupFunction(PlacesUtils.history.clear); + let normalWindow = await BrowserTestUtils.openNewBrowserWindow(); + let privateWindow = await BrowserTestUtils.openNewBrowserWindow({ + private: true, + }); + let tests = [ + { + private: false, + topic: "uri-visit-saved", + subtest: "visited_page.html", + }, + { + private: false, + subtest: "link_page.html", + color: kRed, + message: "Visited link coloring should work outside of private mode", + }, + { + private: true, + subtest: "link_page-2.html", + color: kBlue, + message: "Visited link coloring should not work inside of private mode", + }, + { + private: false, + subtest: "link_page-3.html", + color: kRed, + message: "Visited link coloring should work outside of private mode", + }, + ]; + + let uri = Services.io.newURI(prefix + tests[0].subtest); + for (let test of tests) { + info(test.subtest); + let promise = null; + if (test.topic) { + promise = TestUtils.topicObserved(test.topic, subject => + uri.equals(subject.QueryInterface(Ci.nsIURI)) + ); + } + await BrowserTestUtils.withNewTab( + { + gBrowser: test.private ? privateWindow.gBrowser : normalWindow.gBrowser, + url: prefix + test.subtest, + }, + async function (browser) { + if (promise) { + await promise; + } + + if (test.color) { + // In e10s waiting for visited-status-resolution is not enough to ensure links + // have been updated, because it only tells us that messages to update links + // have been dispatched. We must still wait for the actual links to update. + await TestUtils.waitForCondition(async function () { + let color = await SpecialPowers.spawn( + browser, + [], + async function () { + let elem = content.document.getElementById("link"); + return content.windowUtils.getVisitedDependentComputedStyle( + elem, + "", + "color" + ); + } + ); + return color == test.color; + }, test.message); + // The harness will consider the test as failed overall if there were no + // passes or failures, so record it as a pass. + ok(true, test.message); + } + } + ); + } + + let promisePBExit = TestUtils.topicObserved("last-pb-context-exited"); + await BrowserTestUtils.closeWindow(privateWindow); + await promisePBExit; + await BrowserTestUtils.closeWindow(normalWindow); +}); diff --git a/toolkit/components/places/tests/browser/browser_bug646422.js b/toolkit/components/places/tests/browser/browser_bug646422.js new file mode 100644 index 0000000000..cb6512ed4e --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_bug646422.js @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test for Bug 646224. Make sure that after changing the URI via + * history.pushState, the history service has a title stored for the new URI. + **/ + +add_task(async function () { + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + "http://example.com" + ); + + const newTitlePromise = PlacesTestUtils.waitForNotification( + "page-title-changed", + events => /new_page$/.test(events[0].url) + ); + + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { + let title = content.document.title; + content.history.pushState("", "", "new_page"); + Assert.ok(title, "Content window should initially have a title."); + }); + + const events = await newTitlePromise; + const newtitle = events[0].title; + + await SpecialPowers.spawn( + tab.linkedBrowser, + [{ newtitle }], + async function (args) { + Assert.equal( + args.newtitle, + content.document.title, + "Title after pushstate." + ); + } + ); + + await PlacesUtils.history.clear(); + gBrowser.removeTab(tab); +}); diff --git a/toolkit/components/places/tests/browser/browser_bug680727.js b/toolkit/components/places/tests/browser/browser_bug680727.js new file mode 100644 index 0000000000..2fe2377d34 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_bug680727.js @@ -0,0 +1,127 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Ensure that clicking the button in the Offline mode neterror page updates + global history. See bug 680727. */ +/* TEST_PATH=toolkit/components/places/tests/browser/browser_bug680727.js make -C $(OBJDIR) mochitest-browser-chrome */ + +const kUniqueURI = Services.io.newURI("http://mochi.test:8888/#bug_680727"); +var proxyPrefValue; +var ourTab; + +function test() { + waitForExplicitFinish(); + + // Tests always connect to localhost, and per bug 87717, localhost is now + // reachable in offline mode. To avoid this, disable any proxy. + proxyPrefValue = Services.prefs.getIntPref("network.proxy.type"); + Services.prefs.setIntPref("network.proxy.type", 0); + + // Clear network cache. + Services.cache2.clear(); + + // Go offline, expecting the error page. + Services.io.offline = true; + + BrowserTestUtils.openNewForegroundTab(gBrowser).then(tab => { + ourTab = tab; + BrowserTestUtils.browserLoaded( + ourTab.linkedBrowser, + false, + null, + true + ).then(errorListener); + BrowserTestUtils.loadURIString(ourTab.linkedBrowser, kUniqueURI.spec); + }); +} + +// ------------------------------------------------------------------------------ +// listen to loading the neterror page. (offline mode) +function errorListener() { + ok(Services.io.offline, "Services.io.offline is true."); + + // This is an error page. + SpecialPowers.spawn(ourTab.linkedBrowser, [kUniqueURI.spec], function (uri) { + Assert.equal( + content.document.documentURI.substring(0, 27), + "about:neterror?e=netOffline", + "Document URI is the error page." + ); + + // But location bar should show the original request. + Assert.equal( + content.location.href, + uri, + "Docshell URI is the original URI." + ); + }).then(() => { + // Global history does not record URI of a failed request. + PlacesTestUtils.promiseAsyncUpdates().then(() => { + PlacesUtils.history.hasVisits(kUniqueURI).then(isVisited => { + errorAsyncListener(kUniqueURI, isVisited); + }); + }); + }); +} + +function errorAsyncListener(aURI, aIsVisited) { + ok( + kUniqueURI.equals(aURI) && !aIsVisited, + "The neterror page is not listed in global history." + ); + + Services.prefs.setIntPref("network.proxy.type", proxyPrefValue); + + // Now press the "Try Again" button, with offline mode off. + Services.io.offline = false; + + BrowserTestUtils.browserLoaded(ourTab.linkedBrowser, false, null, true).then( + reloadListener + ); + + SpecialPowers.spawn(ourTab.linkedBrowser, [], function () { + Assert.ok( + content.document.querySelector("#netErrorButtonContainer > .try-again"), + "The error page has got a .try-again element" + ); + content.document + .querySelector("#netErrorButtonContainer > .try-again") + .click(); + }); +} + +// ------------------------------------------------------------------------------ +// listen to reload of neterror. +function reloadListener() { + // This listener catches "DOMContentLoaded" on being called + // nsIWPL::onLocationChange(...). That is right *AFTER* + // IHistory::VisitURI(...) is called. + ok(!Services.io.offline, "Services.io.offline is false."); + + SpecialPowers.spawn(ourTab.linkedBrowser, [kUniqueURI.spec], function (uri) { + // This is not an error page. + Assert.equal( + content.document.documentURI, + uri, + "Document URI is not the offline-error page, but the original URI." + ); + }).then(() => { + // Check if global history remembers the successfully-requested URI. + PlacesTestUtils.promiseAsyncUpdates().then(() => { + PlacesUtils.history.hasVisits(kUniqueURI).then(isVisited => { + reloadAsyncListener(kUniqueURI, isVisited); + }); + }); + }); +} + +function reloadAsyncListener(aURI, aIsVisited) { + ok(kUniqueURI.equals(aURI) && aIsVisited, "We have visited the URI."); + PlacesUtils.history.clear().then(finish); +} + +registerCleanupFunction(async function () { + Services.prefs.setIntPref("network.proxy.type", proxyPrefValue); + Services.io.offline = false; + BrowserTestUtils.removeTab(ourTab); +}); diff --git a/toolkit/components/places/tests/browser/browser_double_redirect.js b/toolkit/components/places/tests/browser/browser_double_redirect.js new file mode 100644 index 0000000000..435bd86f19 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_double_redirect.js @@ -0,0 +1,83 @@ +// Test for bug 411966. +// When a page redirects multiple times, from_visit should point to the +// previous visit in the chain, not to the first visit in the chain. + +add_task(async function () { + await PlacesUtils.history.clear(); + + const BASE_URL = + "http://example.com/tests/toolkit/components/places/tests/browser/"; + const TEST_URI = NetUtil.newURI(BASE_URL + "begin.html"); + const FIRST_REDIRECTING_URI = NetUtil.newURI(BASE_URL + "redirect_twice.sjs"); + const FINAL_URI = NetUtil.newURI( + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html" + ); + + let promiseVisits = new Promise(resolve => { + let observer = { + _notified: [], + onVisit(uri, id, time, referrerId, transition) { + info("Received onVisit: " + uri); + this._notified.push(uri); + + if (uri != FINAL_URI.spec) { + return; + } + + is(this._notified.length, 4); + PlacesObservers.removeListener(["page-visited"], this.handleEvents); + + (async function () { + // Get all pages visited from the original typed one + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.execute( + `SELECT url FROM moz_historyvisits + JOIN moz_places h ON h.id = place_id + WHERE from_visit IN + (SELECT v.id FROM moz_historyvisits v + JOIN moz_places p ON p.id = v.place_id + WHERE p.url_hash = hash(:url) AND p.url = :url) + `, + { url: TEST_URI.spec } + ); + + is(rows.length, 1, "Found right number of visits"); + let visitedUrl = rows[0].getResultByName("url"); + // Check that redirect from_visit is not from the original typed one + is( + visitedUrl, + FIRST_REDIRECTING_URI.spec, + "Check referrer for " + visitedUrl + ); + + resolve(); + })(); + }, + handleEvents(events) { + is(events.length, 1, "Right number of visits notified"); + is(events[0].type, "page-visited"); + let { url, visitId, visitTime, referringVisitId, transitionType } = + events[0]; + this.onVisit(url, visitId, visitTime, referringVisitId, transitionType); + }, + }; + observer.handleEvents = observer.handleEvents.bind(observer); + PlacesObservers.addListener(["page-visited"], observer.handleEvents); + }); + + PlacesUtils.history.markPageAsTyped(TEST_URI); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url: TEST_URI.spec, + }, + async function (browser) { + // Load begin page, click link on page to record visits. + await BrowserTestUtils.synthesizeMouseAtCenter("#clickme", {}, browser); + + await promiseVisits; + } + ); + + await PlacesUtils.history.clear(); +}); diff --git a/toolkit/components/places/tests/browser/browser_favicon_privatebrowsing_perwindowpb.js b/toolkit/components/places/tests/browser/browser_favicon_privatebrowsing_perwindowpb.js new file mode 100644 index 0000000000..35450f0be6 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_favicon_privatebrowsing_perwindowpb.js @@ -0,0 +1,43 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test() { + waitForExplicitFinish(); + + const pageURI = + "http://example.org/tests/toolkit/components/places/tests/browser/favicon.html"; + let windowsToClose = []; + + registerCleanupFunction(function () { + windowsToClose.forEach(function (aWin) { + aWin.close(); + }); + }); + + function testOnWindow(aIsPrivate, aCallback) { + whenNewWindowLoaded({ private: aIsPrivate }, function (aWin) { + windowsToClose.push(aWin); + executeSoon(() => aCallback(aWin)); + }); + } + + function waitForTabLoad(aWin, aCallback) { + BrowserTestUtils.browserLoaded(aWin.gBrowser.selectedBrowser).then( + aCallback + ); + BrowserTestUtils.loadURIString(aWin.gBrowser.selectedBrowser, pageURI); + } + + testOnWindow(true, function (win) { + waitForTabLoad(win, function () { + PlacesUtils.favicons.getFaviconURLForPage( + NetUtil.newURI(pageURI), + function (uri, dataLen, data, mimeType) { + is(uri, null, "No result should be found"); + finish(); + } + ); + }); + }); +} diff --git a/toolkit/components/places/tests/browser/browser_history_post.js b/toolkit/components/places/tests/browser/browser_history_post.js new file mode 100644 index 0000000000..a62592516f --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_history_post.js @@ -0,0 +1,35 @@ +const PAGE_URI = + "http://example.com/tests/toolkit/components/places/tests/browser/history_post.html"; +const SJS_URI = NetUtil.newURI( + "http://example.com/tests/toolkit/components/places/tests/browser/history_post.sjs" +); + +add_task(async function () { + await BrowserTestUtils.withNewTab( + { gBrowser, url: PAGE_URI }, + async function (aBrowser) { + await SpecialPowers.spawn(aBrowser, [], async function () { + let doc = content.document; + let submit = doc.getElementById("submit"); + let iframe = doc.getElementById("post_iframe"); + let p = new Promise((resolve, reject) => { + iframe.addEventListener( + "load", + function () { + resolve(); + }, + { once: true } + ); + }); + submit.click(); + await p; + }); + let visited = await PlacesUtils.history.hasVisits(SJS_URI); + ok(!visited, "The POST page should not be added to history"); + ok( + !(await PlacesTestUtils.isPageInDB(SJS_URI.spec)), + "The page should not be in the database" + ); + } + ); +}); diff --git a/toolkit/components/places/tests/browser/browser_multi_redirect_frecency.js b/toolkit/components/places/tests/browser/browser_multi_redirect_frecency.js new file mode 100644 index 0000000000..a406422a2f --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_multi_redirect_frecency.js @@ -0,0 +1,177 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const ROOT_URI = + "http://mochi.test:8888/tests/toolkit/components/places/tests/browser/"; +const REDIRECT_URI = Services.io.newURI(ROOT_URI + "redirect_thrice.sjs"); +const INTERMEDIATE_URI_1 = Services.io.newURI( + ROOT_URI + "redirect_twice_perma.sjs" +); +const INTERMEDIATE_URI_2 = Services.io.newURI(ROOT_URI + "redirect_once.sjs"); +const TARGET_URI = Services.io.newURI( + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html" +); + +const REDIRECT_SOURCE_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.redirectSourceVisitBonus" +); +const PERM_REDIRECT_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.permRedirectVisitBonus" +); +const TYPED_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.typedVisitBonus" +); + +// Ensure that decay frecency doesn't kick in during tests (as a result +// of idle-daily). +Services.prefs.setCharPref("places.frecency.decayRate", "1.0"); + +registerCleanupFunction(async function () { + Services.prefs.clearUserPref("places.frecency.decayRate"); + await PlacesUtils.history.clear(); +}); + +async function check_uri(uri, frecency, hidden) { + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: uri, + }), + frecency, + "Frecency of the page is the expected one" + ); + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { + url: uri, + }), + hidden, + "Hidden value of the page is the expected one" + ); +} + +async function waitVisitedNotifications() { + let redirectNotified = false; + await PlacesTestUtils.waitForNotification("page-visited", visits => { + is(visits.length, 1, "Was notified for the right number of visits."); + let { url } = visits[0]; + info("Received 'page-visited': " + url); + if (url == REDIRECT_URI.spec) { + redirectNotified = true; + } + return url == TARGET_URI.spec; + }); + return redirectNotified; +} + +let firstRedirectBonus = 0; +let nextRedirectBonus = 0; +let targetBonus = 0; + +add_task(async function test_multiple_redirect() { + // The redirect source bonus overrides the link bonus. + let visitedPromise = waitVisitedNotifications(); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url: REDIRECT_URI.spec, + }, + async function () { + info("Waiting for onVisits"); + let redirectNotified = await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + firstRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(REDIRECT_URI, firstRedirectBonus, 1); + nextRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(INTERMEDIATE_URI_1, nextRedirectBonus, 1); + await check_uri(INTERMEDIATE_URI_2, nextRedirectBonus, 1); + // TODO Bug 487813 - This should be TYPED_VISIT_BONUS, however as we don't + // currently track redirects across multiple redirects, we fallback to the + // PERM_REDIRECT_VISIT_BONUS. + targetBonus += PERM_REDIRECT_VISIT_BONUS; + await check_uri(TARGET_URI, targetBonus, 0); + } + ); +}); + +add_task(async function test_multiple_redirect_typed() { + // The typed bonus wins because the redirect is permanent. + PlacesUtils.history.markPageAsTyped(REDIRECT_URI); + let visitedPromise = waitVisitedNotifications(); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url: REDIRECT_URI.spec, + }, + async function () { + info("Waiting for onVisits"); + let redirectNotified = await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + firstRedirectBonus += TYPED_VISIT_BONUS; + await check_uri(REDIRECT_URI, firstRedirectBonus, 1); + nextRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(INTERMEDIATE_URI_1, nextRedirectBonus, 1); + await check_uri(INTERMEDIATE_URI_2, nextRedirectBonus, 1); + // TODO Bug 487813 - This should be TYPED_VISIT_BONUS, however as we don't + // currently track redirects across multiple redirects, we fallback to the + // PERM_REDIRECT_VISIT_BONUS. + targetBonus += PERM_REDIRECT_VISIT_BONUS; + await check_uri(TARGET_URI, targetBonus, 0); + } + ); +}); + +add_task(async function test_second_typed_visit() { + // The typed bonus wins because the redirect is permanent. + PlacesUtils.history.markPageAsTyped(REDIRECT_URI); + let visitedPromise = waitVisitedNotifications(); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url: REDIRECT_URI.spec, + }, + async function () { + info("Waiting for onVisits"); + let redirectNotified = await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + firstRedirectBonus += TYPED_VISIT_BONUS; + await check_uri(REDIRECT_URI, firstRedirectBonus, 1); + nextRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(INTERMEDIATE_URI_1, nextRedirectBonus, 1); + await check_uri(INTERMEDIATE_URI_2, nextRedirectBonus, 1); + // TODO Bug 487813 - This should be TYPED_VISIT_BONUS, however as we don't + // currently track redirects across multiple redirects, we fallback to the + // PERM_REDIRECT_VISIT_BONUS. + targetBonus += PERM_REDIRECT_VISIT_BONUS; + await check_uri(TARGET_URI, targetBonus, 0); + } + ); +}); + +add_task(async function test_subsequent_link_visit() { + // Another non typed visit. + let visitedPromise = waitVisitedNotifications(); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url: REDIRECT_URI.spec, + }, + async function () { + info("Waiting for onVisits"); + let redirectNotified = await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + firstRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(REDIRECT_URI, firstRedirectBonus, 1); + nextRedirectBonus += REDIRECT_SOURCE_VISIT_BONUS; + await check_uri(INTERMEDIATE_URI_1, nextRedirectBonus, 1); + await check_uri(INTERMEDIATE_URI_2, nextRedirectBonus, 1); + // TODO Bug 487813 - This should be TYPED_VISIT_BONUS, however as we don't + // currently track redirects across multiple redirects, we fallback to the + // PERM_REDIRECT_VISIT_BONUS. + targetBonus += PERM_REDIRECT_VISIT_BONUS; + await check_uri(TARGET_URI, targetBonus, 0); + } + ); +}); diff --git a/toolkit/components/places/tests/browser/browser_notfound.js b/toolkit/components/places/tests/browser/browser_notfound.js new file mode 100644 index 0000000000..6f53866018 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_notfound.js @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function () { + const TEST_URL = "http://mochi.test:8888/notFoundPage.html"; + + // Used to verify errors are not marked as typed. + PlacesUtils.history.markPageAsTyped(NetUtil.newURI(TEST_URL)); + + // Create and add history observer. + let visitedPromise = new Promise(resolve => { + function listener(aEvents) { + is(aEvents.length, 1, "Right number of visits notified"); + is(aEvents[0].type, "page-visited"); + let uri = NetUtil.newURI(aEvents[0].url); + PlacesObservers.removeListener(["page-visited"], listener); + info("Received 'page-visited': " + uri.spec); + fieldForUrl(uri, "frecency", function (aFrecency) { + is(aFrecency, 0, "Frecency should be 0"); + fieldForUrl(uri, "hidden", function (aHidden) { + is(aHidden, 0, "Page should not be hidden"); + fieldForUrl(uri, "typed", function (aTyped) { + is(aTyped, 0, "page should not be marked as typed"); + resolve(); + }); + }); + }); + } + PlacesObservers.addListener(["page-visited"], listener); + }); + + let newTabPromise = BrowserTestUtils.openNewForegroundTab(gBrowser, TEST_URL); + await Promise.all([visitedPromise, newTabPromise]); + + await PlacesUtils.history.clear(); + gBrowser.removeCurrentTab(); +}); diff --git a/toolkit/components/places/tests/browser/browser_onvisit_title_null_for_navigation.js b/toolkit/components/places/tests/browser/browser_onvisit_title_null_for_navigation.js new file mode 100644 index 0000000000..a7c583975a --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_onvisit_title_null_for_navigation.js @@ -0,0 +1,41 @@ +const TEST_PATH = getRootDirectory(gTestPath).replace( + "chrome://mochitests/content", + "http://example.com" +); + +add_task(async function checkTitleNotificationForNavigation() { + const EXPECTED_URL = Services.io.newURI(TEST_PATH + "empty_page.html"); + + const promiseVisit = PlacesTestUtils.waitForNotification( + "page-visited", + events => events[0].url === EXPECTED_URL.spec + ); + + const promiseTitle = PlacesTestUtils.waitForNotification( + "page-title-changed", + events => events[0].url === EXPECTED_URL.spec + ); + + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + EXPECTED_URL.spec + ); + + const visitEvents = await promiseVisit; + Assert.equal(visitEvents.length, 1, "Right number of visits notified"); + Assert.equal(visitEvents[0].type, "page-visited"); + info("'page-visited': " + visitEvents[0].url); + Assert.equal(visitEvents[0].lastKnownTitle, null, "Should not have a title"); + + const titleEvents = await promiseTitle; + Assert.equal(titleEvents.length, 1, "Right number of title changed notified"); + Assert.equal(titleEvents[0].type, "page-title-changed"); + info("'page-title-changed': " + titleEvents[0].url); + Assert.equal( + titleEvents[0].title, + "I am an empty page", + "Should have correct title in titlechanged notification" + ); + + BrowserTestUtils.removeTab(tab); +}); diff --git a/toolkit/components/places/tests/browser/browser_redirect.js b/toolkit/components/places/tests/browser/browser_redirect.js new file mode 100644 index 0000000000..912b817ad1 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_redirect.js @@ -0,0 +1,149 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const ROOT_URI = + "http://mochi.test:8888/tests/toolkit/components/places/tests/browser/"; +const REDIRECT_URI = Services.io.newURI(ROOT_URI + "redirect.sjs"); +const TARGET_URI = Services.io.newURI(ROOT_URI + "redirect-target.html"); + +const REDIRECT_SOURCE_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.redirectSourceVisitBonus" +); +const LINK_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.linkVisitBonus" +); +const TYPED_VISIT_BONUS = Services.prefs.getIntPref( + "places.frecency.typedVisitBonus" +); + +// Ensure that decay frecency doesn't kick in during tests (as a result +// of idle-daily). +Services.prefs.setCharPref("places.frecency.decayRate", "1.0"); + +registerCleanupFunction(async function () { + Services.prefs.clearUserPref("places.frecency.decayRate"); + await PlacesUtils.history.clear(); +}); + +let redirectSourceFrecency = 0; +let redirectTargetFrecency = 0; + +async function check_uri(uri, frecency, hidden) { + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: uri, + }), + frecency, + "Frecency of the page is the expected one" + ); + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { + url: uri, + }), + hidden, + "Hidden value of the page is the expected one" + ); +} + +add_task(async function redirect_check_new_typed_visit() { + // Used to verify the redirect bonus overrides the typed bonus. + PlacesUtils.history.markPageAsTyped(REDIRECT_URI); + + redirectSourceFrecency += REDIRECT_SOURCE_VISIT_BONUS; + redirectTargetFrecency += TYPED_VISIT_BONUS; + let redirectNotified = false; + + let visitedPromise = PlacesTestUtils.waitForNotification( + "page-visited", + visits => { + is(visits.length, 1, "Was notified for the right number of visits."); + let { url } = visits[0]; + info("Received 'page-visited': " + url); + if (url == REDIRECT_URI.spec) { + redirectNotified = true; + } + return url == TARGET_URI.spec; + } + ); + + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + REDIRECT_URI.spec + ); + info("Waiting for onVisits"); + await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + await check_uri(REDIRECT_URI, redirectSourceFrecency, 1); + await check_uri(TARGET_URI, redirectTargetFrecency, 0); + + BrowserTestUtils.removeTab(tab); +}); + +add_task(async function redirect_check_second_typed_visit() { + // A second visit with a typed url. + PlacesUtils.history.markPageAsTyped(REDIRECT_URI); + + redirectSourceFrecency += REDIRECT_SOURCE_VISIT_BONUS; + redirectTargetFrecency += TYPED_VISIT_BONUS; + let redirectNotified = false; + + let visitedPromise = PlacesTestUtils.waitForNotification( + "page-visited", + visits => { + is(visits.length, 1, "Was notified for the right number of visits."); + let { url } = visits[0]; + info("Received 'page-visited': " + url); + if (url == REDIRECT_URI.spec) { + redirectNotified = true; + } + return url == TARGET_URI.spec; + } + ); + + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + REDIRECT_URI.spec + ); + info("Waiting for onVisits"); + await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + await check_uri(REDIRECT_URI, redirectSourceFrecency, 1); + await check_uri(TARGET_URI, redirectTargetFrecency, 0); + + BrowserTestUtils.removeTab(tab); +}); + +add_task(async function redirect_check_subsequent_link_visit() { + // Another visit, but this time as a visited url. + redirectSourceFrecency += REDIRECT_SOURCE_VISIT_BONUS; + redirectTargetFrecency += LINK_VISIT_BONUS; + let redirectNotified = false; + + let visitedPromise = PlacesTestUtils.waitForNotification( + "page-visited", + visits => { + is(visits.length, 1, "Was notified for the right number of visits."); + let { url } = visits[0]; + info("Received 'page-visited': " + url); + if (url == REDIRECT_URI.spec) { + redirectNotified = true; + } + return url == TARGET_URI.spec; + } + ); + + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + REDIRECT_URI.spec + ); + info("Waiting for onVisits"); + await visitedPromise; + ok(redirectNotified, "The redirect should have been notified"); + + await check_uri(REDIRECT_URI, redirectSourceFrecency, 1); + await check_uri(TARGET_URI, redirectTargetFrecency, 0); + + BrowserTestUtils.removeTab(tab); +}); diff --git a/toolkit/components/places/tests/browser/browser_redirect_self.js b/toolkit/components/places/tests/browser/browser_redirect_self.js new file mode 100644 index 0000000000..7ed7ee0af0 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_redirect_self.js @@ -0,0 +1,51 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Tests a page that redirects to itself. On the initial visit the page should + * be marked as hidden, but then the second visit should unhide it. + * This ensures that that the history anti-flooding system doesn't skip the + * second visit. + */ + +add_task(async function () { + await PlacesUtils.history.clear(); + Cc["@mozilla.org/browser/history;1"] + .getService(Ci.mozIAsyncHistory) + .clearCache(); + const url = + "http://mochi.test:8888/tests/toolkit/components/places/tests/browser/redirect_self.sjs"; + let visitCount = 0; + function onVisitsListener(events) { + visitCount++; + Assert.equal(events.length, 1, "Right number of visits notified"); + Assert.equal(events[0].url, url, "Got a visit for the expected url"); + if (visitCount == 1) { + Assert.ok(events[0].hidden, "The visit should be hidden"); + } else { + Assert.ok(!events[0].hidden, "The visit should not be hidden"); + } + } + PlacesObservers.addListener(["page-visited"], onVisitsListener); + registerCleanupFunction(async function () { + PlacesObservers.removeListener(["page-visited"], onVisitsListener); + await PlacesUtils.history.clear(); + }); + await BrowserTestUtils.withNewTab( + { + gBrowser, + url, + }, + async browser => { + await TestUtils.waitForCondition(() => visitCount == 2); + // Check that the visit is not hidden in the database. + Assert.ok( + !(await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { + url, + })), + "The url should not be hidden in the database" + ); + } + ); +}); diff --git a/toolkit/components/places/tests/browser/browser_settitle.js b/toolkit/components/places/tests/browser/browser_settitle.js new file mode 100644 index 0000000000..3519891bbe --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_settitle.js @@ -0,0 +1,48 @@ +var conn = PlacesUtils.history.DBConnection; + +/** + * Gets a single column value from either the places or historyvisits table. + */ +function getColumn(table, column, url) { + var stmt = conn.createStatement( + `SELECT ${column} FROM ${table} WHERE url_hash = hash(:val) AND url = :val` + ); + try { + stmt.params.val = url; + stmt.executeStep(); + return stmt.row[column]; + } finally { + stmt.finalize(); + } +} + +add_task(async function () { + // Make sure titles are correctly saved for a URI with the proper + // notifications. + const titleChangedPromise = + PlacesTestUtils.waitForNotification("page-title-changed"); + + const url1 = + "http://example.com/tests/toolkit/components/places/tests/browser/title1.html"; + await BrowserTestUtils.openNewForegroundTab(gBrowser, url1); + + const url2 = + "http://example.com/tests/toolkit/components/places/tests/browser/title2.html"; + let loadPromise = BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser); + BrowserTestUtils.loadURIString(gBrowser.selectedBrowser, url2); + await loadPromise; + + const events = await titleChangedPromise; + is( + events[0].url, + "http://example.com/tests/toolkit/components/places/tests/browser/title2.html" + ); + is(events[0].title, "Some title"); + is(events[0].pageGuid, getColumn("moz_places", "guid", events[0].url)); + + const title = getColumn("moz_places", "title", events[0].url); + is(title, events[0].title); + + gBrowser.removeCurrentTab(); + await PlacesUtils.history.clear(); +}); diff --git a/toolkit/components/places/tests/browser/browser_visited_notfound.js b/toolkit/components/places/tests/browser/browser_visited_notfound.js new file mode 100644 index 0000000000..1b97c307e2 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_visited_notfound.js @@ -0,0 +1,60 @@ +add_task(async function test() { + const TEST_URL = "http://mochi.test:8888/notFoundPage.html"; + // Ensure that decay frecency doesn't kick in during tests (as a result + // of idle-daily). + Services.prefs.setCharPref("places.frecency.decayRate", "1.0"); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser); + registerCleanupFunction(async function () { + Services.prefs.clearUserPref("places.frecency.decayRate"); + BrowserTestUtils.removeTab(tab); + await PlacesUtils.history.clear(); + }); + + // First add a visit to the page, this will ensure that later we skip + // updating the frecency for a newly not-found page. + await PlacesTestUtils.addVisits({ uri: TEST_URL }); + let frecency = await PlacesTestUtils.getDatabaseValue( + "moz_places", + "frecency", + { url: TEST_URL } + ); + is(frecency, 100, "Check initial frecency"); + + // Used to verify errors are not marked as typed. + PlacesUtils.history.markPageAsTyped(NetUtil.newURI(TEST_URL)); + + let promiseVisit = new Promise(resolve => { + function onVisits(events) { + PlacesObservers.removeListener(["page-visited"], onVisits); + is(events.length, 1, "Right number of visits"); + is(events[0].type, "page-visited"); + is(events[0].url, TEST_URL, "Check visited url"); + resolve(); + } + PlacesObservers.addListener(["page-visited"], onVisits); + }); + BrowserTestUtils.loadURIString(gBrowser.selectedBrowser, TEST_URL); + await promiseVisit; + + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { + url: TEST_URL, + }), + frecency, + "Frecency should be unchanged" + ); + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { + url: TEST_URL, + }), + 0, + "Page should not be hidden" + ); + is( + await PlacesTestUtils.getDatabaseValue("moz_places", "typed", { + url: TEST_URL, + }), + 0, + "page should not be marked as typed" + ); +}); diff --git a/toolkit/components/places/tests/browser/browser_visituri.js b/toolkit/components/places/tests/browser/browser_visituri.js new file mode 100644 index 0000000000..6633ac188b --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_visituri.js @@ -0,0 +1,100 @@ +/** + * One-time observer callback. + */ +function promiseObserve(name, checkFn) { + return new Promise(resolve => { + Services.obs.addObserver(function observer(subject) { + if (checkFn(subject)) { + Services.obs.removeObserver(observer, name); + resolve(); + } + }, name); + }); +} + +var conn = PlacesUtils.history.DBConnection; + +/** + * Gets a single column value from either the places or historyvisits table. + */ +function getColumn(table, column, fromColumnName, fromColumnValue) { + let sql = `SELECT ${column} + FROM ${table} + WHERE ${fromColumnName} = :val + ${fromColumnName == "url" ? "AND url_hash = hash(:val)" : ""} + LIMIT 1`; + let stmt = conn.createStatement(sql); + try { + stmt.params.val = fromColumnValue; + ok(stmt.executeStep(), "Expect to get a row"); + return stmt.row[column]; + } finally { + stmt.reset(); + } +} + +add_task(async function () { + // Make sure places visit chains are saved correctly with a redirect + // transitions. + + // Part 1: observe history events that fire when a visit occurs. + // Make sure visits appear in order, and that the visit chain is correct. + var expectedUrls = [ + "http://example.com/tests/toolkit/components/places/tests/browser/begin.html", + "http://example.com/tests/toolkit/components/places/tests/browser/redirect_twice.sjs", + "http://example.com/tests/toolkit/components/places/tests/browser/redirect_once.sjs", + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html", + ]; + var currentIndex = 0; + + function checkObserver(subject) { + var uri = subject.QueryInterface(Ci.nsIURI); + var expected = expectedUrls[currentIndex]; + is(uri.spec, expected, "Saved URL visit " + uri.spec); + + var placeId = getColumn("moz_places", "id", "url", uri.spec); + var fromVisitId = getColumn( + "moz_historyvisits", + "from_visit", + "place_id", + placeId + ); + + if (currentIndex == 0) { + is(fromVisitId, 0, "First visit has no from visit"); + } else { + var lastVisitId = getColumn( + "moz_historyvisits", + "place_id", + "id", + fromVisitId + ); + var fromVisitUrl = getColumn("moz_places", "url", "id", lastVisitId); + is( + fromVisitUrl, + expectedUrls[currentIndex - 1], + "From visit was " + expectedUrls[currentIndex - 1] + ); + } + + currentIndex++; + return currentIndex >= expectedUrls.length; + } + let visitUriPromise = promiseObserve("uri-visit-saved", checkObserver); + + const testUrl = + "http://example.com/tests/toolkit/components/places/tests/browser/begin.html"; + await BrowserTestUtils.openNewForegroundTab(gBrowser, testUrl); + + // Load begin page, click link on page to record visits. + await BrowserTestUtils.synthesizeMouseAtCenter( + "#clickme", + {}, + gBrowser.selectedBrowser + ); + await visitUriPromise; + + await PlacesUtils.history.clear(); + + gBrowser.removeCurrentTab(); +}); diff --git a/toolkit/components/places/tests/browser/browser_visituri_nohistory.js b/toolkit/components/places/tests/browser/browser_visituri_nohistory.js new file mode 100644 index 0000000000..3708b388e5 --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_visituri_nohistory.js @@ -0,0 +1,44 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +const INITIAL_URL = + "http://example.com/tests/toolkit/components/places/tests/browser/begin.html"; +const FINAL_URL = + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html"; + +/** + * One-time observer callback. + */ +function promiseObserve(name) { + return new Promise(resolve => { + Services.obs.addObserver(function observer(subject) { + Services.obs.removeObserver(observer, name); + resolve(subject); + }, name); + }); +} + +add_task(async function () { + await SpecialPowers.pushPrefEnv({ set: [["places.history.enabled", false]] }); + + let visitUriPromise = promiseObserve("uri-visit-saved"); + + await BrowserTestUtils.openNewForegroundTab(gBrowser, INITIAL_URL); + + await SpecialPowers.popPrefEnv(); + + let browserLoadedPromise = BrowserTestUtils.browserLoaded( + gBrowser.selectedBrowser + ); + BrowserTestUtils.loadURIString(gBrowser, FINAL_URL); + await browserLoadedPromise; + + let subject = await visitUriPromise; + let uri = subject.QueryInterface(Ci.nsIURI); + is(uri.spec, FINAL_URL, "received expected visit"); + + await PlacesUtils.history.clear(); + gBrowser.removeCurrentTab(); +}); diff --git a/toolkit/components/places/tests/browser/browser_visituri_privatebrowsing_perwindowpb.js b/toolkit/components/places/tests/browser/browser_visituri_privatebrowsing_perwindowpb.js new file mode 100644 index 0000000000..75fd2aa46d --- /dev/null +++ b/toolkit/components/places/tests/browser/browser_visituri_privatebrowsing_perwindowpb.js @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const initialURL = + "http://example.com/tests/toolkit/components/places/tests/browser/begin.html"; +const finalURL = + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html"; + +var observer; +var visitSavedPromise; + +add_setup(async function () { + visitSavedPromise = new Promise(resolve => { + observer = { + observe(subject, topic, data) { + // The uri-visit-saved topic should only work when on normal mode. + if (topic == "uri-visit-saved") { + Services.obs.removeObserver(observer, "uri-visit-saved"); + + // The expected visit should be the finalURL because private mode + // should not register a visit with the initialURL. + let uri = subject.QueryInterface(Ci.nsIURI); + resolve(uri.spec); + } + }, + }; + }); + + Services.obs.addObserver(observer, "uri-visit-saved"); + + registerCleanupFunction(async function () { + await PlacesUtils.history.clear(); + }); +}); + +// Note: The private window test must be the first one to run, since we'll listen +// to the first uri-visit-saved notification, and we expect this test to not +// fire any, so we'll just find the non-private window test notification. +add_task(async function test_private_browsing_window() { + await testLoadInWindow({ private: true }, initialURL); +}); + +add_task(async function test_normal_window() { + await testLoadInWindow({ private: false }, finalURL); + + let url = await visitSavedPromise; + Assert.equal(url, finalURL, "Check received expected visit"); +}); + +async function testLoadInWindow(options, url) { + let win = await BrowserTestUtils.openNewBrowserWindow(options); + + registerCleanupFunction(async function () { + await BrowserTestUtils.closeWindow(win); + }); + + let loadedPromise = BrowserTestUtils.browserLoaded( + win.gBrowser.selectedBrowser + ); + BrowserTestUtils.loadURIString(win.gBrowser.selectedBrowser, url); + await loadedPromise; +} diff --git a/toolkit/components/places/tests/browser/empty_page.html b/toolkit/components/places/tests/browser/empty_page.html new file mode 100644 index 0000000000..ac9d144cb4 --- /dev/null +++ b/toolkit/components/places/tests/browser/empty_page.html @@ -0,0 +1,8 @@ + + + + + I am an empty page + + Empty + diff --git a/toolkit/components/places/tests/browser/favicon-normal16.png b/toolkit/components/places/tests/browser/favicon-normal16.png new file mode 100644 index 0000000000..62b69a3d03 Binary files /dev/null and b/toolkit/components/places/tests/browser/favicon-normal16.png differ diff --git a/toolkit/components/places/tests/browser/favicon-normal32.png b/toolkit/components/places/tests/browser/favicon-normal32.png new file mode 100644 index 0000000000..5535363c94 Binary files /dev/null and b/toolkit/components/places/tests/browser/favicon-normal32.png differ diff --git a/toolkit/components/places/tests/browser/favicon.html b/toolkit/components/places/tests/browser/favicon.html new file mode 100644 index 0000000000..a0f5ea9594 --- /dev/null +++ b/toolkit/components/places/tests/browser/favicon.html @@ -0,0 +1,13 @@ + + + + + + + + OK we're done! + + diff --git a/toolkit/components/places/tests/browser/final.html b/toolkit/components/places/tests/browser/final.html new file mode 100644 index 0000000000..ccd5819181 --- /dev/null +++ b/toolkit/components/places/tests/browser/final.html @@ -0,0 +1,10 @@ + + + + + OK we're done! + + diff --git a/toolkit/components/places/tests/browser/head.js b/toolkit/components/places/tests/browser/head.js new file mode 100644 index 0000000000..e4d0b5566b --- /dev/null +++ b/toolkit/components/places/tests/browser/head.js @@ -0,0 +1,74 @@ +ChromeUtils.defineESModuleGetters(this, { + PlacesTestUtils: "resource://testing-common/PlacesTestUtils.sys.mjs", +}); + +const TRANSITION_LINK = PlacesUtils.history.TRANSITION_LINK; +const TRANSITION_TYPED = PlacesUtils.history.TRANSITION_TYPED; +const TRANSITION_BOOKMARK = PlacesUtils.history.TRANSITION_BOOKMARK; +const TRANSITION_REDIRECT_PERMANENT = + PlacesUtils.history.TRANSITION_REDIRECT_PERMANENT; +const TRANSITION_REDIRECT_TEMPORARY = + PlacesUtils.history.TRANSITION_REDIRECT_TEMPORARY; +const TRANSITION_EMBED = PlacesUtils.history.TRANSITION_EMBED; +const TRANSITION_FRAMED_LINK = PlacesUtils.history.TRANSITION_FRAMED_LINK; +const TRANSITION_DOWNLOAD = PlacesUtils.history.TRANSITION_DOWNLOAD; + +/** + * Returns a moz_places field value for a url. + * + * @param {nsIURI|String} aURI + * The URI or spec to get field for. + * @param {String} aFieldName + * The field name to get the value of. + * @param {Function} aCallback + * Callback function that will get the property value. + */ +function fieldForUrl(aURI, aFieldName, aCallback) { + let url = aURI instanceof Ci.nsIURI ? aURI.spec : aURI; + let stmt = PlacesUtils.history.DBConnection.createAsyncStatement( + `SELECT ${aFieldName} FROM moz_places WHERE url_hash = hash(:page_url) AND url = :page_url` + ); + stmt.params.page_url = url; + stmt.executeAsync({ + _value: -1, + handleResult(aResultSet) { + let row = aResultSet.getNextRow(); + if (!row) { + ok(false, "The page should exist in the database"); + } + this._value = row.getResultByName(aFieldName); + }, + handleError() {}, + handleCompletion(aReason) { + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) { + ok(false, "The statement should properly succeed"); + } + aCallback(this._value); + }, + }); + stmt.finalize(); +} + +/** + * Promise wrapper for fieldForUrl. + * + * @param {nsIURI|String} aURI + * The URI or spec to get field for. + * @param {String} aFieldName + * The field name to get the value of. + * @return {Promise} + * A promise that is resolved with the value of the field. + */ +function promiseFieldForUrl(aURI, aFieldName) { + return new Promise(resolve => { + function callback(result) { + resolve(result); + } + fieldForUrl(aURI, aFieldName, callback); + }); +} + +function whenNewWindowLoaded(aOptions, aCallback) { + BrowserTestUtils.waitForNewWindow().then(aCallback); + OpenBrowserWindow(aOptions); +} diff --git a/toolkit/components/places/tests/browser/history_post.html b/toolkit/components/places/tests/browser/history_post.html new file mode 100644 index 0000000000..a579a9b8ae --- /dev/null +++ b/toolkit/components/places/tests/browser/history_post.html @@ -0,0 +1,12 @@ + + + + Test post pages are not added to history + + + +
      + +
      + + diff --git a/toolkit/components/places/tests/browser/history_post.sjs b/toolkit/components/places/tests/browser/history_post.sjs new file mode 100644 index 0000000000..08c1afe853 --- /dev/null +++ b/toolkit/components/places/tests/browser/history_post.sjs @@ -0,0 +1,5 @@ +function handleRequest(request, response) { + response.setStatusLine("1.0", 200, "OK"); + response.setHeader("Content-Type", "text/plain; charset=utf-8", false); + response.write("Ciao"); +} diff --git a/toolkit/components/places/tests/browser/previews/browser.ini b/toolkit/components/places/tests/browser/previews/browser.ini new file mode 100644 index 0000000000..dd77faa323 --- /dev/null +++ b/toolkit/components/places/tests/browser/previews/browser.ini @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +prefs = + browser.pagethumbnails.capturing_disabled=false + places.previews.enabled=true + places.previews.log=true + +[browser_thumbnails.js] diff --git a/toolkit/components/places/tests/browser/previews/browser_thumbnails.js b/toolkit/components/places/tests/browser/previews/browser_thumbnails.js new file mode 100644 index 0000000000..ef776694c2 --- /dev/null +++ b/toolkit/components/places/tests/browser/previews/browser_thumbnails.js @@ -0,0 +1,174 @@ +/* Any copyright is dedicated to the Public Domain. + * https://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests PlacesPreviews.jsm + */ +const { PlacesPreviews } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesPreviews.sys.mjs" +); +const { PlacesTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PlacesTestUtils.sys.mjs" +); + +const TEST_URL1 = "http://example.com/"; +const TEST_URL2 = "http://example.org/"; + +/** + * Counts tombstone entries. + * @returns {integer} number of tombstone entries. + */ +async function countTombstones() { + await PlacesTestUtils.promiseAsyncUpdates(); + let db = await PlacesUtils.promiseDBConnection(); + return ( + await db.execute("SELECT count(*) FROM moz_previews_tombstones") + )[0].getResultByIndex(0); +} + +add_task(async function test_thumbnail() { + registerCleanupFunction(async () => { + await PlacesUtils.history.clear(); + // Ensure tombstones table has been emptied. + await TestUtils.waitForCondition(async () => { + return (await countTombstones()) == 0; + }); + PlacesPreviews.testSetDeletionTimeout(null); + }); + // Sanity check initial state. + Assert.equal(await countTombstones(), 0, "There's no tombstone entries"); + + info("Test preview creation and storage."); + await BrowserTestUtils.withNewTab(TEST_URL1, async browser => { + await retryUpdatePreview(browser.currentURI.spec); + let filePath = PlacesPreviews.getPathForUrl(TEST_URL1); + Assert.ok(await IOUtils.exists(filePath), "The screenshot exists"); + Assert.equal( + filePath.substring(filePath.lastIndexOf(".")), + PlacesPreviews.fileExtension, + "Check extension" + ); + await testImageFile(filePath); + await testMozPageThumb(TEST_URL1); + }); +}); + +add_task(async function test_page_removal() { + info("Store another preview and test page removal."); + await BrowserTestUtils.withNewTab(TEST_URL2, async browser => { + await retryUpdatePreview(browser.currentURI.spec); + let filePath = PlacesPreviews.getPathForUrl(TEST_URL2); + Assert.ok(await IOUtils.exists(filePath), "The screenshot exists"); + }); + + // Set deletion time to a small value so it runs immediately. + PlacesPreviews.testSetDeletionTimeout(0); + info("Wait for deletion, check one preview is removed, not the other one."); + let promiseDeleted = new Promise(resolve => { + PlacesPreviews.once("places-preview-deleted", (topic, filePath) => { + resolve(filePath); + }); + }); + await PlacesUtils.history.remove(TEST_URL1); + + let deletedFilePath = await promiseDeleted; + Assert.ok( + !(await IOUtils.exists(deletedFilePath)), + "Check deleted file has been removed" + ); + + info("Check tombstones table has been emptied."); + Assert.equal(await countTombstones(), 0, "There's no tombstone entries"); + + info("Check the other thumbnail has not been removed."); + let path = PlacesPreviews.getPathForUrl(TEST_URL2); + Assert.ok(await IOUtils.exists(path), "Check non-deleted url is still there"); + await testImageFile(path); + await testMozPageThumb(TEST_URL2); +}); + +add_task(async function async_test_deleteOrphans() { + let path = PlacesPreviews.getPathForUrl(TEST_URL2); + Assert.ok(await IOUtils.exists(path), "Sanity check one preview exists"); + // Create a file in the given path that doesn't have an entry in Places. + let fakePath = PathUtils.join( + PlacesPreviews.getPath(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa." + PlacesPreviews.fileExtension + ); + // File contents don't matter. + await IOUtils.writeJSON(fakePath, { test: true }); + let promiseDeleted = new Promise(resolve => { + PlacesPreviews.once("places-preview-deleted", (topic, filePath) => { + resolve(filePath); + }); + }); + + await PlacesPreviews.deleteOrphans(); + let deletedFilePath = await promiseDeleted; + Assert.equal(deletedFilePath, fakePath, "Check orphan has been deleted"); + Assert.equal(await countTombstones(), 0, "There's no tombstone entries left"); + Assert.ok( + !(await IOUtils.exists(fakePath)), + "Ensure orphan has been deleted" + ); + + Assert.ok(await IOUtils.exists(path), "Ensure valid preview is still there"); +}); + +async function testImageFile(path) { + info("Load the file and check its content type."); + const buffer = await IOUtils.read(path); + const fourcc = new TextDecoder("utf-8").decode(buffer.slice(8, 12)); + Assert.equal(fourcc, "WEBP", "Check the stored preview is webp"); +} + +async function testMozPageThumb(url) { + info("Check moz-page-thumb protocol: " + PlacesPreviews.getPageThumbURL(url)); + let { data, contentType } = await fetchImage( + PlacesPreviews.getPageThumbURL(url) + ); + Assert.equal( + contentType, + PlacesPreviews.fileContentType, + "Check the content type" + ); + const fourcc = data.slice(8, 12); + Assert.equal(fourcc, "WEBP", "Check the returned preview is webp"); +} + +function fetchImage(url) { + return new Promise((resolve, reject) => { + NetUtil.asyncFetch( + { + uri: NetUtil.newURI(url), + loadUsingSystemPrincipal: true, + contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE, + }, + (input, status, request) => { + if (!Components.isSuccessCode(status)) { + reject(new Error("unable to load image")); + return; + } + + try { + let data = NetUtil.readInputStreamToString(input, input.available()); + let contentType = request.QueryInterface(Ci.nsIChannel).contentType; + input.close(); + resolve({ data, contentType }); + } catch (ex) { + reject(ex); + } + } + ); + }); +} + +/** + * Sometimes on macOS fetching the preview fails for timeout/network reasons, + * this retries so the test doesn't intermittently fail over it. + * @param {string} url The url to store a preview for. + * @returns {Promise} resolved once a preview has been captured. + */ +function retryUpdatePreview(url) { + return TestUtils.waitForCondition(() => PlacesPreviews.update(url)); +} diff --git a/toolkit/components/places/tests/browser/redirect-target.html b/toolkit/components/places/tests/browser/redirect-target.html new file mode 100644 index 0000000000..3700263385 --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect-target.html @@ -0,0 +1 @@ +

      Ciao!

      diff --git a/toolkit/components/places/tests/browser/redirect.sjs b/toolkit/components/places/tests/browser/redirect.sjs new file mode 100644 index 0000000000..ab47335ffe --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect.sjs @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function handleRequest(request, response) { + let page = "

      Redirecting...

      "; + + response.setStatusLine(request.httpVersion, "301", "Moved Permanently"); + response.setHeader("Content-Type", "text/html", false); + response.setHeader("Content-Length", page.length + "", false); + response.setHeader("Location", "redirect-target.html", false); + response.write(page); +} diff --git a/toolkit/components/places/tests/browser/redirect_once.sjs b/toolkit/components/places/tests/browser/redirect_once.sjs new file mode 100644 index 0000000000..b9ccd0829a --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect_once.sjs @@ -0,0 +1,13 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +function handleRequest(request, response) { + response.setStatusLine("1.1", 301, "Found"); + response.setHeader( + "Location", + "http://test1.example.com/tests/toolkit/components/places/tests/browser/final.html", + false + ); +} diff --git a/toolkit/components/places/tests/browser/redirect_self.sjs b/toolkit/components/places/tests/browser/redirect_self.sjs new file mode 100644 index 0000000000..953afe5f26 --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect_self.sjs @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Script that redirects to itself once. + +function handleRequest(request, response) { + if ( + request.hasHeader("Cookie") && + request.getHeader("Cookie").includes("redirect-self") + ) { + response.setStatusLine("1.0", 200, "OK"); + // Expire the cookie. + response.setHeader( + "Set-Cookie", + "redirect-self=true; expires=Thu, 01 Jan 1970 00:00:00 GMT", + true + ); + response.setHeader("Content-Type", "text/plain; charset=utf-8", false); + response.write("OK"); + } else { + response.setStatusLine(request.httpVersion, 302, "Moved Temporarily"); + response.setHeader("Set-Cookie", "redirect-self=true", true); + response.setHeader("Location", "redirect_self.sjs"); + response.write("Moved Temporarily"); + } +} diff --git a/toolkit/components/places/tests/browser/redirect_thrice.sjs b/toolkit/components/places/tests/browser/redirect_thrice.sjs new file mode 100644 index 0000000000..55154a736e --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect_thrice.sjs @@ -0,0 +1,9 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +function handleRequest(request, response) { + response.setStatusLine("1.1", 302, "Found"); + response.setHeader("Location", "redirect_twice_perma.sjs", false); +} diff --git a/toolkit/components/places/tests/browser/redirect_twice.sjs b/toolkit/components/places/tests/browser/redirect_twice.sjs new file mode 100644 index 0000000000..099d20022e --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect_twice.sjs @@ -0,0 +1,9 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +function handleRequest(request, response) { + response.setStatusLine("1.1", 302, "Found"); + response.setHeader("Location", "redirect_once.sjs", false); +} diff --git a/toolkit/components/places/tests/browser/redirect_twice_perma.sjs b/toolkit/components/places/tests/browser/redirect_twice_perma.sjs new file mode 100644 index 0000000000..a40abd4170 --- /dev/null +++ b/toolkit/components/places/tests/browser/redirect_twice_perma.sjs @@ -0,0 +1,9 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +function handleRequest(request, response) { + response.setStatusLine("1.1", 301, "Found"); + response.setHeader("Location", "redirect_once.sjs", false); +} diff --git a/toolkit/components/places/tests/browser/title1.html b/toolkit/components/places/tests/browser/title1.html new file mode 100644 index 0000000000..3c98d693ec --- /dev/null +++ b/toolkit/components/places/tests/browser/title1.html @@ -0,0 +1,12 @@ + + + + + + + title1.html + + diff --git a/toolkit/components/places/tests/browser/title2.html b/toolkit/components/places/tests/browser/title2.html new file mode 100644 index 0000000000..8830328796 --- /dev/null +++ b/toolkit/components/places/tests/browser/title2.html @@ -0,0 +1,13 @@ + + + + + Some title + + + title2.html + + diff --git a/toolkit/components/places/tests/chrome/bad_links.atom b/toolkit/components/places/tests/chrome/bad_links.atom new file mode 100644 index 0000000000..4469272524 --- /dev/null +++ b/toolkit/components/places/tests/chrome/bad_links.atom @@ -0,0 +1,74 @@ + + + + Example Feed + + 2003-12-13T18:30:02Z + + + John Doe + + urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6 + + + + First good item + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a + 2003-12-13T18:30:02Z + + Some text. + + + + + data: link + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6b + 2003-12-13T18:30:03Z + + Some text. + + + + + javascript: link + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6c + 2003-12-13T18:30:04Z + + Some text. + + + + + file: link + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6d + 2003-12-13T18:30:05Z + + Some text. + + + + + chrome: link + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6e + 2003-12-13T18:30:06Z + + Some text. + + + + + Last good item + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6b + 2003-12-13T18:30:07Z + + Some text. + + + + diff --git a/toolkit/components/places/tests/chrome/browser_disableglobalhistory.xhtml b/toolkit/components/places/tests/chrome/browser_disableglobalhistory.xhtml new file mode 100644 index 0000000000..f8482ca989 --- /dev/null +++ b/toolkit/components/places/tests/chrome/browser_disableglobalhistory.xhtml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + diff --git a/toolkit/components/places/tests/chrome/chrome.ini b/toolkit/components/places/tests/chrome/chrome.ini new file mode 100644 index 0000000000..20f6e89db9 --- /dev/null +++ b/toolkit/components/places/tests/chrome/chrome.ini @@ -0,0 +1,7 @@ +[DEFAULT] +support-files = head.js + +[test_371798.xhtml] +[test_favicon_annotations.xhtml] +[test_browser_disableglobalhistory.xhtml] +support-files = browser_disableglobalhistory.xhtml diff --git a/toolkit/components/places/tests/chrome/head.js b/toolkit/components/places/tests/chrome/head.js new file mode 100644 index 0000000000..7c03e6f33d --- /dev/null +++ b/toolkit/components/places/tests/chrome/head.js @@ -0,0 +1,8 @@ +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +ChromeUtils.defineESModuleGetters(this, { + PlacesTestUtils: "resource://testing-common/PlacesTestUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); diff --git a/toolkit/components/places/tests/chrome/link-less-items-no-site-uri.rss b/toolkit/components/places/tests/chrome/link-less-items-no-site-uri.rss new file mode 100644 index 0000000000..612b0a5c2e --- /dev/null +++ b/toolkit/components/places/tests/chrome/link-less-items-no-site-uri.rss @@ -0,0 +1,18 @@ + + + + feed title + 180 + + linked feed item + http://feed-item-link.com + + + link-less feed item + + + linked feed item + http://feed-item-link.com + + + diff --git a/toolkit/components/places/tests/chrome/link-less-items.rss b/toolkit/components/places/tests/chrome/link-less-items.rss new file mode 100644 index 0000000000..a30d4a3531 --- /dev/null +++ b/toolkit/components/places/tests/chrome/link-less-items.rss @@ -0,0 +1,19 @@ + + + + feed title + http://feed-link.com + 180 + + linked feed item + http://feed-item-link.com + + + link-less feed item + + + linked feed item + http://feed-item-link.com + + + diff --git a/toolkit/components/places/tests/chrome/rss_as_html.rss b/toolkit/components/places/tests/chrome/rss_as_html.rss new file mode 100644 index 0000000000..e823050353 --- /dev/null +++ b/toolkit/components/places/tests/chrome/rss_as_html.rss @@ -0,0 +1,27 @@ + + + +sadfasdfasdfasfasdf +http://www.example.com +asdfasdfasdf.example.com +de +asdfasdfasdfasdf +Tue, 11 Mar 2008 18:52:52 +0100 +http://blogs.law.harvard.edu/tech/rss +10 + +The First Title +http://www.example.com/index.html +Tue, 11 Mar 2008 18:24:43 +0100 + + +askdlfjas;dfkjas;fkdj +

      +]]> +       +aklsjdhfasdjfahasdfhj +http://foo.example.com/asdfasdf +       +       +       diff --git a/toolkit/components/places/tests/chrome/rss_as_html.rss^headers^ b/toolkit/components/places/tests/chrome/rss_as_html.rss^headers^ new file mode 100644 index 0000000000..04fbaa08fe --- /dev/null +++ b/toolkit/components/places/tests/chrome/rss_as_html.rss^headers^ @@ -0,0 +1,2 @@ +HTTP 200 OK +Content-Type: text/html diff --git a/toolkit/components/places/tests/chrome/sample_feed.atom b/toolkit/components/places/tests/chrome/sample_feed.atom new file mode 100644 index 0000000000..add75efb4d --- /dev/null +++ b/toolkit/components/places/tests/chrome/sample_feed.atom @@ -0,0 +1,23 @@ + + + + Example Feed + + 2003-12-13T18:30:02Z + + + John Doe + + urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6 + + + + Atom-Powered Robots Run Amok + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a + 2003-12-13T18:30:02Z + + Some text. + + + diff --git a/toolkit/components/places/tests/chrome/test_371798.xhtml b/toolkit/components/places/tests/chrome/test_371798.xhtml new file mode 100644 index 0000000000..33e866e51e --- /dev/null +++ b/toolkit/components/places/tests/chrome/test_371798.xhtml @@ -0,0 +1,76 @@ + + + + + + + + diff --git a/toolkit/components/places/tests/chrome/test_browser_disableglobalhistory.xhtml b/toolkit/components/places/tests/chrome/test_browser_disableglobalhistory.xhtml new file mode 100644 index 0000000000..6a7d32dabe --- /dev/null +++ b/toolkit/components/places/tests/chrome/test_browser_disableglobalhistory.xhtml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + diff --git a/toolkit/components/places/tests/chrome/test_favicon_annotations.xhtml b/toolkit/components/places/tests/chrome/test_favicon_annotations.xhtml new file mode 100644 index 0000000000..a08347460f --- /dev/null +++ b/toolkit/components/places/tests/chrome/test_favicon_annotations.xhtml @@ -0,0 +1,135 @@ + + + + + + + + + + +

      + + 
      
+  
+      
diff --git a/toolkit/components/places/tests/expiration/head_expiration.js b/toolkit/components/places/tests/expiration/head_expiration.js
new file mode 100644
index 0000000000..ce9fe48348
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/head_expiration.js
@@ -0,0 +1,112 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+/**
+ * Causes expiration component to start, otherwise it would wait for the first
+ * history notification.
+ */
+function force_expiration_start() {
+  Cc["@mozilla.org/places/expiration;1"]
+    .getService(Ci.nsIObserver)
+    .observe(null, "testing-mode", null);
+}
+
+/**
+ * Forces an expiration run.
+ *
+ * @param [optional] aLimit
+ *        Limit for the expiration.  Pass -1 for unlimited.
+ *        Any other non-positive value will just expire orphans.
+ *
+ * @return {Promise}
+ * @resolves When expiration finishes.
+ * @rejects Never.
+ */
+function promiseForceExpirationStep(aLimit) {
+  let promise = promiseTopicObserved(PlacesUtils.TOPIC_EXPIRATION_FINISHED);
+  let expire = Cc["@mozilla.org/places/expiration;1"].getService(
+    Ci.nsIObserver
+  );
+  expire.observe(null, "places-debug-start-expiration", aLimit);
+  return promise;
+}
+
+/**
+ * Expiration preferences helpers.
+ */
+
+function setInterval(aNewInterval) {
+  Services.prefs.setIntPref(
+    "places.history.expiration.interval_seconds",
+    aNewInterval
+  );
+}
+function getInterval() {
+  return Services.prefs.getIntPref(
+    "places.history.expiration.interval_seconds"
+  );
+}
+function clearInterval() {
+  try {
+    Services.prefs.clearUserPref("places.history.expiration.interval_seconds");
+  } catch (ex) {}
+}
+
+function setMaxPages(aNewMaxPages) {
+  Services.prefs.setIntPref(
+    "places.history.expiration.max_pages",
+    aNewMaxPages
+  );
+}
+function getMaxPages() {
+  return Services.prefs.getIntPref("places.history.expiration.max_pages");
+}
+function clearMaxPages() {
+  try {
+    Services.prefs.clearUserPref("places.history.expiration.max_pages");
+  } catch (ex) {}
+}
+
+function setHistoryEnabled(aHistoryEnabled) {
+  Services.prefs.setBoolPref("places.history.enabled", aHistoryEnabled);
+}
+function getHistoryEnabled() {
+  return Services.prefs.getBoolPref("places.history.enabled");
+}
+function clearHistoryEnabled() {
+  try {
+    Services.prefs.clearUserPref("places.history.enabled");
+  } catch (ex) {}
+}
+
+/**
+ * Returns a PRTime in the past usable to add expirable visits.
+ *
+ * param [optional] daysAgo
+ *       Expiration ignores any visit added in the last 7 days, so by default
+ *       this will be set to 7.
+ * @note to be safe against DST issues we go back one day more.
+ */
+function getExpirablePRTime(daysAgo = 7) {
+  let dateObj = new Date();
+  // Normalize to midnight
+  dateObj.setHours(0);
+  dateObj.setMinutes(0);
+  dateObj.setSeconds(0);
+  dateObj.setMilliseconds(0);
+  dateObj = new Date(dateObj.getTime() - (daysAgo + 1) * 86400000);
+  return dateObj.getTime() * 1000;
+}
diff --git a/toolkit/components/places/tests/expiration/test_annos_expire_never.js b/toolkit/components/places/tests/expiration/test_annos_expire_never.js
new file mode 100644
index 0000000000..39c55ecc04
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_annos_expire_never.js
@@ -0,0 +1,72 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * What this is aimed to test:
+ *
+ * EXPIRE_NEVER annotations should be expired when a page is removed from the
+ * database.
+ * If the annotation is a page annotation this will happen when the page is
+ * expired, namely when the page has no visits and is not bookmarked.
+ */
+
+add_task(async function test_annos_expire_never() {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  // Expire all expirable pages.
+  setMaxPages(0);
+
+  // Add some visited page and a couple expire never annotations for each.
+  let now = getExpirablePRTime();
+  for (let i = 0; i < 5; i++) {
+    let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
+    await PlacesTestUtils.addVisits({ uri: pageURI, visitDate: now++ });
+    await PlacesUtils.history.update({
+      url: pageURI,
+      annotations: new Map([
+        ["page_expire1", "test"],
+        ["page_expire2", "test"],
+      ]),
+    });
+  }
+
+  let pages = await getPagesWithAnnotation("page_expire1");
+  Assert.equal(pages.length, 5);
+  pages = await getPagesWithAnnotation("page_expire2");
+  Assert.equal(pages.length, 5);
+
+  // Add other visited page and a couple expire never annotations for each.
+  // We won't expire these visits, so the annotations should survive.
+  for (let i = 0; i < 5; i++) {
+    let pageURI = uri("http://persist_page_anno." + i + ".mozilla.org/");
+    await PlacesTestUtils.addVisits({ uri: pageURI, visitDate: now++ });
+    await PlacesUtils.history.update({
+      url: pageURI,
+      annotations: new Map([
+        ["page_persist1", "test"],
+        ["page_persist2", "test"],
+      ]),
+    });
+  }
+
+  pages = await getPagesWithAnnotation("page_persist1");
+  Assert.equal(pages.length, 5);
+  pages = await getPagesWithAnnotation("page_persist2");
+  Assert.equal(pages.length, 5);
+
+  // Expire all visits for the first 5 pages and the bookmarks.
+  await promiseForceExpirationStep(5);
+
+  pages = await getPagesWithAnnotation("page_expire1");
+  Assert.equal(pages.length, 0);
+  pages = await getPagesWithAnnotation("page_expire2");
+  Assert.equal(pages.length, 0);
+  pages = await getPagesWithAnnotation("page_persist1");
+  Assert.equal(pages.length, 5);
+  pages = await getPagesWithAnnotation("page_persist2");
+  Assert.equal(pages.length, 5);
+});
diff --git a/toolkit/components/places/tests/expiration/test_clearHistory.js b/toolkit/components/places/tests/expiration/test_clearHistory.js
new file mode 100644
index 0000000000..a4684f0269
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_clearHistory.js
@@ -0,0 +1,57 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * What this is aimed to test:
+ *
+ * History.clear() should expire everything but bookmarked pages and valid
+ * annos.
+ */
+
+add_task(async function test_historyClear() {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  // Expire all expirable pages.
+  setMaxPages(0);
+
+  // Add some bookmarked page with visit and annotations.
+  for (let i = 0; i < 5; i++) {
+    let pageURI = uri("http://item_anno." + i + ".mozilla.org/");
+    // This visit will be expired.
+    await PlacesTestUtils.addVisits({ uri: pageURI });
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: pageURI,
+      title: null,
+    });
+    // Will persist because the page is bookmarked.
+    await PlacesUtils.history.update({
+      url: pageURI,
+      annotations: new Map([["persist", "test"]]),
+    });
+  }
+
+  // Add some visited page and annotations for each.
+  for (let i = 0; i < 5; i++) {
+    // All page annotations related to these expired pages are expected to
+    // expire as well.
+    let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
+    await PlacesTestUtils.addVisits({ uri: pageURI });
+    await PlacesUtils.history.update({
+      url: pageURI,
+      annotations: new Map([["expire", "test"]]),
+    });
+  }
+
+  // Expire all visits for the bookmarks
+  await PlacesUtils.history.clear();
+
+  Assert.equal((await getPagesWithAnnotation("expire")).length, 0);
+
+  let pages = await getPagesWithAnnotation("persist");
+  Assert.equal(pages.length, 5);
+});
diff --git a/toolkit/components/places/tests/expiration/test_debug_expiration.js b/toolkit/components/places/tests/expiration/test_debug_expiration.js
new file mode 100644
index 0000000000..204295d46c
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_debug_expiration.js
@@ -0,0 +1,469 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * What this is aimed to test:
+ *
+ * Expiration can be manually triggered through a debug topic, but that should
+ * only expire orphan entries, unless -1 is passed as limit.
+ */
+
+const EXPIRE_DAYS = 90;
+var gExpirableTime = getExpirablePRTime(EXPIRE_DAYS);
+var gNonExpirableTime = getExpirablePRTime(EXPIRE_DAYS - 2);
+
+add_task(async function test_expire_orphans() {
+  // Add visits to 2 pages and force a orphan expiration. Visits should survive.
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://page1.mozilla.org/"),
+    visitDate: gExpirableTime++,
+  });
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://page2.mozilla.org/"),
+    visitDate: gExpirableTime++,
+  });
+  // Create a orphan place.
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://page3.mozilla.org/",
+    title: "",
+  });
+  await PlacesUtils.bookmarks.remove(bm);
+
+  // Expire now.
+  await promiseForceExpirationStep(0);
+
+  // Check that visits survived.
+  Assert.equal(visits_in_database("http://page1.mozilla.org/"), 1);
+  Assert.equal(visits_in_database("http://page2.mozilla.org/"), 1);
+  Assert.ok(!page_in_database("http://page3.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_orphans_optionalarg() {
+  // Add visits to 2 pages and force a orphan expiration. Visits should survive.
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://page1.mozilla.org/"),
+    visitDate: gExpirableTime++,
+  });
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://page2.mozilla.org/"),
+    visitDate: gExpirableTime++,
+  });
+  // Create a orphan place.
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://page3.mozilla.org/",
+    title: "",
+  });
+  await PlacesUtils.bookmarks.remove(bm);
+
+  // Expire now.
+  await promiseForceExpirationStep();
+
+  // Check that visits survived.
+  Assert.equal(visits_in_database("http://page1.mozilla.org/"), 1);
+  Assert.equal(visits_in_database("http://page2.mozilla.org/"), 1);
+  Assert.ok(!page_in_database("http://page3.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_limited() {
+  await PlacesTestUtils.addVisits([
+    {
+      // Should be expired cause it's the oldest visit
+      uri: "http://old.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+    {
+      // Should not be expired cause we limit 1
+      uri: "http://new.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+  ]);
+
+  // Expire now.
+  await promiseForceExpirationStep(1);
+
+  // Check that newer visit survived.
+  Assert.equal(visits_in_database("http://new.mozilla.org/"), 1);
+  // Other visits should have been expired.
+  Assert.ok(!page_in_database("http://old.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_visitcount_longurl() {
+  let longurl = "http://long.mozilla.org/" + "a".repeat(232);
+  let longurl2 = "http://long2.mozilla.org/" + "a".repeat(232);
+  await PlacesTestUtils.addVisits([
+    {
+      // Should be expired cause it's the oldest visit
+      uri: "http://old.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+    {
+      // Should not be expired cause it has 2 visits.
+      uri: longurl,
+      visitDate: gExpirableTime++,
+    },
+    {
+      uri: longurl,
+      visitDate: gNonExpirableTime,
+    },
+    {
+      // Should be expired cause it has 1 old visit.
+      uri: longurl2,
+      visitDate: gExpirableTime++,
+    },
+  ]);
+
+  await promiseForceExpirationStep(1);
+
+  // Check that some visits survived.
+  Assert.equal(visits_in_database(longurl), 2);
+  // Check visit has been removed.
+  Assert.equal(visits_in_database(longurl2), 0);
+
+  // Other visits should have been expired.
+  Assert.ok(!page_in_database("http://old.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_limited_exoticurl() {
+  await PlacesTestUtils.addVisits([
+    {
+      // Should be expired cause it's the oldest visit
+      uri: "http://old.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+    {
+      // Should not be expired cause younger than EXPIRE_DAYS.
+      uri: "http://nonexpirable-download.mozilla.org",
+      visitDate: gNonExpirableTime,
+      transition: PlacesUtils.history.TRANSITIONS.DOWNLOAD,
+    },
+    {
+      // Should be expired cause it's a long url older than EXPIRE_DAYS.
+      uri: "http://download.mozilla.org",
+      visitDate: gExpirableTime++,
+      transition: 7,
+    },
+  ]);
+
+  await promiseForceExpirationStep(1);
+
+  // Check that some visits survived.
+  Assert.equal(
+    visits_in_database("http://nonexpirable-download.mozilla.org/"),
+    1
+  );
+  // The visits are gone, the url is not yet, cause we limited the expiration
+  // to one entry, and we already removed http://old.mozilla.org/.
+  // The page normally would be expired by the next expiration run.
+  Assert.equal(visits_in_database("http://download.mozilla.org/"), 0);
+  // Other visits should have been expired.
+  Assert.ok(!page_in_database("http://old.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_exotic_hidden() {
+  let visits = [
+    {
+      // Should be expired cause it's the oldest visit
+      uri: "http://old.mozilla.org/",
+      visitDate: gExpirableTime++,
+      expectedCount: 0,
+    },
+    {
+      // Expirable typed hidden url.
+      uri: "https://typedhidden.mozilla.org/",
+      visitDate: gExpirableTime++,
+      transition: PlacesUtils.history.TRANSITIONS.FRAMED_LINK,
+      expectedCount: 2,
+    },
+    {
+      // Mark as typed.
+      uri: "https://typedhidden.mozilla.org/",
+      visitDate: gExpirableTime++,
+      transition: PlacesUtils.history.TRANSITIONS.TYPED,
+      expectedCount: 2,
+    },
+    {
+      // Expirable non-typed hidden url.
+      uri: "https://hidden.mozilla.org/",
+      visitDate: gExpirableTime++,
+      transition: PlacesUtils.history.TRANSITIONS.FRAMED_LINK,
+      expectedCount: 0,
+    },
+  ];
+  await PlacesTestUtils.addVisits(visits);
+  for (let visit of visits) {
+    Assert.greater(visits_in_database(visit.uri), 0);
+  }
+
+  await promiseForceExpirationStep(1);
+
+  for (let visit of visits) {
+    Assert.equal(
+      visits_in_database(visit.uri),
+      visit.expectedCount,
+      `${visit.uri} should${
+        visit.expectedCount == 0 ? " " : " not "
+      }have been expired`
+    );
+  }
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_unlimited() {
+  let longurl = "http://long.mozilla.org/" + "a".repeat(232);
+  await PlacesTestUtils.addVisits([
+    {
+      uri: "http://old.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+    {
+      uri: "http://new.mozilla.org/",
+      visitDate: gExpirableTime++,
+    },
+    // Add expirable visits.
+    {
+      uri: "http://download.mozilla.org/",
+      visitDate: gExpirableTime++,
+      transition: PlacesUtils.history.TRANSITION_DOWNLOAD,
+    },
+    {
+      uri: longurl,
+      visitDate: gExpirableTime++,
+    },
+
+    // Add non-expirable visits
+    {
+      uri: "http://nonexpirable.mozilla.org/",
+      visitDate: getExpirablePRTime(5),
+    },
+    {
+      uri: "http://nonexpirable-download.mozilla.org/",
+      visitDate: getExpirablePRTime(5),
+      transition: PlacesUtils.history.TRANSITION_DOWNLOAD,
+    },
+    {
+      uri: longurl,
+      visitDate: getExpirablePRTime(5),
+    },
+  ]);
+
+  await promiseForceExpirationStep(-1);
+
+  // Check that some visits survived.
+  Assert.equal(visits_in_database("http://nonexpirable.mozilla.org/"), 1);
+  Assert.equal(
+    visits_in_database("http://nonexpirable-download.mozilla.org/"),
+    1
+  );
+  Assert.equal(visits_in_database(longurl), 1);
+  // Other visits should have been expired.
+  Assert.ok(!page_in_database("http://old.mozilla.org/"));
+  Assert.ok(!page_in_database("http://download.mozilla.org/"));
+  Assert.ok(!page_in_database("http://new.mozilla.org/"));
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_expire_icons() {
+  const dataUrl =
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAA" +
+    "AAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg==";
+
+  const entries = [
+    {
+      desc: "Not expired because recent",
+      page: "https://recent.notexpired.org/",
+      icon: "https://recent.notexpired.org/test_icon.png",
+      root: "https://recent.notexpired.org/favicon.ico",
+      iconExpired: false,
+      removed: false,
+    },
+    {
+      desc: "Not expired because recent, no root",
+      page: "https://recentnoroot.notexpired.org/",
+      icon: "https://recentnoroot.notexpired.org/test_icon.png",
+      iconExpired: false,
+      removed: false,
+    },
+    {
+      desc: "Expired because old with root",
+      page: "https://oldroot.expired.org/",
+      icon: "https://oldroot.expired.org/test_icon.png",
+      root: "https://oldroot.expired.org/favicon.ico",
+      iconExpired: true,
+      removed: true,
+    },
+    {
+      desc: "Not expired because bookmarked, even if old with root",
+      page: "https://oldrootbm.notexpired.org/",
+      icon: "https://oldrootbm.notexpired.org/test_icon.png",
+      root: "https://oldrootbm.notexpired.org/favicon.ico",
+      bookmarked: true,
+      iconExpired: true,
+      removed: false,
+    },
+    {
+      desc: "Not Expired because old but has no root",
+      page: "https://old.notexpired.org/",
+      icon: "https://old.notexpired.org/test_icon.png",
+      iconExpired: true,
+      removed: false,
+    },
+    {
+      desc: "Expired because it's an orphan page",
+      page: "http://root.ref.org/#test",
+      icon: undefined,
+      iconExpired: false,
+      removed: true,
+    },
+    {
+      desc: "Expired because it's an orphan page",
+      page: "http://root.ref.org/#test",
+      icon: undefined,
+      skipHistory: true,
+      iconExpired: false,
+      removed: true,
+    },
+  ];
+
+  for (let entry of entries) {
+    if (!entry.skipHistory) {
+      await PlacesTestUtils.addVisits(entry.page);
+    }
+    if (entry.bookmarked) {
+      await PlacesUtils.bookmarks.insert({
+        url: entry.page,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      });
+    }
+
+    if (entry.icon) {
+      PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+        Services.io.newURI(entry.icon),
+        dataUrl,
+        0,
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+      await PlacesTestUtils.addFavicons(new Map([[entry.page, entry.icon]]));
+      Assert.equal(
+        await getFaviconUrlForPage(entry.page),
+        entry.icon,
+        "Sanity check the icon exists"
+      );
+    } else {
+      // This is an orphan page entry.
+      await PlacesUtils.withConnectionWrapper("addOrphanPage", async db => {
+        await db.execute(
+          `INSERT INTO moz_pages_w_icons (page_url, page_url_hash)
+           VALUES (:url, hash(:url))`,
+          { url: entry.page }
+        );
+      });
+    }
+
+    if (entry.root) {
+      PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+        Services.io.newURI(entry.root),
+        dataUrl,
+        0,
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+      await PlacesTestUtils.addFavicons(new Map([[entry.page, entry.root]]));
+    }
+
+    if (entry.iconExpired) {
+      // Set an expired time on the icon.
+      await PlacesUtils.withConnectionWrapper("expireFavicon", async db => {
+        await db.execute(
+          `UPDATE moz_icons_to_pages SET expire_ms = 1
+           WHERE icon_id = (SELECT id FROM moz_icons WHERE icon_url = :url)`,
+          { url: entry.icon }
+        );
+        if (entry.root) {
+          await db.execute(
+            `UPDATE moz_icons SET expire_ms = 1 WHERE icon_url = :url`,
+            { url: entry.root }
+          );
+        }
+      });
+    }
+    if (entry.icon) {
+      Assert.equal(
+        await getFaviconUrlForPage(entry.page),
+        entry.icon,
+        "Sanity check the initial icon value"
+      );
+    }
+  }
+
+  info("Run expiration");
+  await promiseForceExpirationStep(-1);
+
+  info("Check expiration");
+  for (let entry of entries) {
+    Assert.ok(page_in_database(entry.page));
+
+    if (!entry.removed) {
+      Assert.equal(
+        await getFaviconUrlForPage(entry.page),
+        entry.icon,
+        entry.desc
+      );
+      continue;
+    }
+
+    if (entry.root) {
+      Assert.equal(
+        await getFaviconUrlForPage(entry.page),
+        entry.root,
+        entry.desc
+      );
+      continue;
+    }
+
+    if (entry.icon) {
+      await Assert.rejects(
+        getFaviconUrlForPage(entry.page),
+        /Unable to find an icon/,
+        entry.desc
+      );
+      continue;
+    }
+
+    // This was an orphan page entry.
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(
+      `SELECT count(*) FROM moz_pages_w_icons WHERE page_url_hash = hash(:url)`,
+      { url: entry.page }
+    );
+    Assert.equal(rows[0].getResultByIndex(0), 0, "Orphan page was removed");
+  }
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+add_setup(async function () {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+  // Set maxPages to a low value, so it's easy to go over it.
+  setMaxPages(1);
+});
diff --git a/toolkit/components/places/tests/expiration/test_idle_daily.js b/toolkit/components/places/tests/expiration/test_idle_daily.js
new file mode 100644
index 0000000000..11547e37dc
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_idle_daily.js
@@ -0,0 +1,22 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that expiration runs on idle-daily.
+
+add_task(async function test_expiration_on_idle_daily() {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  let expirationPromise = TestUtils.topicObserved(
+    PlacesUtils.TOPIC_EXPIRATION_FINISHED
+  );
+
+  let expire = Cc["@mozilla.org/places/expiration;1"].getService(
+    Ci.nsIObserver
+  );
+  expire.observe(null, "idle-daily", null);
+
+  await expirationPromise;
+});
diff --git a/toolkit/components/places/tests/expiration/test_interactions_expiration.js b/toolkit/components/places/tests/expiration/test_interactions_expiration.js
new file mode 100644
index 0000000000..67b4b466c3
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_interactions_expiration.js
@@ -0,0 +1,102 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Tests expiration of Places interactions data.
+ */
+// Number of days in the past where interactions will be expired.
+const EXPIRE_DAYS = 60;
+// Should be more recent than EXPIRED_DAYS.
+const RECENT_DATE = new Date() - (EXPIRE_DAYS - 1) * 86400000;
+
+add_task(async function setup() {
+  Services.prefs.setBoolPref("browser.places.interactions.enabled", true);
+  Services.prefs.setIntPref(
+    "browser.places.interactions.expireDays",
+    EXPIRE_DAYS
+  );
+});
+
+add_task(async function test_expire_interactions() {
+  // Add visits and metadata to 2 pages and force expiration.
+  await PlacesTestUtils.addVisits([
+    "https://expired.mozilla.org/",
+    "https://interactions-expired.mozilla.org/",
+    "https://some-interaction-expired.mozilla.org/",
+    "https://not-expired.mozilla.org/",
+  ]);
+  // Insert dummy interactions for all the pages.
+  await addDummyInteractions("https://removed.mozilla.org/", [0]);
+  await addDummyInteractions("https://interactions-expired.mozilla.org/", [
+    EXPIRE_DAYS + 10,
+  ]);
+  await addDummyInteractions("https://some-interactions-expired.mozilla.org/", [
+    0,
+    EXPIRE_DAYS + 10,
+  ]);
+  await addDummyInteractions("https://not-expired.mozilla.org/", [
+    0,
+    EXPIRE_DAYS / 2,
+  ]);
+
+  info("Remove a page from history and check interactions are removed");
+  await PlacesUtils.history.remove("https://removed.mozilla.org/");
+  await checkDummyInteractions("https://removed.mozilla.org/", 0);
+
+  // Expire now.
+  await promiseForceExpirationStep(-1);
+
+  info("Test interactions expiration result");
+  await checkDummyInteractions("https://interactions-expired.mozilla.org/", 0);
+  await checkDummyInteractions(
+    "https://some-interactions-expired.mozilla.org/",
+    1
+  );
+  await checkDummyInteractions("https://not-expired.mozilla.org/", 2);
+
+  // Clean up.
+  await PlacesUtils.history.clear();
+});
+
+async function addDummyInteractions(url, interactionDaysAgo) {
+  await PlacesTestUtils.addVisits(url);
+  await PlacesUtils.withConnectionWrapper(
+    "test_interactions_expiration.js: addDummyInteraction",
+    async db => {
+      await db.execute(
+        `INSERT INTO moz_places_metadata (place_id, created_at, updated_at) VALUES (
+        (SELECT id FROM moz_places WHERE url_hash = hash(:url)),
+        strftime('%s','now','localtime','-' || :days || ' day','start of day','utc') * 1000,
+        strftime('%s','now','localtime','-' || :days || ' day','start of day','utc') * 1000
+      )`,
+        interactionDaysAgo.map(days => ({ url, days }))
+      );
+    }
+  );
+}
+
+async function checkDummyInteractions(url, interactionsLen) {
+  info("Check interactions for " + url);
+  await PlacesUtils.withConnectionWrapper(
+    "test_interactions_expiration.js: addDummyInteraction",
+    async db => {
+      let rows = await db.execute(
+        `SELECT updated_at
+         FROM moz_places_metadata
+         WHERE place_id = (SELECT id FROM moz_places WHERE url_hash = hash(:url))
+         ORDER BY updated_at DESC`,
+        { url }
+      );
+      let dates = rows.map(r => new Date(r.getResultByName("updated_at")));
+      Assert.equal(
+        rows.length,
+        interactionsLen,
+        "Found expected number of interactions"
+      );
+      Assert.ok(
+        dates.every(d => d > RECENT_DATE),
+        "All interactions are recent"
+      );
+    }
+  );
+}
diff --git a/toolkit/components/places/tests/expiration/test_notifications.js b/toolkit/components/places/tests/expiration/test_notifications.js
new file mode 100644
index 0000000000..d52319a9c9
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_notifications.js
@@ -0,0 +1,36 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * What this is aimed to test:
+ *
+ * Ensure that History (through category cache) notifies us just once.
+ */
+
+var gObserver = {
+  notifications: 0,
+  observe(aSubject, aTopic, aData) {
+    this.notifications++;
+  },
+};
+Services.obs.addObserver(gObserver, PlacesUtils.TOPIC_EXPIRATION_FINISHED);
+
+add_task(async function test_history_expirations_notify_just_once() {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  promiseForceExpirationStep(1);
+
+  await new Promise(resolve => {
+    do_timeout(2000, resolve);
+  });
+
+  Assert.equal(gObserver.notifications, 1);
+
+  Services.obs.removeObserver(gObserver, PlacesUtils.TOPIC_EXPIRATION_FINISHED);
+});
diff --git a/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_allVisits.js b/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_allVisits.js
new file mode 100644
index 0000000000..172f29cf96
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_allVisits.js
@@ -0,0 +1,156 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * What this is aimed to test:
+ *
+ * Expiring only visits for a page, but not the full page, should fire an
+ * page-removed for all visits notification.
+ */
+
+var tests = [
+  {
+    desc: "Add 1 bookmarked page.",
+    addPages: 1,
+    visitsPerPage: 1,
+    addBookmarks: 1,
+    limitExpiration: -1,
+    expectedNotifications: 1, // Will expire visits for 1 page.
+    expectedIsPartialRemoval: true,
+  },
+
+  {
+    desc: "Add 2 pages, 1 bookmarked.",
+    addPages: 2,
+    visitsPerPage: 1,
+    addBookmarks: 1,
+    limitExpiration: -1,
+    expectedNotifications: 1, // Will expire visits for 1 page.
+    expectedIsPartialRemoval: true,
+  },
+
+  {
+    desc: "Add 10 pages, none bookmarked.",
+    addPages: 10,
+    visitsPerPage: 1,
+    addBookmarks: 0,
+    limitExpiration: -1,
+    expectedNotifications: 0, // Will expire only full pages.
+    expectedIsPartialRemoval: false,
+  },
+
+  {
+    desc: "Add 10 pages, all bookmarked.",
+    addPages: 10,
+    visitsPerPage: 1,
+    addBookmarks: 10,
+    limitExpiration: -1,
+    expectedNotifications: 10, // Will expire visits for all pages.
+    expectedIsPartialRemoval: true,
+  },
+
+  {
+    desc: "Add 10 pages with lot of visits, none bookmarked.",
+    addPages: 10,
+    visitsPerPage: 10,
+    addBookmarks: 0,
+    limitExpiration: 10,
+    expectedNotifications: 10, // Will expire 1 visit for each page, but won't
+    // expire pages since they still have visits.
+    expectedIsPartialRemoval: true,
+  },
+];
+
+add_task(async () => {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  // Expire anything that is expirable.
+  setMaxPages(0);
+
+  for (let testIndex = 1; testIndex <= tests.length; testIndex++) {
+    let currentTest = tests[testIndex - 1];
+    info("TEST " + testIndex + ": " + currentTest.desc);
+    currentTest.receivedNotifications = 0;
+
+    // Setup visits.
+    let timeInMicroseconds = getExpirablePRTime(8);
+
+    function newTimeInMicroseconds() {
+      timeInMicroseconds = timeInMicroseconds + 1000;
+      return timeInMicroseconds;
+    }
+
+    for (let j = 0; j < currentTest.visitsPerPage; j++) {
+      for (let i = 0; i < currentTest.addPages; i++) {
+        let page = "http://" + testIndex + "." + i + ".mozilla.org/";
+        await PlacesTestUtils.addVisits({
+          uri: uri(page),
+          visitDate: newTimeInMicroseconds(),
+        });
+      }
+    }
+
+    // Setup bookmarks.
+    currentTest.bookmarks = [];
+    for (let i = 0; i < currentTest.addBookmarks; i++) {
+      let page = "http://" + testIndex + "." + i + ".mozilla.org/";
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        title: null,
+        url: page,
+      });
+      currentTest.bookmarks.push(page);
+    }
+
+    // Observe history.
+    let notificationsHandled = new Promise(resolve => {
+      const listener = async events => {
+        for (const event of events) {
+          Assert.equal(event.type, "page-removed");
+          Assert.equal(event.reason, PlacesVisitRemoved.REASON_EXPIRED);
+
+          if (event.isRemovedFromStore) {
+            // Check this uri was not bookmarked.
+            Assert.equal(currentTest.bookmarks.indexOf(event.url), -1);
+            do_check_valid_places_guid(event.pageGuid);
+          } else {
+            currentTest.receivedNotifications++;
+            await check_guid_for_uri(
+              Services.io.newURI(event.url),
+              event.pageGuid
+            );
+            Assert.equal(
+              event.isPartialVisistsRemoval,
+              currentTest.expectedIsPartialRemoval,
+              "Should have the correct flag setting for partial removal"
+            );
+          }
+        }
+        PlacesObservers.removeListener(["page-removed"], listener);
+        resolve();
+      };
+      PlacesObservers.addListener(["page-removed"], listener);
+    });
+
+    // Expire now.
+    await promiseForceExpirationStep(currentTest.limitExpiration);
+    await notificationsHandled;
+
+    Assert.equal(
+      currentTest.receivedNotifications,
+      currentTest.expectedNotifications
+    );
+
+    // Clean up.
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  }
+
+  clearMaxPages();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_fromStore.js b/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_fromStore.js
new file mode 100644
index 0000000000..c8f7cf4aa0
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_notifications_pageRemoved_fromStore.js
@@ -0,0 +1,110 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * What this is aimed to test:
+ *
+ * Expiring a full page should fire an page-removed event notification.
+ */
+
+var tests = [
+  {
+    desc: "Add 1 bookmarked page.",
+    addPages: 1,
+    addBookmarks: 1,
+    expectedNotifications: 0, // No expirable pages.
+  },
+
+  {
+    desc: "Add 2 pages, 1 bookmarked.",
+    addPages: 2,
+    addBookmarks: 1,
+    expectedNotifications: 1, // Only one expirable page.
+  },
+
+  {
+    desc: "Add 10 pages, none bookmarked.",
+    addPages: 10,
+    addBookmarks: 0,
+    expectedNotifications: 10, // Will expire everything.
+  },
+
+  {
+    desc: "Add 10 pages, all bookmarked.",
+    addPages: 10,
+    addBookmarks: 10,
+    expectedNotifications: 0, // No expirable pages.
+  },
+];
+
+add_task(async () => {
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  // Expire anything that is expirable.
+  setMaxPages(0);
+
+  for (let testIndex = 1; testIndex <= tests.length; testIndex++) {
+    let currentTest = tests[testIndex - 1];
+    print("\nTEST " + testIndex + ": " + currentTest.desc);
+    currentTest.receivedNotifications = 0;
+
+    // Setup visits.
+    let now = getExpirablePRTime();
+    for (let i = 0; i < currentTest.addPages; i++) {
+      let page = "http://" + testIndex + "." + i + ".mozilla.org/";
+      await PlacesTestUtils.addVisits({ uri: uri(page), visitDate: now++ });
+    }
+
+    // Setup bookmarks.
+    currentTest.bookmarks = [];
+    for (let i = 0; i < currentTest.addBookmarks; i++) {
+      let page = "http://" + testIndex + "." + i + ".mozilla.org/";
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        title: null,
+        url: page,
+      });
+      currentTest.bookmarks.push(page);
+    }
+
+    // Observe history.
+    const listener = events => {
+      for (const event of events) {
+        Assert.equal(event.type, "page-removed");
+
+        if (!event.isRemovedFromStore) {
+          continue;
+        }
+
+        currentTest.receivedNotifications++;
+        // Check this uri was not bookmarked.
+        Assert.equal(currentTest.bookmarks.indexOf(event.url), -1);
+        do_check_valid_places_guid(event.pageGuid);
+        Assert.equal(event.reason, PlacesVisitRemoved.REASON_EXPIRED);
+      }
+    };
+    PlacesObservers.addListener(["page-removed"], listener);
+
+    // Expire now.
+    await promiseForceExpirationStep(-1);
+
+    PlacesObservers.removeListener(["page-removed"], listener);
+
+    Assert.equal(
+      currentTest.receivedNotifications,
+      currentTest.expectedNotifications
+    );
+
+    // Clean up.
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  }
+
+  clearMaxPages();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/expiration/test_pref_interval.js b/toolkit/components/places/tests/expiration/test_pref_interval.js
new file mode 100644
index 0000000000..5bf340e7c4
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_pref_interval.js
@@ -0,0 +1,62 @@
+/**
+ * What this is aimed to test:
+ *
+ * Expiration relies on an interval, that is user-preffable setting
+ * "places.history.expiration.interval_seconds".
+ * On pref change it will stop current interval timer and fire a new one,
+ * that will obey the new value.
+ * If the pref is set to a number <= 0 we will use the default value.
+ */
+
+// Default timer value for expiration in seconds.  Must have same value as
+// PREF_INTERVAL_SECONDS_NOTSET in nsPlacesExpiration.
+const DEFAULT_TIMER_DELAY_SECONDS = 3 * 60;
+
+// Sync this with the const value in the component.
+const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3;
+
+var tests = [
+  {
+    desc: "Set interval to 1s.",
+    interval: 1,
+    expectedTimerDelay: 1 * EXPIRE_AGGRESSIVITY_MULTIPLIER,
+  },
+
+  {
+    desc: "Set interval to a negative value.",
+    interval: -1,
+    expectedTimerDelay:
+      DEFAULT_TIMER_DELAY_SECONDS * EXPIRE_AGGRESSIVITY_MULTIPLIER,
+  },
+
+  {
+    desc: "Set interval to 0.",
+    interval: 0,
+    expectedTimerDelay:
+      DEFAULT_TIMER_DELAY_SECONDS * EXPIRE_AGGRESSIVITY_MULTIPLIER,
+  },
+
+  {
+    desc: "Set interval to a large value.",
+    interval: 100,
+    expectedTimerDelay: 100 * EXPIRE_AGGRESSIVITY_MULTIPLIER,
+  },
+];
+
+add_task(async function test() {
+  // The pref should not exist by default.
+  Assert.throws(() => getInterval(), /NS_ERROR_UNEXPECTED/);
+
+  // Force the component, so it will start observing preferences.
+  force_expiration_start();
+
+  for (let currentTest of tests) {
+    currentTest = tests.shift();
+    print(currentTest.desc);
+    let promise = promiseTopicObserved("test-interval-changed");
+    setInterval(currentTest.interval);
+    let [, data] = await promise;
+    Assert.equal(data, currentTest.expectedTimerDelay);
+  }
+  clearInterval();
+});
diff --git a/toolkit/components/places/tests/expiration/test_pref_maxpages.js b/toolkit/components/places/tests/expiration/test_pref_maxpages.js
new file mode 100644
index 0000000000..e4583359f1
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/test_pref_maxpages.js
@@ -0,0 +1,116 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * What this is aimed to test:
+ *
+ * Expiration will obey to hardware spec, but user can set a custom maximum
+ * number of pages to retain, to restrict history, through
+ * "places.history.expiration.max_pages".
+ * This limit is used at next expiration run.
+ * If the pref is set to a number < 0 we will use the default value.
+ */
+
+var tests = [
+  {
+    desc: "Set max_pages to a negative value, with 1 page.",
+    maxPages: -1,
+    addPages: 1,
+    expectedNotifications: 0, // Will ignore and won't expire anything.
+  },
+
+  {
+    desc: "Set max_pages to 0.",
+    maxPages: 0,
+    addPages: 1,
+    expectedNotifications: 1,
+  },
+
+  {
+    desc: "Set max_pages to 0, with 2 pages.",
+    maxPages: 0,
+    addPages: 2,
+    expectedNotifications: 2, // Will expire everything.
+  },
+
+  // Notice if we are over limit we do a full step of expiration.  So we ensure
+  // that we will expire if we are over the limit, but we don't ensure that we
+  // will expire exactly up to the limit.  Thus in this case we expire
+  // everything.
+  {
+    desc: "Set max_pages to 1 with 2 pages.",
+    maxPages: 1,
+    addPages: 2,
+    expectedNotifications: 2, // Will expire everything (in this case).
+  },
+
+  {
+    desc: "Set max_pages to 10, with 9 pages.",
+    maxPages: 10,
+    addPages: 9,
+    expectedNotifications: 0, // We are at the limit, won't expire anything.
+  },
+
+  {
+    desc: "Set max_pages to 10 with 10 pages.",
+    maxPages: 10,
+    addPages: 10,
+    expectedNotifications: 0, // We are below the limit, won't expire anything.
+  },
+];
+
+add_task(async function test_pref_maxpages() {
+  // The pref should not exist by default.
+  try {
+    getMaxPages();
+    do_throw("interval pref should not exist by default");
+  } catch (ex) {}
+
+  // Set interval to a large value so we don't expire on it.
+  setInterval(3600); // 1h
+
+  for (let testIndex = 1; testIndex <= tests.length; testIndex++) {
+    let currentTest = tests[testIndex - 1];
+    print("\nTEST " + testIndex + ": " + currentTest.desc);
+    currentTest.receivedNotifications = 0;
+
+    // Setup visits.
+    let now = getExpirablePRTime();
+    for (let i = 0; i < currentTest.addPages; i++) {
+      let page = "http://" + testIndex + "." + i + ".mozilla.org/";
+      await PlacesTestUtils.addVisits({ uri: uri(page), visitDate: now-- });
+    }
+
+    const listener = events => {
+      for (const event of events) {
+        print("page-removed " + event.url);
+        Assert.equal(event.type, "page-removed");
+        Assert.ok(event.isRemovedFromStore);
+        Assert.equal(event.reason, PlacesVisitRemoved.REASON_EXPIRED);
+        currentTest.receivedNotifications++;
+      }
+    };
+    PlacesObservers.addListener(["page-removed"], listener);
+
+    setMaxPages(currentTest.maxPages);
+
+    // Expire now.
+    await promiseForceExpirationStep(-1);
+
+    PlacesObservers.removeListener(["page-removed"], listener);
+
+    Assert.equal(
+      currentTest.receivedNotifications,
+      currentTest.expectedNotifications
+    );
+
+    // Clean up.
+    await PlacesUtils.history.clear();
+  }
+
+  clearMaxPages();
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/expiration/xpcshell.ini b/toolkit/components/places/tests/expiration/xpcshell.ini
new file mode 100644
index 0000000000..72d276685c
--- /dev/null
+++ b/toolkit/components/places/tests/expiration/xpcshell.ini
@@ -0,0 +1,14 @@
+[DEFAULT]
+head = head_expiration.js
+skip-if = toolkit == 'android'
+
+[test_annos_expire_never.js]
+[test_clearHistory.js]
+[test_debug_expiration.js]
+[test_idle_daily.js]
+[test_interactions_expiration.js]
+[test_notifications.js]
+[test_notifications_pageRemoved_allVisits.js]
+[test_notifications_pageRemoved_fromStore.js]
+[test_pref_interval.js]
+[test_pref_maxpages.js]
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-animated16.png.png b/toolkit/components/places/tests/favicons/expected-favicon-animated16.png.png
new file mode 100644
index 0000000000..22f825c500
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-animated16.png.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-big16.ico.png b/toolkit/components/places/tests/favicons/expected-favicon-big16.ico.png
new file mode 100644
index 0000000000..fa61cc5046
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-big16.ico.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-big32.jpg.png b/toolkit/components/places/tests/favicons/expected-favicon-big32.jpg.png
new file mode 100644
index 0000000000..42640cbb53
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-big32.jpg.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-big4.jpg.png b/toolkit/components/places/tests/favicons/expected-favicon-big4.jpg.png
new file mode 100644
index 0000000000..81d1b8ae19
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-big4.jpg.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-big48.ico.png b/toolkit/components/places/tests/favicons/expected-favicon-big48.ico.png
new file mode 100644
index 0000000000..7983889098
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-big48.ico.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-big64.png.png b/toolkit/components/places/tests/favicons/expected-favicon-big64.png.png
new file mode 100644
index 0000000000..2756cf0cb3
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-big64.png.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-scale160x3.jpg.png b/toolkit/components/places/tests/favicons/expected-favicon-scale160x3.jpg.png
new file mode 100644
index 0000000000..fc464f8e99
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-scale160x3.jpg.png differ
diff --git a/toolkit/components/places/tests/favicons/expected-favicon-scale3x160.jpg.png b/toolkit/components/places/tests/favicons/expected-favicon-scale3x160.jpg.png
new file mode 100644
index 0000000000..c1412038a3
Binary files /dev/null and b/toolkit/components/places/tests/favicons/expected-favicon-scale3x160.jpg.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-animated16.png b/toolkit/components/places/tests/favicons/favicon-animated16.png
new file mode 100644
index 0000000000..8913387fc9
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-animated16.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-big16.ico b/toolkit/components/places/tests/favicons/favicon-big16.ico
new file mode 100644
index 0000000000..d44438903b
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-big16.ico differ
diff --git a/toolkit/components/places/tests/favicons/favicon-big32.jpg b/toolkit/components/places/tests/favicons/favicon-big32.jpg
new file mode 100644
index 0000000000..b2131bf0c1
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-big32.jpg differ
diff --git a/toolkit/components/places/tests/favicons/favicon-big4.jpg b/toolkit/components/places/tests/favicons/favicon-big4.jpg
new file mode 100644
index 0000000000..b84fcd35a6
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-big4.jpg differ
diff --git a/toolkit/components/places/tests/favicons/favicon-big48.ico b/toolkit/components/places/tests/favicons/favicon-big48.ico
new file mode 100644
index 0000000000..f22522411d
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-big48.ico differ
diff --git a/toolkit/components/places/tests/favicons/favicon-big64.png b/toolkit/components/places/tests/favicons/favicon-big64.png
new file mode 100644
index 0000000000..2756cf0cb3
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-big64.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-multi-frame16.png b/toolkit/components/places/tests/favicons/favicon-multi-frame16.png
new file mode 100644
index 0000000000..519e08cc21
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-multi-frame16.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-multi-frame32.png b/toolkit/components/places/tests/favicons/favicon-multi-frame32.png
new file mode 100644
index 0000000000..5ae61de789
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-multi-frame32.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-multi-frame64.png b/toolkit/components/places/tests/favicons/favicon-multi-frame64.png
new file mode 100644
index 0000000000..57123f351b
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-multi-frame64.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-multi.ico b/toolkit/components/places/tests/favicons/favicon-multi.ico
new file mode 100644
index 0000000000..e98adcafeb
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-multi.ico differ
diff --git a/toolkit/components/places/tests/favicons/favicon-normal16.png b/toolkit/components/places/tests/favicons/favicon-normal16.png
new file mode 100644
index 0000000000..62b69a3d03
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-normal16.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-normal32.png b/toolkit/components/places/tests/favicons/favicon-normal32.png
new file mode 100644
index 0000000000..5535363c94
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-normal32.png differ
diff --git a/toolkit/components/places/tests/favicons/favicon-scale160x3.jpg b/toolkit/components/places/tests/favicons/favicon-scale160x3.jpg
new file mode 100644
index 0000000000..422ee7ea0b
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-scale160x3.jpg differ
diff --git a/toolkit/components/places/tests/favicons/favicon-scale3x160.jpg b/toolkit/components/places/tests/favicons/favicon-scale3x160.jpg
new file mode 100644
index 0000000000..e8514966a0
Binary files /dev/null and b/toolkit/components/places/tests/favicons/favicon-scale3x160.jpg differ
diff --git a/toolkit/components/places/tests/favicons/head_favicons.js b/toolkit/components/places/tests/favicons/head_favicons.js
new file mode 100644
index 0000000000..d8109c66e0
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/head_favicons.js
@@ -0,0 +1,81 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+const systemPrincipal = Services.scriptSecurityManager.getSystemPrincipal();
+
+/**
+ * Checks that the favicon for the given page matches the provided data.
+ *
+ * @param aPageURI
+ *        nsIURI object for the page to check.
+ * @param aExpectedMimeType
+ *        Expected MIME type of the icon, for example "image/png".
+ * @param aExpectedData
+ *        Expected icon data, expressed as an array of byte values.
+ * @param aCallback
+ *        This function is called after the check finished.
+ */
+function checkFaviconDataForPage(
+  aPageURI,
+  aExpectedMimeType,
+  aExpectedData,
+  aCallback
+) {
+  PlacesUtils.favicons.getFaviconDataForPage(
+    aPageURI,
+    async function (aURI, aDataLen, aData, aMimeType) {
+      Assert.equal(aExpectedMimeType, aMimeType);
+      Assert.ok(compareArrays(aExpectedData, aData));
+      await check_guid_for_uri(aPageURI);
+      aCallback();
+    }
+  );
+}
+
+/**
+ * Checks that the given page has no associated favicon.
+ *
+ * @param aPageURI
+ *        nsIURI object for the page to check.
+ * @param aCallback
+ *        This function is called after the check finished.
+ */
+function checkFaviconMissingForPage(aPageURI, aCallback) {
+  PlacesUtils.favicons.getFaviconURLForPage(
+    aPageURI,
+    function (aURI, aDataLen, aData, aMimeType) {
+      Assert.ok(aURI === null);
+      aCallback();
+    }
+  );
+}
+
+function promiseFaviconMissingForPage(aPageURI) {
+  return new Promise(resolve => checkFaviconMissingForPage(aPageURI, resolve));
+}
+
+function promiseFaviconChanged(aExpectedPageURI, aExpectedFaviconURI) {
+  return new Promise(resolve => {
+    PlacesTestUtils.waitForNotification("favicon-changed", async events => {
+      for (let e of events) {
+        if (e.url == aExpectedPageURI.spec) {
+          Assert.equal(e.faviconUrl, aExpectedFaviconURI.spec);
+          await check_guid_for_uri(aExpectedPageURI, e.pageGuid);
+          resolve();
+        }
+      }
+    });
+  });
+}
diff --git a/toolkit/components/places/tests/favicons/noise.png b/toolkit/components/places/tests/favicons/noise.png
new file mode 100644
index 0000000000..d6876295cd
Binary files /dev/null and b/toolkit/components/places/tests/favicons/noise.png differ
diff --git a/toolkit/components/places/tests/favicons/test_copyFavicons.js b/toolkit/components/places/tests/favicons/test_copyFavicons.js
new file mode 100644
index 0000000000..687b799a4b
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_copyFavicons.js
@@ -0,0 +1,166 @@
+const TEST_URI1 = Services.io.newURI("http://mozilla.com/");
+const TEST_URI2 = Services.io.newURI("http://places.com/");
+const TEST_URI3 = Services.io.newURI("http://bookmarked.com/");
+const LOAD_NON_PRIVATE = PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE;
+const LOAD_PRIVATE = PlacesUtils.favicons.FAVICON_LOAD_PRIVATE;
+
+function copyFavicons(source, dest, inPrivate) {
+  return new Promise(resolve => {
+    PlacesUtils.favicons.copyFavicons(
+      source,
+      dest,
+      inPrivate ? LOAD_PRIVATE : LOAD_NON_PRIVATE,
+      resolve
+    );
+  });
+}
+
+function promisePageChanged(url) {
+  return PlacesTestUtils.waitForNotification("favicon-changed", events =>
+    events.some(e => e.url == url)
+  );
+}
+
+add_task(async function test_copyFavicons_inputcheck() {
+  Assert.throws(
+    () => PlacesUtils.favicons.copyFavicons(null, TEST_URI2, LOAD_PRIVATE),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => PlacesUtils.favicons.copyFavicons(TEST_URI1, null, LOAD_PRIVATE),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => PlacesUtils.favicons.copyFavicons(TEST_URI1, TEST_URI2, 3),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => PlacesUtils.favicons.copyFavicons(TEST_URI1, TEST_URI2, -1),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => PlacesUtils.favicons.copyFavicons(TEST_URI1, TEST_URI2, null),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+});
+
+add_task(async function test_copyFavicons_noop() {
+  info("Unknown uris");
+  Assert.equal(
+    await copyFavicons(TEST_URI1, TEST_URI2, false),
+    null,
+    "Icon should not have been copied"
+  );
+
+  info("Unknown dest uri");
+  await PlacesTestUtils.addVisits(TEST_URI1);
+  Assert.equal(
+    await copyFavicons(TEST_URI1, TEST_URI2, false),
+    null,
+    "Icon should not have been copied"
+  );
+
+  info("Unknown dest uri");
+  await PlacesTestUtils.addVisits(TEST_URI1);
+  Assert.equal(
+    await copyFavicons(TEST_URI1, TEST_URI2, false),
+    null,
+    "Icon should not have been copied"
+  );
+
+  info("Unknown dest uri, source has icon");
+  await setFaviconForPage(TEST_URI1, SMALLPNG_DATA_URI);
+  Assert.equal(
+    await copyFavicons(TEST_URI1, TEST_URI2, false),
+    null,
+    "Icon should not have been copied"
+  );
+
+  info("Known uris, source has icon, private");
+  await PlacesTestUtils.addVisits(TEST_URI2);
+  Assert.equal(
+    await copyFavicons(TEST_URI1, TEST_URI2, true),
+    null,
+    "Icon should not have been copied"
+  );
+
+  PlacesUtils.favicons.expireAllFavicons();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_copyFavicons() {
+  info("Normal copy across 2 pages");
+  await PlacesTestUtils.addVisits(TEST_URI1);
+  await setFaviconForPage(TEST_URI1, SMALLPNG_DATA_URI);
+  await setFaviconForPage(TEST_URI1, SMALLSVG_DATA_URI);
+  await PlacesTestUtils.addVisits(TEST_URI2);
+  let promiseChange = promisePageChanged(TEST_URI2.spec);
+  Assert.equal(
+    (await copyFavicons(TEST_URI1, TEST_URI2, false)).spec,
+    SMALLSVG_DATA_URI.spec,
+    "Icon should have been copied"
+  );
+  await promiseChange;
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI2, 1),
+    SMALLPNG_DATA_URI.spec,
+    "Small icon found"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI2),
+    SMALLSVG_DATA_URI.spec,
+    "Large icon found"
+  );
+
+  info("Private copy to a bookmarked page");
+  await PlacesUtils.bookmarks.insert({
+    url: TEST_URI3,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  promiseChange = promisePageChanged(TEST_URI3.spec);
+  Assert.equal(
+    (await copyFavicons(TEST_URI1, TEST_URI3, true)).spec,
+    SMALLSVG_DATA_URI.spec,
+    "Icon should have been copied"
+  );
+  await promiseChange;
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI3, 1),
+    SMALLPNG_DATA_URI.spec,
+    "Small icon found"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI3),
+    SMALLSVG_DATA_URI.spec,
+    "Large icon found"
+  );
+
+  PlacesUtils.favicons.expireAllFavicons();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_copyFavicons_overlap() {
+  info("Copy to a page that has one of the favicons already");
+  await PlacesTestUtils.addVisits(TEST_URI1);
+  await setFaviconForPage(TEST_URI1, SMALLPNG_DATA_URI);
+  await setFaviconForPage(TEST_URI1, SMALLSVG_DATA_URI);
+  await PlacesTestUtils.addVisits(TEST_URI2);
+  await setFaviconForPage(TEST_URI2, SMALLPNG_DATA_URI);
+  let promiseChange = promisePageChanged(TEST_URI2.spec);
+  Assert.equal(
+    (await copyFavicons(TEST_URI1, TEST_URI2, false)).spec,
+    SMALLSVG_DATA_URI.spec,
+    "Icon should have been copied"
+  );
+  await promiseChange;
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI2, 1),
+    SMALLPNG_DATA_URI.spec,
+    "Small icon found"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URI2),
+    SMALLSVG_DATA_URI.spec,
+    "Large icon found"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_expireAllFavicons.js b/toolkit/components/places/tests/favicons/test_expireAllFavicons.js
new file mode 100644
index 0000000000..73c3ca6e4b
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_expireAllFavicons.js
@@ -0,0 +1,38 @@
+/**
+ * This file tests that favicons are correctly expired by expireAllFavicons.
+ */
+
+"use strict";
+
+const TEST_PAGE_URI = NetUtil.newURI("http://example.com/");
+const BOOKMARKED_PAGE_URI = NetUtil.newURI("http://example.com/bookmarked");
+
+add_task(async function test_expireAllFavicons() {
+  // Add a visited page.
+  await PlacesTestUtils.addVisits({
+    uri: TEST_PAGE_URI,
+    transition: TRANSITION_TYPED,
+  });
+
+  // Set a favicon for our test page.
+  await setFaviconForPage(TEST_PAGE_URI, SMALLPNG_DATA_URI);
+
+  // Add a page with a bookmark.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    url: BOOKMARKED_PAGE_URI,
+    title: "Test bookmark",
+  });
+
+  // Set a favicon for our bookmark.
+  await setFaviconForPage(BOOKMARKED_PAGE_URI, SMALLPNG_DATA_URI);
+
+  // Start expiration only after data has been saved in the database.
+  let promise = promiseTopicObserved(PlacesUtils.TOPIC_FAVICONS_EXPIRED);
+  PlacesUtils.favicons.expireAllFavicons();
+  await promise;
+
+  // Check that the favicons for the pages we added were removed.
+  await promiseFaviconMissingForPage(TEST_PAGE_URI);
+  await promiseFaviconMissingForPage(BOOKMARKED_PAGE_URI);
+});
diff --git a/toolkit/components/places/tests/favicons/test_expire_migrated_icons.js b/toolkit/components/places/tests/favicons/test_expire_migrated_icons.js
new file mode 100644
index 0000000000..00516a2a0b
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_expire_migrated_icons.js
@@ -0,0 +1,30 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests that favicons migrated from a previous profile, having a 0
+ * expiration, will be properly expired when fetching new ones.
+ */
+
+add_task(async function test_storing_a_normal_16x16_icon() {
+  const PAGE_URL = "http://places.test";
+  await PlacesTestUtils.addVisits(PAGE_URL);
+  await setFaviconForPage(PAGE_URL, SMALLPNG_DATA_URI);
+
+  // Now set expiration to 0 and change the payload.
+  info("Set expiration to 0 and replace favicon data");
+  await PlacesUtils.withConnectionWrapper("Change favicons payload", db => {
+    return db.execute(`UPDATE moz_icons SET expire_ms = 0, data = "test"`);
+  });
+
+  let { data, mimeType } = await getFaviconDataForPage(PAGE_URL);
+  Assert.equal(mimeType, "image/png");
+  Assert.deepEqual(
+    data,
+    "test".split("").map(c => c.charCodeAt(0))
+  );
+
+  info("Refresh favicon");
+  await setFaviconForPage(PAGE_URL, SMALLPNG_DATA_URI, false);
+  await compareFavicons("page-icon:" + PAGE_URL, SMALLPNG_DATA_URI);
+});
diff --git a/toolkit/components/places/tests/favicons/test_expire_on_new_icons.js b/toolkit/components/places/tests/favicons/test_expire_on_new_icons.js
new file mode 100644
index 0000000000..d5a7c42ba3
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_expire_on_new_icons.js
@@ -0,0 +1,151 @@
+/* Any copyright is dedicated to the Public Domain.
+ * https://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * Tests that adding new icons for a page expired old ones.
+ */
+
+add_task(async function test_expire_associated() {
+  const TEST_URL = "http://mozilla.com/";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  const TEST_URL2 = "http://test.mozilla.com/";
+  await PlacesTestUtils.addVisits(TEST_URL2);
+
+  let favicons = [
+    {
+      name: "favicon-normal16.png",
+      mimeType: "image/png",
+      expired: true,
+    },
+    {
+      name: "favicon-normal32.png",
+      mimeType: "image/png",
+    },
+    {
+      name: "favicon-big64.png",
+      mimeType: "image/png",
+    },
+  ];
+
+  for (let icon of favicons) {
+    let data = readFileData(do_get_file(icon.name));
+    PlacesUtils.favicons.replaceFaviconData(
+      NetUtil.newURI(TEST_URL + icon.name),
+      data,
+      icon.mimeType
+    );
+    await setFaviconForPage(TEST_URL, TEST_URL + icon.name);
+    if (icon.expired) {
+      await expireIconRelationsForPage(TEST_URL);
+      // Add the same icon to another page.
+      PlacesUtils.favicons.replaceFaviconData(
+        NetUtil.newURI(TEST_URL + icon.name),
+        data,
+        icon.mimeType,
+        icon.expire
+      );
+      await setFaviconForPage(TEST_URL2, TEST_URL + icon.name);
+    }
+  }
+
+  // Only the second and the third icons should have survived.
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URL, 16),
+    TEST_URL + favicons[1].name,
+    "Should retrieve the 32px icon, not the 16px one."
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URL, 64),
+    TEST_URL + favicons[2].name,
+    "Should retrieve the 64px icon"
+  );
+
+  // The expired icon for page 2 should have survived.
+  Assert.equal(
+    await getFaviconUrlForPage(TEST_URL2, 16),
+    TEST_URL + favicons[0].name,
+    "Should retrieve the expired 16px icon"
+  );
+});
+
+add_task(async function test_expire_root() {
+  async function countEntries(tablename) {
+    await PlacesTestUtils.promiseAsyncUpdates();
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute("SELECT * FROM " + tablename);
+    return rows.length;
+  }
+
+  // Clear the database.
+  let promise = promiseTopicObserved(PlacesUtils.TOPIC_FAVICONS_EXPIRED);
+  PlacesUtils.favicons.expireAllFavicons();
+  await promise;
+
+  Assert.equal(await countEntries("moz_icons"), 0, "There should be no icons");
+
+  let pageURI = NetUtil.newURI("http://root.mozilla.com/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  // Insert an expired icon.
+  let iconURI = NetUtil.newURI(pageURI.spec + "favicon-normal16.png");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    iconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, iconURI);
+  Assert.equal(
+    await countEntries("moz_icons_to_pages"),
+    1,
+    "There should be 1 association"
+  );
+  // Set an expired time on the icon-page relation.
+  await expireIconRelationsForPage(pageURI.spec);
+
+  // Now insert a new root icon.
+  let rootIconURI = NetUtil.newURI(pageURI.spec + "favicon.ico");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    rootIconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, rootIconURI);
+
+  // Only the root icon should have survived.
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI, 16),
+    rootIconURI.spec,
+    "Should retrieve the root icon."
+  );
+  Assert.equal(
+    await countEntries("moz_icons_to_pages"),
+    0,
+    "There should be no associations"
+  );
+});
+
+async function expireIconRelationsForPage(url) {
+  // Set an expired time on the icon-page relation.
+  await PlacesUtils.withConnectionWrapper("expireFavicon", async db => {
+    await db.execute(
+      `
+      UPDATE moz_icons_to_pages SET expire_ms = 0
+      WHERE page_id = (SELECT id FROM moz_pages_w_icons WHERE page_url = :url)
+      `,
+      { url }
+    );
+    // Also ensure the icon is not expired, here we should only replace entries
+    // based on their association expiration, not the icon expiration.
+    let count = (
+      await db.execute(
+        `
+        SELECT count(*) FROM moz_icons
+        WHERE expire_ms < strftime('%s','now','localtime','utc') * 1000
+        `
+      )
+    )[0].getResultByIndex(0);
+    Assert.equal(count, 0, "All the icons should have future expiration");
+  });
+}
diff --git a/toolkit/components/places/tests/favicons/test_favicons_conversions.js b/toolkit/components/places/tests/favicons/test_favicons_conversions.js
new file mode 100644
index 0000000000..28a0fffb7f
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_favicons_conversions.js
@@ -0,0 +1,192 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests the image conversions done by the favicon service.
+ */
+
+// Globals
+
+// The pixel values we get on Windows are sometimes +/- 1 value compared to
+// other platforms, so we need to skip some image content tests.
+var isWindows = "@mozilla.org/windows-registry-key;1" in Cc;
+
+/**
+ * Checks the conversion of the given test image file.
+ *
+ * @param aFileName
+ *        File that contains the favicon image, located in the test folder.
+ * @param aFileMimeType
+ *        MIME type of the image contained in the file.
+ * @param aFileLength
+ *        Expected length of the file.
+ * @param aExpectConversion
+ *        If false, the icon should be stored as is.  If true, the expected data
+ *        is loaded from a file named "expected-" + aFileName + ".png".
+ * @param aVaryOnWindows
+ *        Indicates that the content of the converted image can be different on
+ *        Windows and should not be checked on that platform.
+ * @param aCallback
+ *        This function is called after the check finished.
+ */
+async function checkFaviconDataConversion(
+  aFileName,
+  aFileMimeType,
+  aFileLength,
+  aExpectConversion,
+  aVaryOnWindows
+) {
+  let pageURI = NetUtil.newURI("http://places.test/page/" + aFileName);
+  await PlacesTestUtils.addVisits({
+    uri: pageURI,
+    transition: TRANSITION_TYPED,
+  });
+  let faviconURI = NetUtil.newURI("http://places.test/icon/" + aFileName);
+  let fileData = readFileOfLength(aFileName, aFileLength);
+
+  PlacesUtils.favicons.replaceFaviconData(faviconURI, fileData, aFileMimeType);
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      pageURI,
+      faviconURI,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      (aURI, aDataLen, aData, aMimeType) => {
+        if (!aExpectConversion) {
+          Assert.ok(compareArrays(aData, fileData));
+          Assert.equal(aMimeType, aFileMimeType);
+        } else {
+          if (!aVaryOnWindows || !isWindows) {
+            let expectedFile = do_get_file("expected-" + aFileName + ".png");
+            Assert.ok(compareArrays(aData, readFileData(expectedFile)));
+          }
+          Assert.equal(aMimeType, "image/png");
+        }
+        resolve();
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+}
+
+add_task(async function test_storing_a_normal_16x16_icon() {
+  // 16x16 png, 286 bytes.
+  // optimized: no
+  await checkFaviconDataConversion(
+    "favicon-normal16.png",
+    "image/png",
+    286,
+    false,
+    false
+  );
+});
+
+add_task(async function test_storing_a_normal_32x32_icon() {
+  // 32x32 png, 344 bytes.
+  // optimized: no
+  await checkFaviconDataConversion(
+    "favicon-normal32.png",
+    "image/png",
+    344,
+    false,
+    false
+  );
+});
+
+add_task(async function test_storing_a_big_16x16_icon() {
+  //  in: 16x16 ico, 1406 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-big16.ico",
+    "image/x-icon",
+    1406,
+    true,
+    false
+  );
+});
+
+add_task(async function test_storing_an_oversize_4x4_icon() {
+  //  in: 4x4 jpg, 4751 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-big4.jpg",
+    "image/jpeg",
+    4751,
+    true,
+    false
+  );
+});
+
+add_task(async function test_storing_an_oversize_32x32_icon() {
+  //  in: 32x32 jpg, 3494 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-big32.jpg",
+    "image/jpeg",
+    3494,
+    true,
+    true
+  );
+});
+
+add_task(async function test_storing_an_oversize_48x48_icon() {
+  //  in: 48x48 ico, 56646 bytes.
+  // (howstuffworks.com icon, contains 13 icons with sizes from 16x16 to
+  // 48x48 in varying depths)
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-big48.ico",
+    "image/x-icon",
+    56646,
+    true,
+    false
+  );
+});
+
+add_task(async function test_storing_an_oversize_64x64_icon() {
+  //  in: 64x64 png, 10698 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-big64.png",
+    "image/png",
+    10698,
+    true,
+    false
+  );
+});
+
+add_task(async function test_scaling_an_oversize_160x3_icon() {
+  //  in: 160x3 jpg, 5095 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-scale160x3.jpg",
+    "image/jpeg",
+    5095,
+    true,
+    false
+  );
+});
+
+add_task(async function test_scaling_an_oversize_3x160_icon() {
+  //  in: 3x160 jpg, 5059 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-scale3x160.jpg",
+    "image/jpeg",
+    5059,
+    true,
+    false
+  );
+});
+
+add_task(async function test_animated_16x16_icon() {
+  // in: 16x16 apng, 1791 bytes.
+  // optimized: yes
+  await checkFaviconDataConversion(
+    "favicon-animated16.png",
+    "image/png",
+    1791,
+    true,
+    false
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_favicons_protocols_ref.js b/toolkit/components/places/tests/favicons/test_favicons_protocols_ref.js
new file mode 100644
index 0000000000..a8e3774830
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_favicons_protocols_ref.js
@@ -0,0 +1,114 @@
+/**
+ * This file tests the size ref on the icons protocols.
+ */
+
+const PAGE_URL = "http://icon.mozilla.org/";
+const ICON16_URL = "http://places.test/favicon-normal16.png";
+const ICON32_URL = "http://places.test/favicon-normal32.png";
+
+add_task(async function () {
+  await PlacesTestUtils.addVisits(PAGE_URL);
+  // Add 2 differently sized favicons for this page.
+
+  let data = readFileData(do_get_file("favicon-normal16.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ICON16_URL),
+    data,
+    "image/png"
+  );
+  await setFaviconForPage(PAGE_URL, ICON16_URL);
+  data = readFileData(do_get_file("favicon-normal32.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ICON32_URL),
+    data,
+    "image/png"
+  );
+  await setFaviconForPage(PAGE_URL, ICON32_URL);
+
+  const PAGE_ICON_URL = "page-icon:" + PAGE_URL;
+
+  await compareFavicons(
+    PAGE_ICON_URL,
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Not specifying a ref should return the bigger icon"
+  );
+  // Fake window object.
+  let win = { devicePixelRatio: 1.0 };
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 16),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON16_URL)),
+    "Size=16 should return the 16px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 32),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=32 should return the 32px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 33),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=33 should return the 32px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 17),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=17 should return the 32px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 1),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON16_URL)),
+    "Size=1 should return the 16px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 0),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=0 should return the bigger icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, -1),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Invalid size should return the bigger icon"
+  );
+
+  // Add the icon also for the page with ref.
+  await PlacesTestUtils.addVisits(PAGE_URL + "#other§=12");
+  await setFaviconForPage(PAGE_URL + "#other§=12", ICON16_URL, false);
+  await setFaviconForPage(PAGE_URL + "#other§=12", ICON32_URL, false);
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL + "#other§=12", 16),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON16_URL)),
+    "Pre-existing refs should be retained"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL + "#other§=12", 32),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Pre-existing refs should be retained"
+  );
+
+  // If the ref-ed url is unknown, should still try to fetch icon for the unref-ed url.
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL + "#randomstuff", 32),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Non-existing refs should be ignored"
+  );
+
+  win = { devicePixelRatio: 1.1 };
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 16),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=16 with HIDPI should return the 32px icon"
+  );
+  await compareFavicons(
+    PlacesUtils.urlWithSizeRef(win, PAGE_ICON_URL, 32),
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON32_URL)),
+    "Size=32 with HIDPI should return the 32px icon"
+  );
+
+  // Check setting a different default preferred size works.
+  PlacesUtils.favicons.setDefaultIconURIPreferredSize(16);
+  await compareFavicons(
+    PAGE_ICON_URL,
+    PlacesUtils.favicons.getFaviconLinkForIcon(NetUtil.newURI(ICON16_URL)),
+    "Not specifying a ref should return the set default size icon"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_getFaviconDataForPage.js b/toolkit/components/places/tests/favicons/test_getFaviconDataForPage.js
new file mode 100644
index 0000000000..80f498f33f
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_getFaviconDataForPage.js
@@ -0,0 +1,131 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const FAVICON_URI = NetUtil.newURI(do_get_file("favicon-normal32.png"));
+const FAVICON_DATA = readFileData(do_get_file("favicon-normal32.png"));
+const FAVICON_MIMETYPE = "image/png";
+const ICON32_URL = "http://places.test/favicon-normal32.png";
+
+add_task(async function test_normal() {
+  Assert.equal(FAVICON_DATA.length, 344);
+  let pageURI = NetUtil.newURI("http://example.com/normal");
+
+  await PlacesTestUtils.addVisits(pageURI);
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      pageURI,
+      FAVICON_URI,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function () {
+        PlacesUtils.favicons.getFaviconDataForPage(
+          pageURI,
+          function (aURI, aDataLen, aData, aMimeType) {
+            Assert.ok(aURI.equals(FAVICON_URI));
+            Assert.equal(FAVICON_DATA.length, aDataLen);
+            Assert.ok(compareArrays(FAVICON_DATA, aData));
+            Assert.equal(FAVICON_MIMETYPE, aMimeType);
+            resolve();
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+});
+
+add_task(async function test_missing() {
+  let pageURI = NetUtil.newURI("http://example.com/missing");
+
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      pageURI,
+      function (aURI, aDataLen, aData, aMimeType) {
+        // Check also the expected data types.
+        Assert.ok(aURI === null);
+        Assert.ok(aDataLen === 0);
+        Assert.ok(aData.length === 0);
+        Assert.ok(aMimeType === "");
+        resolve();
+      }
+    );
+  });
+});
+
+add_task(async function test_fallback() {
+  const ROOT_URL = "https://www.example.com/";
+  const ROOT_ICON_URL = ROOT_URL + "favicon.ico";
+  const SUBPAGE_URL = ROOT_URL + "/missing";
+
+  info("Set icon for the root");
+  await PlacesTestUtils.addVisits(ROOT_URL);
+  let data = readFileData(do_get_file("favicon-normal16.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ROOT_ICON_URL),
+    data,
+    "image/png"
+  );
+  await setFaviconForPage(ROOT_URL, ROOT_ICON_URL);
+
+  info("check fallback icons");
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      NetUtil.newURI(ROOT_URL),
+      (aURI, aDataLen, aData, aMimeType) => {
+        Assert.equal(aURI.spec, ROOT_ICON_URL);
+        Assert.equal(aDataLen, data.length);
+        Assert.deepEqual(aData, data);
+        Assert.equal(aMimeType, "image/png");
+        resolve();
+      }
+    );
+  });
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      NetUtil.newURI(SUBPAGE_URL),
+      (aURI, aDataLen, aData, aMimeType) => {
+        Assert.equal(aURI.spec, ROOT_ICON_URL);
+        Assert.equal(aDataLen, data.length);
+        Assert.deepEqual(aData, data);
+        Assert.equal(aMimeType, "image/png");
+        resolve();
+      }
+    );
+  });
+
+  info("Now add a proper icon for the page");
+  await PlacesTestUtils.addVisits(SUBPAGE_URL);
+  let data32 = readFileData(do_get_file("favicon-normal32.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ICON32_URL),
+    data32,
+    "image/png"
+  );
+  await setFaviconForPage(SUBPAGE_URL, ICON32_URL);
+
+  info("check no fallback icons");
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      NetUtil.newURI(ROOT_URL),
+      (aURI, aDataLen, aData, aMimeType) => {
+        Assert.equal(aURI.spec, ROOT_ICON_URL);
+        Assert.equal(aDataLen, data.length);
+        Assert.deepEqual(aData, data);
+        Assert.equal(aMimeType, "image/png");
+        resolve();
+      }
+    );
+  });
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      NetUtil.newURI(SUBPAGE_URL),
+      (aURI, aDataLen, aData, aMimeType) => {
+        Assert.equal(aURI.spec, ICON32_URL);
+        Assert.equal(aDataLen, data32.length);
+        Assert.deepEqual(aData, data32);
+        Assert.equal(aMimeType, "image/png");
+        resolve();
+      }
+    );
+  });
+});
diff --git a/toolkit/components/places/tests/favicons/test_getFaviconURLForPage.js b/toolkit/components/places/tests/favicons/test_getFaviconURLForPage.js
new file mode 100644
index 0000000000..e8f459cb08
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_getFaviconURLForPage.js
@@ -0,0 +1,101 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const ICON32_URL = "http://places.test/favicon-normal32.png";
+
+add_task(async function test_normal() {
+  let pageURI = NetUtil.newURI("http://example.com/normal");
+
+  await PlacesTestUtils.addVisits(pageURI);
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      pageURI,
+      SMALLPNG_DATA_URI,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function () {
+        PlacesUtils.favicons.getFaviconURLForPage(
+          pageURI,
+          function (aURI, aDataLen, aData, aMimeType) {
+            Assert.ok(aURI.equals(SMALLPNG_DATA_URI));
+
+            // Check also the expected data types.
+            Assert.ok(aDataLen === 0);
+            Assert.ok(aData.length === 0);
+            Assert.ok(aMimeType === "");
+            resolve();
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+});
+
+add_task(async function test_missing() {
+  let pageURI = NetUtil.newURI("http://example.com/missing");
+
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconURLForPage(
+      pageURI,
+      function (aURI, aDataLen, aData, aMimeType) {
+        // Check also the expected data types.
+        Assert.ok(aURI === null);
+        Assert.ok(aDataLen === 0);
+        Assert.ok(aData.length === 0);
+        Assert.ok(aMimeType === "");
+        resolve();
+      }
+    );
+  });
+});
+
+add_task(async function test_fallback() {
+  const ROOT_URL = "https://www.example.com/";
+  const ROOT_ICON_URL = ROOT_URL + "favicon.ico";
+  const SUBPAGE_URL = ROOT_URL + "/missing";
+
+  info("Set icon for the root");
+  await PlacesTestUtils.addVisits(ROOT_URL);
+  let data = readFileData(do_get_file("favicon-normal16.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ROOT_ICON_URL),
+    data,
+    "image/png"
+  );
+  await setFaviconForPage(ROOT_URL, ROOT_ICON_URL);
+
+  info("check fallback icons");
+  Assert.equal(
+    await getFaviconUrlForPage(ROOT_URL),
+    ROOT_ICON_URL,
+    "The root should have its favicon"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(SUBPAGE_URL),
+    ROOT_ICON_URL,
+    "The page should fallback to the root icon"
+  );
+
+  info("Now add a proper icon for the page");
+  await PlacesTestUtils.addVisits(SUBPAGE_URL);
+  let data32 = readFileData(do_get_file("favicon-normal32.png"));
+  PlacesUtils.favicons.replaceFaviconData(
+    NetUtil.newURI(ICON32_URL),
+    data32,
+    "image/png"
+  );
+  await setFaviconForPage(SUBPAGE_URL, ICON32_URL);
+
+  info("check no fallback icons");
+  Assert.equal(
+    await getFaviconUrlForPage(ROOT_URL),
+    ROOT_ICON_URL,
+    "The root should still have its favicon"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(SUBPAGE_URL),
+    ICON32_URL,
+    "The page should also have its icon"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_heavy_favicon.js b/toolkit/components/places/tests/favicons/test_heavy_favicon.js
new file mode 100644
index 0000000000..09adcaf6fa
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_heavy_favicon.js
@@ -0,0 +1,36 @@
+/* Any copyright is dedicated to the Public Domain.
+ *    http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * Tests a png with a large file size that can't fit MAX_FAVICON_BUFFER_SIZE,
+ * it should be downsized until it can be stored, rather than thrown away.
+ */
+
+add_task(async function () {
+  let file = do_get_file("noise.png");
+  let icon = {
+    file,
+    uri: NetUtil.newURI(file),
+    data: readFileData(file),
+    mimetype: "image/png",
+  };
+
+  // If this should fail, it means MAX_FAVICON_BUFFER_SIZE has been made bigger
+  // than this icon. For this test to make sense the icon shoul always be
+  // bigger than MAX_FAVICON_BUFFER_SIZE. Please update the icon!
+  Assert.ok(
+    icon.data.length > Ci.nsIFaviconService.MAX_FAVICON_BUFFER_SIZE,
+    "The test icon file size must be larger than Ci.nsIFaviconService.MAX_FAVICON_BUFFER_SIZE"
+  );
+
+  let pageURI = uri("http://foo.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  PlacesUtils.favicons.replaceFaviconData(icon.uri, icon.data, icon.mimetype);
+  await setFaviconForPage(pageURI, icon.uri);
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI),
+    icon.uri.spec,
+    "A resampled version of the icon should be stored"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_incremental_vacuum.js b/toolkit/components/places/tests/favicons/test_incremental_vacuum.js
new file mode 100644
index 0000000000..ab93121d47
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_incremental_vacuum.js
@@ -0,0 +1,48 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests incremental vacuum of the favicons database.
+
+const { PlacesDBUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesDBUtils.sys.mjs"
+);
+
+add_task(async function () {
+  let icon = {
+    file: do_get_file("noise.png"),
+    mimetype: "image/png",
+  };
+
+  let url = "http://foo.bar/";
+  await PlacesTestUtils.addVisits(url);
+  for (let i = 0; i < 10; ++i) {
+    let iconUri = NetUtil.newURI("http://mozilla.org/" + i);
+    let data = readFileData(icon.file);
+    PlacesUtils.favicons.replaceFaviconData(iconUri, data, icon.mimetype);
+    await setFaviconForPage(url, iconUri);
+  }
+
+  let promise = TestUtils.topicObserved("places-favicons-expired");
+  PlacesUtils.favicons.expireAllFavicons();
+  await promise;
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let state = (
+    await db.execute("PRAGMA favicons.auto_vacuum")
+  )[0].getResultByIndex(0);
+  Assert.equal(state, 2, "auto_vacuum should be incremental");
+  let count = (
+    await db.execute("PRAGMA favicons.freelist_count")
+  )[0].getResultByIndex(0);
+  info(`Found ${count} freelist pages`);
+  let log = await PlacesDBUtils.incrementalVacuum();
+  info(log);
+  let newCount = (
+    await db.execute("PRAGMA favicons.freelist_count")
+  )[0].getResultByIndex(0);
+  info(`Found ${newCount} freelist pages`);
+  Assert.ok(
+    newCount < count,
+    "The number of freelist pages should have reduced"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_moz-anno_favicon_mime_type.js b/toolkit/components/places/tests/favicons/test_moz-anno_favicon_mime_type.js
new file mode 100644
index 0000000000..59e87a8225
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_moz-anno_favicon_mime_type.js
@@ -0,0 +1,88 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This test ensures that the mime type is set for moz-anno channels of favicons
+ * properly.  Added with work in bug 481227.
+ */
+
+const testFaviconData =
+  "data:image/png,%89PNG%0D%0A%1A%0A%00%00%00%0DIHDR%00%00%00%10%00%00%00%10%08%06%00%00%00%1F%F3%FFa%00%00%00%04gAMA%00%00%AF%C87%05%8A%E9%00%00%00%19tEXtSoftware%00Adobe%20ImageReadyq%C9e%3C%00%00%01%D6IDATx%DAb%FC%FF%FF%3F%03%25%00%20%80%98%909%EF%DF%BFg%EF%EC%EC%FC%AD%AC%AC%FC%DF%95%91%F1%BF%89%89%C9%7F%20%FF%D7%EA%D5%AB%B7%DF%BBwO%16%9B%01%00%01%C4%00r%01%08%9F9s%C6%CD%D8%D8%F8%BF%0B%03%C3%FF3%40%BC%0A%88%EF%02q%1A%10%BB%40%F1%AAU%ABv%C1%D4%C30%40%00%81%89%993g%3E%06%1A%F6%3F%14%AA%11D%97%03%F1%7Fc%08%0D%E2%2B))%FD%17%04%89%A1%19%00%10%40%0C%D00%F8%0F3%00%C8%F8%BF%1B%E4%0Ac%88a%E5%60%17%19%FF%0F%0D%0D%05%1B%02v%D9%DD%BB%0A0%03%00%02%08%AC%B9%A3%A3%E3%17%03%D4v%90%01%EF%18%106%C3%0Cz%07%C5%BB%A1%DE%82y%07%20%80%A0%A6%08B%FCn%0C1%60%26%D4%20d%C3VA%C3%06%26%BE%0A%EA-%80%00%82%B9%E0%F7L4%0D%EF%90%F8%C6%60%2F%0A%82%BD%01%13%07%0700%D0%01%02%88%11%E4%02P%B41%DC%BB%C7%D0%014%0D%E8l%06W%20%06%BA%88%A1%1C%1AS%15%40%7C%16%CA6.%2Fgx%BFg%0F%83%CB%D9%B3%0C%7B%80%7C%80%00%02%BB%00%E8%9F%ED%20%1B%3A%A0%A6%9F%81%DA%DC%01%C5%B0%80%ED%80%FA%BF%BC%BC%FC%3F%83%12%90%9D%96%F6%1F%20%80%18%DE%BD%7B%C7%0E%8E%05AD%20%FEGr%A6%A0%A0%E0%7F%25P%80%02%9D%0F%D28%13%18%23%C6%C0%B0%02E%3D%C8%F5%00%01%04%8F%05P%A8%BA%40my%87%E4%12c%A8%8D%20%8B%D0%D3%00%08%03%04%10%9C%01R%E4%82d%3B%C8%A0%99%C6%90%90%C6%A5%19%84%01%02%08%9E%17%80%C9x%F7%7B%A0%DBVC%F9%A0%C0%5C%7D%16%2C%CE%00%F4%C6O%5C%99%09%20%800L%04y%A5%03%1A%95%A0%80%05%05%14.%DBA%18%20%80%18)%CD%CE%00%01%06%00%0C'%94%C7%C0k%C9%2C%00%00%00%00IEND%AEB%60%82";
+const testIconURI = uri("http://mozilla.org/favicon.png");
+
+function streamListener(aExpectedContentType) {
+  this._expectedContentType = aExpectedContentType;
+  this.done = PromiseUtils.defer();
+}
+streamListener.prototype = {
+  onStartRequest() {},
+  onStopRequest(aRequest, aContext, aStatusCode) {
+    let channel = aRequest.QueryInterface(Ci.nsIChannel);
+    Assert.equal(
+      channel.contentType,
+      this._expectedContentType,
+      "The channel content type is the expected one"
+    );
+    this.done.resolve();
+  },
+  onDataAvailable(aRequest, aInputStream, aOffset, aCount) {
+    aRequest.cancel(Cr.NS_ERROR_ABORT);
+    throw Components.Exception("", Cr.NS_ERROR_ABORT);
+  },
+};
+
+add_task(async function () {
+  info("Test that the default icon has the right content type.");
+  let channel = NetUtil.newChannel({
+    uri: PlacesUtils.favicons.defaultFavicon,
+    loadUsingSystemPrincipal: true,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
+  });
+  let listener = new streamListener(
+    PlacesUtils.favicons.defaultFaviconMimeType
+  );
+  channel.asyncOpen(listener);
+  await listener.done.promise;
+});
+
+add_task(async function () {
+  info(
+    "Test icon URI that we don't know anything about.  Will serve the default icon."
+  );
+  let channel = NetUtil.newChannel({
+    uri: PlacesUtils.favicons.getFaviconLinkForIcon(testIconURI).spec,
+    loadUsingSystemPrincipal: true,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
+  });
+  let listener = new streamListener(
+    PlacesUtils.favicons.defaultFaviconMimeType
+  );
+  channel.asyncOpen(listener);
+  await listener.done.promise;
+});
+
+add_task(async function () {
+  info("Test that the content type of a favicon we add is correct.");
+  let testURI = uri("http://mozilla.org/");
+  // Add the data before opening
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    testIconURI,
+    testFaviconData,
+    0,
+    systemPrincipal
+  );
+  await PlacesTestUtils.addVisits(testURI);
+  await setFaviconForPage(testURI, testIconURI);
+  // Open the channel
+  let channel = NetUtil.newChannel({
+    uri: PlacesUtils.favicons.getFaviconLinkForIcon(testIconURI).spec,
+    loadUsingSystemPrincipal: true,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
+  });
+  let listener = new streamListener("image/png");
+  channel.asyncOpen(listener);
+  await listener.done.promise;
+});
diff --git a/toolkit/components/places/tests/favicons/test_multiple_frames.js b/toolkit/components/places/tests/favicons/test_multiple_frames.js
new file mode 100644
index 0000000000..eb59e7f9c6
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_multiple_frames.js
@@ -0,0 +1,46 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests support for icons with multiple frames (like .ico files).
+ */
+
+add_task(async function () {
+  //  in: 48x48 ico, 56646 bytes.
+  // (howstuffworks.com icon, contains 13 icons with sizes from 16x16 to
+  // 48x48 in varying depths)
+  let pageURI = NetUtil.newURI("http://places.test/page/");
+  await PlacesTestUtils.addVisits(pageURI);
+  let faviconURI = NetUtil.newURI("http://places.test/icon/favicon-multi.ico");
+  // Fake window.
+  let win = { devicePixelRatio: 1.0 };
+  let icoData = readFileData(do_get_file("favicon-multi.ico"));
+  PlacesUtils.favicons.replaceFaviconData(faviconURI, icoData, "image/x-icon");
+  await setFaviconForPage(pageURI, faviconURI);
+
+  for (let size of [16, 32, 64]) {
+    let file = do_get_file(`favicon-multi-frame${size}.png`);
+    let data = readFileData(file);
+
+    info("Check getFaviconDataForPage");
+    let icon = await getFaviconDataForPage(pageURI, size);
+    Assert.equal(icon.mimeType, "image/png");
+    Assert.deepEqual(icon.data, data);
+
+    info("Check moz-anno:favicon protocol");
+    await compareFavicons(
+      Services.io.newFileURI(file),
+      PlacesUtils.urlWithSizeRef(
+        win,
+        PlacesUtils.favicons.getFaviconLinkForIcon(faviconURI).spec,
+        size
+      )
+    );
+
+    info("Check page-icon protocol");
+    await compareFavicons(
+      Services.io.newFileURI(file),
+      PlacesUtils.urlWithSizeRef(win, "page-icon:" + pageURI.spec, size)
+    );
+  }
+});
diff --git a/toolkit/components/places/tests/favicons/test_page-icon_protocol.js b/toolkit/components/places/tests/favicons/test_page-icon_protocol.js
new file mode 100644
index 0000000000..932040bafb
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_page-icon_protocol.js
@@ -0,0 +1,243 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const ICON_DATAURL =
+  "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg==";
+const TEST_URI = NetUtil.newURI("http://mozilla.org/");
+const ICON_URI = NetUtil.newURI("http://mozilla.org/favicon.ico");
+
+const { XPCShellContentUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/XPCShellContentUtils.sys.mjs"
+);
+
+const PAGE_ICON_TEST_URLS = [
+  "page-icon:http://example.com/",
+  "page-icon:http://a-site-never-before-seen.test",
+  // For the following, the page-icon protocol is expected to successfully
+  // return the default favicon.
+  "page-icon:test",
+  "page-icon:",
+  "page-icon:chrome://something.html",
+  "page-icon:foo://bar/baz",
+];
+
+XPCShellContentUtils.init(this);
+
+const HTML = String.raw`
+
+
+  
+
+
+  Hello from example.com!
+
+`;
+
+const server = XPCShellContentUtils.createHttpServer({
+  hosts: ["example.com"],
+});
+
+server.registerPathHandler("/", (request, response) => {
+  response.setHeader("Content-Type", "text/html");
+  response.write(HTML);
+});
+
+function fetchIconForSpec(spec) {
+  return new Promise((resolve, reject) => {
+    NetUtil.asyncFetch(
+      {
+        uri: NetUtil.newURI(spec),
+        loadUsingSystemPrincipal: true,
+        contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
+      },
+      (input, status, request) => {
+        if (!Components.isSuccessCode(status)) {
+          reject(new Error("unable to load icon"));
+          return;
+        }
+
+        try {
+          let data = NetUtil.readInputStreamToString(input, input.available());
+          let contentType = request.QueryInterface(Ci.nsIChannel).contentType;
+          input.close();
+          resolve({ data, contentType });
+        } catch (ex) {
+          reject(ex);
+        }
+      }
+    );
+  });
+}
+
+var gDefaultFavicon;
+var gFavicon;
+
+add_task(async function setup() {
+  await PlacesTestUtils.addVisits(TEST_URI);
+
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    ICON_URI,
+    ICON_DATAURL,
+    (Date.now() + 8640000) * 1000,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      TEST_URI,
+      ICON_URI,
+      false,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      resolve,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  gDefaultFavicon = await fetchIconForSpec(
+    PlacesUtils.favicons.defaultFavicon.spec
+  );
+  gFavicon = await fetchIconForSpec(ICON_DATAURL);
+});
+
+add_task(async function known_url() {
+  let { data, contentType } = await fetchIconForSpec(
+    "page-icon:" + TEST_URI.spec
+  );
+  Assert.equal(contentType, gFavicon.contentType);
+  Assert.deepEqual(data, gFavicon.data, "Got the favicon data");
+});
+
+add_task(async function unknown_url() {
+  let { data, contentType } = await fetchIconForSpec(
+    "page-icon:http://www.moz.org/"
+  );
+  Assert.equal(contentType, gDefaultFavicon.contentType);
+  Assert.deepEqual(data, gDefaultFavicon.data, "Got the default favicon data");
+});
+
+add_task(async function invalid_url() {
+  let { data, contentType } = await fetchIconForSpec("page-icon:test");
+  Assert.equal(contentType, gDefaultFavicon.contentType);
+  Assert.ok(data == gDefaultFavicon.data, "Got the default favicon data");
+});
+
+add_task(async function subpage_url_fallback() {
+  let { data, contentType } = await fetchIconForSpec(
+    "page-icon:http://mozilla.org/missing"
+  );
+  Assert.equal(contentType, gFavicon.contentType);
+  Assert.deepEqual(data, gFavicon.data, "Got the root favicon data");
+});
+
+add_task(async function svg_icon() {
+  let faviconURI = NetUtil.newURI("http://places.test/favicon.svg");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    faviconURI,
+    SMALLSVG_DATA_URI.spec,
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+  await setFaviconForPage(TEST_URI, faviconURI);
+  let svgIcon = await fetchIconForSpec(SMALLSVG_DATA_URI.spec);
+  info(svgIcon.contentType);
+  let pageIcon = await fetchIconForSpec("page-icon:" + TEST_URI.spec);
+  Assert.equal(svgIcon.contentType, pageIcon.contentType);
+  Assert.deepEqual(svgIcon.data, pageIcon.data, "Got the root favicon data");
+});
+
+add_task(async function page_with_ref() {
+  for (let url of [
+    "http://places.test.ref/#myref",
+    "http://places.test.ref/#!&b=16",
+    "http://places.test.ref/#",
+  ]) {
+    await PlacesTestUtils.addVisits(url);
+    await setFaviconForPage(url, ICON_URI, false);
+    let { data, contentType } = await fetchIconForSpec("page-icon:" + url);
+    Assert.equal(contentType, gFavicon.contentType);
+    Assert.deepEqual(data, gFavicon.data, "Got the favicon data");
+    await PlacesUtils.history.remove(url);
+  }
+});
+
+/**
+ * Tests that page-icon does not work in a normal content process.
+ */
+add_task(async function page_content_process() {
+  let contentPage = await XPCShellContentUtils.loadContentPage(
+    "http://example.com/",
+    {
+      remote: true,
+    }
+  );
+  Assert.notEqual(
+    contentPage.browsingContext.currentRemoteType,
+    "privilegedabout"
+  );
+
+  await contentPage.spawn([PAGE_ICON_TEST_URLS], async URLS => {
+    // We expect each of these URLs to produce an error event when
+    // we attempt to load them in this process type.
+    /* global content */
+    for (let url of URLS) {
+      let img = content.document.createElement("img");
+      img.src = url;
+      let imgPromise = new Promise((resolve, reject) => {
+        img.addEventListener("error", e => {
+          Assert.ok(true, "Got expected load error.");
+          resolve();
+        });
+        img.addEventListener("load", e => {
+          Assert.ok(false, "Did not expect a successful load.");
+          reject();
+        });
+      });
+      content.document.body.appendChild(img);
+      await imgPromise;
+    }
+  });
+
+  await contentPage.close();
+});
+
+/**
+ * Tests that page-icon does work for privileged about content process
+ */
+add_task(async function page_privileged_about_content_process() {
+  // about:certificate loads in the privileged about content process.
+  let contentPage = await XPCShellContentUtils.loadContentPage(
+    "about:certificate",
+    {
+      remote: true,
+    }
+  );
+  Assert.equal(
+    contentPage.browsingContext.currentRemoteType,
+    "privilegedabout"
+  );
+
+  await contentPage.spawn([PAGE_ICON_TEST_URLS], async URLS => {
+    // We expect each of these URLs to load correctly in this process
+    // type.
+    for (let url of URLS) {
+      let img = content.document.createElement("img");
+      img.src = url;
+      let imgPromise = new Promise((resolve, reject) => {
+        img.addEventListener("error", e => {
+          Assert.ok(false, "Did not expect an error. ");
+          reject();
+        });
+        img.addEventListener("load", e => {
+          Assert.ok(true, "Got expected load event.");
+          resolve();
+        });
+      });
+      content.document.body.appendChild(img);
+      await imgPromise;
+    }
+  });
+
+  await contentPage.close();
+});
diff --git a/toolkit/components/places/tests/favicons/test_query_result_favicon_changed_on_child.js b/toolkit/components/places/tests/favicons/test_query_result_favicon_changed_on_child.js
new file mode 100644
index 0000000000..4e5c55e50a
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_query_result_favicon_changed_on_child.js
@@ -0,0 +1,153 @@
+/**
+ * Test for bug 451499 :
+ * Wrong folder icon appears on queries.
+ */
+
+"use strict";
+
+add_task(async function test_query_result_favicon_changed_on_child() {
+  // Bookmark our test page, so it will appear in the query resultset.
+  const PAGE_URI = Services.io.newURI("http://example.com/test_query_result");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "test_bookmark",
+    url: PAGE_URI,
+  });
+
+  // Get the last 10 bookmarks added to the menu or the toolbar.
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+  ]);
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+  options.maxResults = 10;
+  options.excludeQueries = 1;
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let resultObserver = {
+    containerStateChanged(aContainerNode, aOldState, aNewState) {
+      if (aNewState == Ci.nsINavHistoryContainerResultNode.STATE_OPENED) {
+        // We set a favicon on PAGE_URI while the container is open.  The
+        // favicon for the page must have data associated with it in order for
+        // the icon changed notifications to be sent, so we use a valid image
+        // data URI.
+        PlacesUtils.favicons.setAndFetchFaviconForPage(
+          PAGE_URI,
+          SMALLPNG_DATA_URI,
+          false,
+          PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+          null,
+          Services.scriptSecurityManager.getSystemPrincipal()
+        );
+      }
+    },
+    nodeIconChanged(aNode) {
+      if (PlacesUtils.nodeIsContainer(aNode)) {
+        do_throw(
+          "The icon should be set only for the page," +
+            " not for the containing query."
+        );
+      }
+    },
+  };
+  Object.setPrototypeOf(resultObserver, NavHistoryResultObserver.prototype);
+  result.addObserver(resultObserver);
+
+  // Open the container and wait for containerStateChanged. We should start
+  // observing before setting |containerOpen| as that's caused by the
+  // setAndFetchFaviconForPage() call caused by the containerStateChanged
+  // observer above.
+  let promise = promiseFaviconChanged(PAGE_URI, SMALLPNG_DATA_URI);
+  result.root.containerOpen = true;
+  await promise;
+
+  // We must wait for the asynchronous database thread to finish the
+  // operation, and then for the main thread to process any pending
+  // notifications that came from the asynchronous thread, before we can be
+  // sure that nodeIconChanged was not invoked in the meantime.
+  await PlacesTestUtils.promiseAsyncUpdates();
+  result.removeObserver(resultObserver);
+
+  // Free the resources immediately.
+  result.root.containerOpen = false;
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(
+  async function test_query_result_favicon_changed_not_affect_lastmodified() {
+    // Bookmark our test page, so it will appear in the query resultset.
+    const PAGE_URI2 = Services.io.newURI(
+      "http://example.com/test_query_result"
+    );
+    let bm = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      title: "test_bookmark",
+      url: PAGE_URI2,
+    });
+
+    let result = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.menuGuid);
+
+    Assert.equal(
+      result.root.childCount,
+      1,
+      "Should have only one item in the query"
+    );
+    Assert.equal(
+      result.root.getChild(0).uri,
+      PAGE_URI2.spec,
+      "Should have the correct child"
+    );
+    Assert.equal(
+      result.root.getChild(0).lastModified,
+      PlacesUtils.toPRTime(bm.lastModified),
+      "Should have the expected last modified date."
+    );
+
+    let promise = promiseFaviconChanged(PAGE_URI2, SMALLPNG_DATA_URI);
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      PAGE_URI2,
+      SMALLPNG_DATA_URI,
+      false,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      null,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+    await promise;
+
+    // Open the container and wait for containerStateChanged. We should start
+    // observing before setting |containerOpen| as that's caused by the
+    // setAndFetchFaviconForPage() call caused by the containerStateChanged
+    // observer above.
+
+    // We must wait for the asynchronous database thread to finish the
+    // operation, and then for the main thread to process any pending
+    // notifications that came from the asynchronous thread, before we can be
+    // sure that nodeIconChanged was not invoked in the meantime.
+    await PlacesTestUtils.promiseAsyncUpdates();
+
+    Assert.equal(
+      result.root.childCount,
+      1,
+      "Should have only one item in the query"
+    );
+    Assert.equal(
+      result.root.getChild(0).uri,
+      PAGE_URI2.spec,
+      "Should have the correct child"
+    );
+    Assert.equal(
+      result.root.getChild(0).lastModified,
+      PlacesUtils.toPRTime(bm.lastModified),
+      "Should not have changed the last modified date."
+    );
+
+    // Free the resources immediately.
+    result.root.containerOpen = false;
+  }
+);
diff --git a/toolkit/components/places/tests/favicons/test_replaceFaviconData.js b/toolkit/components/places/tests/favicons/test_replaceFaviconData.js
new file mode 100644
index 0000000000..2e9835eaa9
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_replaceFaviconData.js
@@ -0,0 +1,395 @@
+/* Any copyright is dedicated to the Public Domain.
+ *    http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * Tests for replaceFaviconData()
+ */
+
+var iconsvc = PlacesUtils.favicons;
+
+var originalFavicon = {
+  file: do_get_file("favicon-normal16.png"),
+  uri: uri(do_get_file("favicon-normal16.png")),
+  data: readFileData(do_get_file("favicon-normal16.png")),
+  mimetype: "image/png",
+};
+
+var uniqueFaviconId = 0;
+function createFavicon(fileName) {
+  let tempdir = Services.dirsvc.get("TmpD", Ci.nsIFile);
+
+  // remove any existing file at the path we're about to copy to
+  let outfile = tempdir.clone();
+  outfile.append(fileName);
+  try {
+    outfile.remove(false);
+  } catch (e) {}
+
+  originalFavicon.file.copyToFollowingLinks(tempdir, fileName);
+
+  let stream = Cc["@mozilla.org/network/file-output-stream;1"].createInstance(
+    Ci.nsIFileOutputStream
+  );
+  stream.init(outfile, 0x02 | 0x08 | 0x10, 0o600, 0);
+
+  // append some data that sniffers/encoders will ignore that will distinguish
+  // the different favicons we'll create
+  uniqueFaviconId++;
+  let uniqueStr = "uid:" + uniqueFaviconId;
+  stream.write(uniqueStr, uniqueStr.length);
+  stream.close();
+
+  Assert.equal(outfile.leafName.substr(0, fileName.length), fileName);
+
+  return {
+    file: outfile,
+    uri: uri(outfile),
+    data: readFileData(outfile),
+    mimetype: "image/png",
+  };
+}
+
+function checkCallbackSucceeded(
+  callbackMimetype,
+  callbackData,
+  sourceMimetype,
+  sourceData
+) {
+  Assert.equal(callbackMimetype, sourceMimetype);
+  Assert.ok(compareArrays(callbackData, sourceData));
+}
+
+function run_test() {
+  // check that the favicon loaded correctly
+  Assert.equal(originalFavicon.data.length, 286);
+  run_next_test();
+}
+
+add_task(async function test_replaceFaviconData_validHistoryURI() {
+  info("test replaceFaviconData for valid history uri");
+
+  let pageURI = uri("http://test1.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let favicon = createFavicon("favicon1.png");
+
+  iconsvc.replaceFaviconData(favicon.uri, favicon.data, favicon.mimetype);
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      favicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconData_validHistoryURI_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        dump("GOT " + aMimeType + "\n");
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          favicon.mimetype,
+          favicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          favicon.mimetype,
+          favicon.data,
+          function test_replaceFaviconData_validHistoryURI_callback() {
+            favicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      systemPrincipal
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_overrideDefaultFavicon() {
+  info("test replaceFaviconData to override a later setAndFetchFaviconForPage");
+
+  let pageURI = uri("http://test2.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let firstFavicon = createFavicon("favicon2.png");
+  let secondFavicon = createFavicon("favicon3.png");
+
+  iconsvc.replaceFaviconData(
+    firstFavicon.uri,
+    secondFavicon.data,
+    secondFavicon.mimetype
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      firstFavicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconData_overrideDefaultFavicon_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          secondFavicon.mimetype,
+          secondFavicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          secondFavicon.mimetype,
+          secondFavicon.data,
+          function test_replaceFaviconData_overrideDefaultFavicon_callback() {
+            firstFavicon.file.remove(false);
+            secondFavicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      systemPrincipal
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_replaceExisting() {
+  info(
+    "test replaceFaviconData to override a previous setAndFetchFaviconForPage"
+  );
+
+  let pageURI = uri("http://test3.bar");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let firstFavicon = createFavicon("favicon4.png");
+  let secondFavicon = createFavicon("favicon5.png");
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      firstFavicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconData_replaceExisting_firstSet_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          firstFavicon.mimetype,
+          firstFavicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          firstFavicon.mimetype,
+          firstFavicon.data,
+          function test_replaceFaviconData_overrideDefaultFavicon_firstCallback() {
+            iconsvc.replaceFaviconData(
+              firstFavicon.uri,
+              secondFavicon.data,
+              secondFavicon.mimetype
+            );
+            PlacesTestUtils.promiseAsyncUpdates().then(() => {
+              checkFaviconDataForPage(
+                pageURI,
+                secondFavicon.mimetype,
+                secondFavicon.data,
+                function test_replaceFaviconData_overrideDefaultFavicon_secondCallback() {
+                  firstFavicon.file.remove(false);
+                  secondFavicon.file.remove(false);
+                  resolve();
+                },
+                systemPrincipal
+              );
+            });
+          }
+        );
+      },
+      systemPrincipal
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_unrelatedReplace() {
+  info("test replaceFaviconData to not make unrelated changes");
+
+  let pageURI = uri("http://test4.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let favicon = createFavicon("favicon6.png");
+  let unrelatedFavicon = createFavicon("favicon7.png");
+
+  iconsvc.replaceFaviconData(
+    unrelatedFavicon.uri,
+    unrelatedFavicon.data,
+    unrelatedFavicon.mimetype
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      favicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconData_unrelatedReplace_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          favicon.mimetype,
+          favicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          favicon.mimetype,
+          favicon.data,
+          function test_replaceFaviconData_unrelatedReplace_callback() {
+            favicon.file.remove(false);
+            unrelatedFavicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      systemPrincipal
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_badInputs() {
+  info("test replaceFaviconData to throw on bad inputs");
+  let icon = createFavicon("favicon8.png");
+
+  Assert.throws(
+    () => iconsvc.replaceFaviconData(icon.uri, icon.data, ""),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => iconsvc.replaceFaviconData(icon.uri, icon.data, "not-an-image"),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => iconsvc.replaceFaviconData(null, icon.data, icon.mimetype),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => iconsvc.replaceFaviconData(icon.uri, [], icon.mimetype),
+    /NS_ERROR_ILLEGAL_VALUE/
+  );
+  Assert.throws(
+    () => iconsvc.replaceFaviconData(icon.uri, null, icon.mimetype),
+    /NS_ERROR_XPC_CANT_CONVERT_PRIMITIVE_TO_ARRAY/
+  );
+
+  icon.file.remove(false);
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_twiceReplace() {
+  info("test replaceFaviconData on multiple replacements");
+
+  let pageURI = uri("http://test5.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let firstFavicon = createFavicon("favicon9.png");
+  let secondFavicon = createFavicon("favicon10.png");
+
+  iconsvc.replaceFaviconData(
+    firstFavicon.uri,
+    firstFavicon.data,
+    firstFavicon.mimetype
+  );
+  iconsvc.replaceFaviconData(
+    firstFavicon.uri,
+    secondFavicon.data,
+    secondFavicon.mimetype
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      firstFavicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconData_twiceReplace_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          secondFavicon.mimetype,
+          secondFavicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          secondFavicon.mimetype,
+          secondFavicon.data,
+          function test_replaceFaviconData_twiceReplace_callback() {
+            firstFavicon.file.remove(false);
+            secondFavicon.file.remove(false);
+            resolve();
+          },
+          systemPrincipal
+        );
+      },
+      systemPrincipal
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconData_rootOverwrite() {
+  info("test replaceFaviconData doesn't overwrite root = 1");
+
+  async function getRootValue(url) {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(
+      "SELECT root FROM moz_icons WHERE icon_url = :url",
+      { url }
+    );
+    return rows[0].getResultByName("root");
+  }
+
+  const PAGE_URL = "http://rootoverwrite.bar/";
+  let pageURI = Services.io.newURI(PAGE_URL);
+  const ICON_URL = "http://rootoverwrite.bar/favicon.ico";
+  let iconURI = Services.io.newURI(ICON_URL);
+
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let icon = createFavicon("favicon9.png");
+  PlacesUtils.favicons.replaceFaviconData(iconURI, icon.data, icon.mimetype);
+  await PlacesTestUtils.addFavicons(new Map([[PAGE_URL, ICON_URL]]));
+
+  Assert.equal(await getRootValue(ICON_URL), 1, "Check root == 1");
+  let icon2 = createFavicon("favicon10.png");
+  PlacesUtils.favicons.replaceFaviconData(iconURI, icon2.data, icon2.mimetype);
+  // replaceFaviconData doesn't have a callback, but we must wait its updated.
+  await PlacesTestUtils.promiseAsyncUpdates();
+  Assert.equal(await getRootValue(ICON_URL), 1, "Check root did not change");
+
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/favicons/test_replaceFaviconDataFromDataURL.js b/toolkit/components/places/tests/favicons/test_replaceFaviconDataFromDataURL.js
new file mode 100644
index 0000000000..c1b83fc8a7
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_replaceFaviconDataFromDataURL.js
@@ -0,0 +1,537 @@
+/* Any copyright is dedicated to the Public Domain.
+ *    http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * Tests for replaceFaviconData()
+ */
+
+var iconsvc = PlacesUtils.favicons;
+
+var originalFavicon = {
+  file: do_get_file("favicon-normal16.png"),
+  uri: uri(do_get_file("favicon-normal16.png")),
+  data: readFileData(do_get_file("favicon-normal16.png")),
+  mimetype: "image/png",
+};
+
+var uniqueFaviconId = 0;
+function createFavicon(fileName) {
+  let tempdir = Services.dirsvc.get("TmpD", Ci.nsIFile);
+
+  // remove any existing file at the path we're about to copy to
+  let outfile = tempdir.clone();
+  outfile.append(fileName);
+  try {
+    outfile.remove(false);
+  } catch (e) {}
+
+  originalFavicon.file.copyToFollowingLinks(tempdir, fileName);
+
+  let stream = Cc["@mozilla.org/network/file-output-stream;1"].createInstance(
+    Ci.nsIFileOutputStream
+  );
+  stream.init(outfile, 0x02 | 0x08 | 0x10, 0o600, 0);
+
+  // append some data that sniffers/encoders will ignore that will distinguish
+  // the different favicons we'll create
+  uniqueFaviconId++;
+  let uniqueStr = "uid:" + uniqueFaviconId;
+  stream.write(uniqueStr, uniqueStr.length);
+  stream.close();
+
+  Assert.equal(outfile.leafName.substr(0, fileName.length), fileName);
+
+  return {
+    file: outfile,
+    uri: uri(outfile),
+    data: readFileData(outfile),
+    mimetype: "image/png",
+  };
+}
+
+function createDataURLForFavicon(favicon) {
+  return "data:" + favicon.mimetype + ";base64," + toBase64(favicon.data);
+}
+
+function checkCallbackSucceeded(
+  callbackMimetype,
+  callbackData,
+  sourceMimetype,
+  sourceData
+) {
+  Assert.equal(callbackMimetype, sourceMimetype);
+  Assert.ok(compareArrays(callbackData, sourceData));
+}
+
+function run_test() {
+  // check that the favicon loaded correctly
+  Assert.equal(originalFavicon.data.length, 286);
+  run_next_test();
+}
+
+add_task(async function test_replaceFaviconDataFromDataURL_validHistoryURI() {
+  info("test replaceFaviconDataFromDataURL for valid history uri");
+
+  let pageURI = uri("http://test1.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let favicon = createFavicon("favicon1.png");
+  iconsvc.replaceFaviconDataFromDataURL(
+    favicon.uri,
+    createDataURLForFavicon(favicon),
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      favicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconDataFromDataURL_validHistoryURI_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          favicon.mimetype,
+          favicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          favicon.mimetype,
+          favicon.data,
+          function test_replaceFaviconDataFromDataURL_validHistoryURI_callback() {
+            favicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(
+  async function test_replaceFaviconDataFromDataURL_overrideDefaultFavicon() {
+    info(
+      "test replaceFaviconDataFromDataURL to override a later setAndFetchFaviconForPage"
+    );
+
+    let pageURI = uri("http://test2.bar/");
+    await PlacesTestUtils.addVisits(pageURI);
+
+    let firstFavicon = createFavicon("favicon2.png");
+    let secondFavicon = createFavicon("favicon3.png");
+
+    iconsvc.replaceFaviconDataFromDataURL(
+      firstFavicon.uri,
+      createDataURLForFavicon(secondFavicon),
+      0,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+
+    await new Promise(resolve => {
+      iconsvc.setAndFetchFaviconForPage(
+        pageURI,
+        firstFavicon.uri,
+        true,
+        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+        function test_replaceFaviconDataFromDataURL_overrideDefaultFavicon_check(
+          aURI,
+          aDataLen,
+          aData,
+          aMimeType
+        ) {
+          checkCallbackSucceeded(
+            aMimeType,
+            aData,
+            secondFavicon.mimetype,
+            secondFavicon.data
+          );
+          checkFaviconDataForPage(
+            pageURI,
+            secondFavicon.mimetype,
+            secondFavicon.data,
+            function test_replaceFaviconDataFromDataURL_overrideDefaultFavicon_callback() {
+              firstFavicon.file.remove(false);
+              secondFavicon.file.remove(false);
+              resolve();
+            }
+          );
+        },
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+    });
+
+    await PlacesUtils.history.clear();
+  }
+);
+
+add_task(async function test_replaceFaviconDataFromDataURL_replaceExisting() {
+  info(
+    "test replaceFaviconDataFromDataURL to override a previous setAndFetchFaviconForPage"
+  );
+
+  let pageURI = uri("http://test3.bar");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let firstFavicon = createFavicon("favicon4.png");
+  let secondFavicon = createFavicon("favicon5.png");
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      firstFavicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconDataFromDataURL_replaceExisting_firstSet_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          firstFavicon.mimetype,
+          firstFavicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          firstFavicon.mimetype,
+          firstFavicon.data,
+          function test_replaceFaviconDataFromDataURL_replaceExisting_firstCallback() {
+            iconsvc.replaceFaviconDataFromDataURL(
+              firstFavicon.uri,
+              createDataURLForFavicon(secondFavicon),
+              0,
+              Services.scriptSecurityManager.getSystemPrincipal()
+            );
+            checkFaviconDataForPage(
+              pageURI,
+              secondFavicon.mimetype,
+              secondFavicon.data,
+              function test_replaceFaviconDataFromDataURL_replaceExisting_secondCallback() {
+                firstFavicon.file.remove(false);
+                secondFavicon.file.remove(false);
+                resolve();
+              }
+            );
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconDataFromDataURL_unrelatedReplace() {
+  info("test replaceFaviconDataFromDataURL to not make unrelated changes");
+
+  let pageURI = uri("http://test4.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let favicon = createFavicon("favicon6.png");
+  let unrelatedFavicon = createFavicon("favicon7.png");
+
+  iconsvc.replaceFaviconDataFromDataURL(
+    unrelatedFavicon.uri,
+    createDataURLForFavicon(unrelatedFavicon),
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      favicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconDataFromDataURL_unrelatedReplace_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          favicon.mimetype,
+          favicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          favicon.mimetype,
+          favicon.data,
+          function test_replaceFaviconDataFromDataURL_unrelatedReplace_callback() {
+            favicon.file.remove(false);
+            unrelatedFavicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconDataFromDataURL_badInputs() {
+  info("test replaceFaviconDataFromDataURL to throw on bad inputs");
+
+  let favicon = createFavicon("favicon8.png");
+
+  let ex = null;
+  try {
+    iconsvc.replaceFaviconDataFromDataURL(
+      favicon.uri,
+      "",
+      0,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  } catch (e) {
+    ex = e;
+  } finally {
+    Assert.ok(!!ex);
+  }
+
+  ex = null;
+  try {
+    iconsvc.replaceFaviconDataFromDataURL(
+      null,
+      createDataURLForFavicon(favicon),
+      0,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  } catch (e) {
+    ex = e;
+  } finally {
+    Assert.ok(!!ex);
+  }
+
+  favicon.file.remove(false);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_replaceFaviconDataFromDataURL_twiceReplace() {
+  info("test replaceFaviconDataFromDataURL on multiple replacements");
+
+  let pageURI = uri("http://test5.bar/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let firstFavicon = createFavicon("favicon9.png");
+  let secondFavicon = createFavicon("favicon10.png");
+
+  iconsvc.replaceFaviconDataFromDataURL(
+    firstFavicon.uri,
+    createDataURLForFavicon(firstFavicon),
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+  iconsvc.replaceFaviconDataFromDataURL(
+    firstFavicon.uri,
+    createDataURLForFavicon(secondFavicon),
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await new Promise(resolve => {
+    iconsvc.setAndFetchFaviconForPage(
+      pageURI,
+      firstFavicon.uri,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      function test_replaceFaviconDataFromDataURL_twiceReplace_check(
+        aURI,
+        aDataLen,
+        aData,
+        aMimeType
+      ) {
+        checkCallbackSucceeded(
+          aMimeType,
+          aData,
+          secondFavicon.mimetype,
+          secondFavicon.data
+        );
+        checkFaviconDataForPage(
+          pageURI,
+          secondFavicon.mimetype,
+          secondFavicon.data,
+          function test_replaceFaviconDataFromDataURL_twiceReplace_callback() {
+            firstFavicon.file.remove(false);
+            secondFavicon.file.remove(false);
+            resolve();
+          }
+        );
+      },
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(
+  async function test_replaceFaviconDataFromDataURL_afterRegularAssign() {
+    info("test replaceFaviconDataFromDataURL after replaceFaviconData");
+
+    let pageURI = uri("http://test6.bar/");
+    await PlacesTestUtils.addVisits(pageURI);
+
+    let firstFavicon = createFavicon("favicon11.png");
+    let secondFavicon = createFavicon("favicon12.png");
+
+    iconsvc.replaceFaviconData(
+      firstFavicon.uri,
+      firstFavicon.data,
+      firstFavicon.mimetype
+    );
+    iconsvc.replaceFaviconDataFromDataURL(
+      firstFavicon.uri,
+      createDataURLForFavicon(secondFavicon),
+      0,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+
+    await new Promise(resolve => {
+      iconsvc.setAndFetchFaviconForPage(
+        pageURI,
+        firstFavicon.uri,
+        true,
+        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+        function test_replaceFaviconDataFromDataURL_afterRegularAssign_check(
+          aURI,
+          aDataLen,
+          aData,
+          aMimeType
+        ) {
+          checkCallbackSucceeded(
+            aMimeType,
+            aData,
+            secondFavicon.mimetype,
+            secondFavicon.data
+          );
+          checkFaviconDataForPage(
+            pageURI,
+            secondFavicon.mimetype,
+            secondFavicon.data,
+            function test_replaceFaviconDataFromDataURL_afterRegularAssign_callback() {
+              firstFavicon.file.remove(false);
+              secondFavicon.file.remove(false);
+              resolve();
+            }
+          );
+        },
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+    });
+
+    await PlacesUtils.history.clear();
+  }
+);
+
+add_task(
+  async function test_replaceFaviconDataFromDataURL_beforeRegularAssign() {
+    info("test replaceFaviconDataFromDataURL before replaceFaviconData");
+
+    let pageURI = uri("http://test7.bar/");
+    await PlacesTestUtils.addVisits(pageURI);
+
+    let firstFavicon = createFavicon("favicon13.png");
+    let secondFavicon = createFavicon("favicon14.png");
+
+    iconsvc.replaceFaviconDataFromDataURL(
+      firstFavicon.uri,
+      createDataURLForFavicon(firstFavicon),
+      0,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+    iconsvc.replaceFaviconData(
+      firstFavicon.uri,
+      secondFavicon.data,
+      secondFavicon.mimetype
+    );
+
+    await new Promise(resolve => {
+      iconsvc.setAndFetchFaviconForPage(
+        pageURI,
+        firstFavicon.uri,
+        true,
+        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+        function test_replaceFaviconDataFromDataURL_beforeRegularAssign_check(
+          aURI,
+          aDataLen,
+          aData,
+          aMimeType
+        ) {
+          checkCallbackSucceeded(
+            aMimeType,
+            aData,
+            secondFavicon.mimetype,
+            secondFavicon.data
+          );
+          checkFaviconDataForPage(
+            pageURI,
+            secondFavicon.mimetype,
+            secondFavicon.data,
+            function test_replaceFaviconDataFromDataURL_beforeRegularAssign_callback() {
+              firstFavicon.file.remove(false);
+              secondFavicon.file.remove(false);
+              resolve();
+            }
+          );
+        },
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+    });
+
+    await PlacesUtils.history.clear();
+  }
+);
+
+/* toBase64 copied from image/test/unit/test_encoder_png.js */
+
+/* Convert data (an array of integers) to a Base64 string. */
+const toBase64Table =
+  // eslint-disable-next-line no-useless-concat
+  "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789+/";
+const base64Pad = "=";
+function toBase64(data) {
+  let result = "";
+  let length = data.length;
+  let i;
+  // Convert every three bytes to 4 ascii characters.
+  for (i = 0; i < length - 2; i += 3) {
+    result += toBase64Table[data[i] >> 2];
+    result += toBase64Table[((data[i] & 0x03) << 4) + (data[i + 1] >> 4)];
+    result += toBase64Table[((data[i + 1] & 0x0f) << 2) + (data[i + 2] >> 6)];
+    result += toBase64Table[data[i + 2] & 0x3f];
+  }
+
+  // Convert the remaining 1 or 2 bytes, pad out to 4 characters.
+  if (length % 3) {
+    i = length - (length % 3);
+    result += toBase64Table[data[i] >> 2];
+    if (length % 3 == 2) {
+      result += toBase64Table[((data[i] & 0x03) << 4) + (data[i + 1] >> 4)];
+      result += toBase64Table[(data[i + 1] & 0x0f) << 2];
+      result += base64Pad;
+    } else {
+      result += toBase64Table[(data[i] & 0x03) << 4];
+      result += base64Pad + base64Pad;
+    }
+  }
+
+  return result;
+}
diff --git a/toolkit/components/places/tests/favicons/test_root_icons.js b/toolkit/components/places/tests/favicons/test_root_icons.js
new file mode 100644
index 0000000000..f0487cc162
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_root_icons.js
@@ -0,0 +1,246 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests root icons associations and expiration
+ */
+
+add_task(async function () {
+  let pageURI = NetUtil.newURI("http://www.places.test/page/");
+  await PlacesTestUtils.addVisits(pageURI);
+  let faviconURI = NetUtil.newURI("http://www.places.test/favicon.ico");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    faviconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, faviconURI);
+
+  // Sanity checks.
+  Assert.equal(await getFaviconUrlForPage(pageURI), faviconURI.spec);
+  Assert.equal(
+    await getFaviconUrlForPage("https://places.test/somethingelse/"),
+    faviconURI.spec
+  );
+
+  // Check database entries.
+  await PlacesTestUtils.promiseAsyncUpdates();
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute("SELECT * FROM moz_icons");
+  Assert.equal(rows.length, 1, "There should only be 1 icon entry");
+  Assert.equal(
+    rows[0].getResultByName("root"),
+    1,
+    "It should be marked as a root icon"
+  );
+  rows = await db.execute("SELECT * FROM moz_pages_w_icons");
+  Assert.equal(rows.length, 0, "There should be no page entry");
+  rows = await db.execute("SELECT * FROM moz_icons_to_pages");
+  Assert.equal(rows.length, 0, "There should be no relation entry");
+
+  // Add another pages to the same host. The icon should not be removed.
+  await PlacesTestUtils.addVisits("http://places.test/page2/");
+  await PlacesUtils.history.remove(pageURI);
+
+  // Still works since the icon has not been removed.
+  Assert.equal(await getFaviconUrlForPage(pageURI), faviconURI.spec);
+
+  // Remove all the pages for the given domain.
+  await PlacesUtils.history.remove("http://places.test/page2/");
+  // The icon should be removed along with the domain.
+  rows = await db.execute("SELECT * FROM moz_icons");
+  Assert.equal(rows.length, 0, "The icon should have been removed");
+});
+
+add_task(async function test_removePagesByTimeframe() {
+  const BASE_URL = "http://www.places.test";
+  // Add a visit in the past with no directly associated icon.
+  let oldPageURI = NetUtil.newURI(`${BASE_URL}/old/`);
+  await PlacesTestUtils.addVisits({
+    uri: oldPageURI,
+    visitDate: new Date(Date.now() - 86400000),
+  });
+  // And another more recent visit.
+  let pageURI = NetUtil.newURI(`${BASE_URL}/page/`);
+  await PlacesTestUtils.addVisits({
+    uri: pageURI,
+    visitDate: new Date(Date.now() - 7200000),
+  });
+
+  // Add a normal icon to the most recent page.
+  let faviconURI = NetUtil.newURI(`${BASE_URL}/page/favicon.ico`);
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    faviconURI,
+    SMALLSVG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, faviconURI);
+  // Add a root icon to the most recent page.
+  let rootIconURI = NetUtil.newURI(`${BASE_URL}/favicon.ico`);
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    rootIconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, rootIconURI);
+
+  // Sanity checks.
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI),
+    faviconURI.spec,
+    "Should get the biggest icon"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI, 1),
+    rootIconURI.spec,
+    "Should get the smallest icon"
+  );
+  Assert.equal(
+    await getFaviconUrlForPage(oldPageURI),
+    rootIconURI.spec,
+    "Should get the root icon"
+  );
+
+  info("Removing the newer page, not the old one");
+  await PlacesUtils.history.removeByFilter({
+    beginDate: new Date(Date.now() - 14400000),
+    endDate: new Date(),
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute("SELECT * FROM moz_icons");
+  Assert.equal(rows.length, 1, "There should only be 1 icon entry");
+  Assert.equal(
+    rows[0].getResultByName("root"),
+    1,
+    "It should be marked as a root icon"
+  );
+  rows = await db.execute("SELECT * FROM moz_pages_w_icons");
+  Assert.equal(rows.length, 0, "There should be no page entry");
+  rows = await db.execute("SELECT * FROM moz_icons_to_pages");
+  Assert.equal(rows.length, 0, "There should be no relation entry");
+
+  await PlacesUtils.history.removeByFilter({
+    beginDate: new Date(0),
+    endDt: new Date(),
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  rows = await db.execute("SELECT * FROM moz_icons");
+  // Debug logging for possible intermittent failure (bug 1358368).
+  if (rows.length) {
+    dump_table("moz_icons");
+  }
+  Assert.equal(rows.length, 0, "There should be no icon entry");
+});
+
+add_task(async function test_different_host() {
+  let pageURI = NetUtil.newURI("http://places.test/page/");
+  await PlacesTestUtils.addVisits(pageURI);
+  let faviconURI = NetUtil.newURI("http://mozilla.test/favicon.ico");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    faviconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI, faviconURI);
+
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI),
+    faviconURI.spec,
+    "Should get the png icon"
+  );
+  // Check the icon is not marked as a root icon in the database.
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(
+    "SELECT root FROM moz_icons WHERE icon_url = :url",
+    { url: faviconURI.spec }
+  );
+  Assert.strictEqual(rows[0].getResultByName("root"), 0);
+});
+
+add_task(async function test_same_size() {
+  // Add two icons with the same size, one is a root icon. Check that the
+  // non-root icon is preferred when a smaller size is requested.
+  let data = readFileData(do_get_file("favicon-normal32.png"));
+  let pageURI = NetUtil.newURI("http://new_places.test/page/");
+  await PlacesTestUtils.addVisits(pageURI);
+
+  let faviconURI = NetUtil.newURI("http://new_places.test/favicon.ico");
+  PlacesUtils.favicons.replaceFaviconData(faviconURI, data, "image/png");
+  await setFaviconForPage(pageURI, faviconURI);
+  faviconURI = NetUtil.newURI("http://new_places.test/another_icon.ico");
+  PlacesUtils.favicons.replaceFaviconData(faviconURI, data, "image/png");
+  await setFaviconForPage(pageURI, faviconURI);
+
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI, 20),
+    faviconURI.spec,
+    "Should get the non-root icon"
+  );
+});
+
+add_task(async function test_root_on_different_host() {
+  async function getRootValue(url) {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(
+      "SELECT root FROM moz_icons WHERE icon_url = :url",
+      { url }
+    );
+    return rows[0].getResultByName("root");
+  }
+
+  // Check that a root icon associated to 2 domains is not removed when the
+  // root domain is removed.
+  const TEST_URL1 = "http://places1.test/page/";
+  let pageURI1 = NetUtil.newURI(TEST_URL1);
+  await PlacesTestUtils.addVisits(pageURI1);
+
+  const TEST_URL2 = "http://places2.test/page/";
+  let pageURI2 = NetUtil.newURI(TEST_URL2);
+  await PlacesTestUtils.addVisits(pageURI2);
+
+  // Root favicon for TEST_URL1.
+  const ICON_URL = "http://places1.test/favicon.ico";
+  let iconURI = NetUtil.newURI(ICON_URL);
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    iconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI1, iconURI);
+  Assert.equal(await getRootValue(ICON_URL), 1, "Check root == 1");
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI1, 16),
+    ICON_URL,
+    "The icon should been found"
+  );
+
+  // Same favicon for TEST_URL2.
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    iconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    systemPrincipal
+  );
+  await setFaviconForPage(pageURI2, iconURI);
+  Assert.equal(await getRootValue(ICON_URL), 1, "Check root == 1");
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI2, 16),
+    ICON_URL,
+    "The icon should be found"
+  );
+
+  await PlacesUtils.history.remove(pageURI1);
+
+  Assert.equal(
+    await getFaviconUrlForPage(pageURI2, 16),
+    ICON_URL,
+    "The icon should not have been removed"
+  );
+});
diff --git a/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage.js b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage.js
new file mode 100644
index 0000000000..1b4ea87ec0
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage.js
@@ -0,0 +1,123 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file tests the normal operation of setAndFetchFaviconForPage.
+
+let gTests = [
+  {
+    desc: "Normal test",
+    href: "http://example.com/normal",
+    loadType: PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    async setup() {
+      await PlacesTestUtils.addVisits({
+        uri: this.href,
+        transition: TRANSITION_TYPED,
+      });
+    },
+  },
+  {
+    desc: "Bookmarked about: uri",
+    href: "about:testAboutURI_bookmarked",
+    loadType: PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    async setup() {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: this.href,
+      });
+    },
+  },
+  {
+    desc: "Bookmarked in private window",
+    href: "http://example.com/privateBrowsing_bookmarked",
+    loadType: PlacesUtils.favicons.FAVICON_LOAD_PRIVATE,
+    async setup() {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: this.href,
+      });
+    },
+  },
+  {
+    desc: "Bookmarked with disabled history",
+    href: "http://example.com/disabledHistory_bookmarked",
+    loadType: PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    async setup() {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: this.href,
+      });
+      Services.prefs.setBoolPref("places.history.enabled", false);
+    },
+    clean() {
+      Services.prefs.setBoolPref("places.history.enabled", true);
+    },
+  },
+];
+
+add_task(async function () {
+  let faviconURI = SMALLPNG_DATA_URI;
+  let faviconMimeType = "image/png";
+
+  registerCleanupFunction(async function () {
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  });
+
+  for (let test of gTests) {
+    info(test.desc);
+    let pageURI = PlacesUtils.toURI(test.href);
+
+    await test.setup();
+
+    let pageGuid;
+    let promise = PlacesTestUtils.waitForNotification(
+      "favicon-changed",
+      events =>
+        events.some(e => {
+          if (e.url == pageURI.spec && e.faviconUrl == faviconURI.spec) {
+            pageGuid = e.pageGuid;
+            return true;
+          }
+          return false;
+        })
+    );
+
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      pageURI,
+      faviconURI,
+      true,
+      test.private,
+      null,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+
+    await promise;
+
+    Assert.equal(
+      pageGuid,
+      await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+        url: pageURI,
+      }),
+      "Page guid is correct"
+    );
+    let { dataLen, data, mimeType } = await PlacesUtils.promiseFaviconData(
+      pageURI.spec
+    );
+    Assert.equal(faviconMimeType, mimeType, "Check expected MimeType");
+    Assert.equal(
+      SMALLPNG_DATA_LEN,
+      data.length,
+      "Check favicon data for the given page matches the provided data"
+    );
+    Assert.equal(
+      dataLen,
+      data.length,
+      "Check favicon dataLen for the given page matches the provided data"
+    );
+
+    if (test.clean) {
+      await test.clean();
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_failures.js b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_failures.js
new file mode 100644
index 0000000000..1901ca86a7
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_failures.js
@@ -0,0 +1,156 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file tests setAndFetchFaviconForPage when it is called with invalid
+ * arguments, and when no favicon is stored for the given arguments.
+ */
+
+let faviconURI = Services.io.newURI(
+  "http://example.org/tests/toolkit/components/places/tests/browser/favicon-normal16.png"
+);
+add_task(async function () {
+  registerCleanupFunction(async function () {
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  });
+
+  // We'll listen for favicon changes for the whole test, to ensure only the
+  // last one will send a notification. Due to thread serialization, at that
+  // point we can be sure previous calls didn't send a notification.
+  let lastPageURI = Services.io.newURI("http://example.com/verification");
+  let promiseIconChanged = PlacesTestUtils.waitForNotification(
+    "favicon-changed",
+    events =>
+      events.some(
+        e => e.url == lastPageURI.spec && e.faviconUrl == SMALLPNG_DATA_URI.spec
+      )
+  );
+
+  info("Test null page uri");
+  Assert.throws(
+    () => {
+      PlacesUtils.favicons.setAndFetchFaviconForPage(
+        null,
+        faviconURI,
+        true,
+        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+        null,
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+    },
+    /NS_ERROR_ILLEGAL_VALUE/,
+    "Exception expected because aPageURI is null"
+  );
+
+  info("Test null favicon uri");
+  Assert.throws(
+    () => {
+      PlacesUtils.favicons.setAndFetchFaviconForPage(
+        Services.io.newURI("http://example.com/null_faviconURI"),
+        null,
+        true,
+        PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+        null,
+        Services.scriptSecurityManager.getSystemPrincipal()
+      );
+    },
+    /NS_ERROR_ILLEGAL_VALUE/,
+    "Exception expected because aFaviconURI is null."
+  );
+
+  info("Test about uri");
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    Services.io.newURI("about:testAboutURI"),
+    faviconURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  info("Test private browsing non bookmarked uri");
+  let pageURI = Services.io.newURI("http://example.com/privateBrowsing");
+  await PlacesTestUtils.addVisits({
+    uri: pageURI,
+    transitionType: TRANSITION_TYPED,
+  });
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    pageURI,
+    faviconURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  info("Test disabled history");
+  pageURI = Services.io.newURI("http://example.com/disabledHistory");
+  await PlacesTestUtils.addVisits({
+    uri: pageURI,
+    transition: TRANSITION_TYPED,
+  });
+  Services.prefs.setBoolPref("places.history.enabled", false);
+
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    pageURI,
+    faviconURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  // The setAndFetchFaviconForPage function calls CanAddURI synchronously, thus
+  // we can set the preference back to true immediately.
+  Services.prefs.setBoolPref("places.history.enabled", true);
+
+  info("Test error icon");
+  // This error icon must stay in sync with FAVICON_ERRORPAGE_URL in
+  // nsIFaviconService.idl and aboutNetError.xhtml.
+  let faviconErrorPageURI = Services.io.newURI(
+    "chrome://global/skin/icons/info.svg"
+  );
+  pageURI = Services.io.newURI("http://example.com/errorIcon");
+  await PlacesTestUtils.addVisits({
+    uri: pageURI,
+    transition: TRANSITION_TYPED,
+  });
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    pageURI,
+    faviconErrorPageURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  info("Test nonexisting page");
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    Services.io.newURI("http://example.com/nonexistingPage"),
+    faviconURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  info("Final sanity check");
+  // This is the only test that should cause the waitForFaviconChanged
+  // callback to be invoked.
+  await PlacesTestUtils.addVisits({
+    uri: lastPageURI,
+    transition: TRANSITION_TYPED,
+  });
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    lastPageURI,
+    SMALLPNG_DATA_URI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await promiseIconChanged;
+});
diff --git a/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_redirects.js b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_redirects.js
new file mode 100644
index 0000000000..feda238f97
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_setAndFetchFaviconForPage_redirects.js
@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file tests setAndFetchFaviconForPage on bookmarked redirects.
+
+add_task(async function same_host_redirect() {
+  // Add a bookmarked page that redirects to another page, set a favicon on the
+  // latter and check the former gets it too, if they are in the same host.
+  let srcUrl = "http://bookmarked.com/";
+  let destUrl = "https://other.bookmarked.com/";
+  await PlacesTestUtils.addVisits([
+    { uri: srcUrl, transition: TRANSITION_LINK },
+    {
+      uri: destUrl,
+      transition: TRANSITION_REDIRECT_TEMPORARY,
+      referrer: srcUrl,
+    },
+  ]);
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: srcUrl,
+  });
+
+  registerCleanupFunction(async function () {
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  });
+
+  let promise = PlacesTestUtils.waitForNotification("favicon-changed", events =>
+    events.some(e => e.url == srcUrl && e.faviconUrl == SMALLPNG_DATA_URI.spec)
+  );
+
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    Services.io.newURI(destUrl),
+    SMALLPNG_DATA_URI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await promise;
+
+  // The favicon should be set also on the bookmarked url that redirected.
+  let { dataLen } = await PlacesUtils.promiseFaviconData(srcUrl);
+  Assert.equal(dataLen, SMALLPNG_DATA_LEN, "Check favicon dataLen");
+});
+
+add_task(async function other_host_redirect() {
+  // Add a bookmarked page that redirects to another page, set a favicon on the
+  // latter and check the former gets it too, if they are in the same host.
+  let srcUrl = "http://first.com/";
+  let destUrl = "https://notfirst.com/";
+  await PlacesTestUtils.addVisits([
+    { uri: srcUrl, transition: TRANSITION_LINK },
+    {
+      uri: destUrl,
+      transition: TRANSITION_REDIRECT_TEMPORARY,
+      referrer: srcUrl,
+    },
+  ]);
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: srcUrl,
+  });
+
+  let promise = Promise.race([
+    PlacesTestUtils.waitForNotification("favicon-changed", events =>
+      events.some(
+        e => e.url == srcUrl && e.faviconUrl == SMALLPNG_DATA_URI.spec
+      )
+    ),
+    new Promise((resolve, reject) =>
+      do_timeout(300, () => reject(new Error("timeout")))
+    ),
+  ]);
+
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    Services.io.newURI(destUrl),
+    SMALLPNG_DATA_URI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await Assert.rejects(promise, /timeout/);
+});
diff --git a/toolkit/components/places/tests/favicons/test_svg_favicon.js b/toolkit/components/places/tests/favicons/test_svg_favicon.js
new file mode 100644
index 0000000000..8d9f2edf11
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/test_svg_favicon.js
@@ -0,0 +1,34 @@
+const PAGEURI = NetUtil.newURI("http://deliciousbacon.com/");
+
+add_task(async function () {
+  // First, add a history entry or else Places can't save a favicon.
+  await PlacesTestUtils.addVisits({
+    uri: PAGEURI,
+    transition: TRANSITION_LINK,
+    visitDate: Date.now() * 1000,
+  });
+
+  await new Promise(resolve => {
+    function onSetComplete(aURI, aDataLen, aData, aMimeType, aWidth) {
+      equal(aURI.spec, SMALLSVG_DATA_URI.spec, "setFavicon aURI check");
+      equal(aDataLen, 263, "setFavicon aDataLen check");
+      equal(aMimeType, "image/svg+xml", "setFavicon aMimeType check");
+      dump(aWidth);
+      resolve();
+    }
+
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      PAGEURI,
+      SMALLSVG_DATA_URI,
+      false,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      onSetComplete,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  let data = await PlacesUtils.promiseFaviconData(PAGEURI.spec);
+  equal(data.uri.spec, SMALLSVG_DATA_URI.spec, "getFavicon aURI check");
+  equal(data.dataLen, 263, "getFavicon aDataLen check");
+  equal(data.mimeType, "image/svg+xml", "getFavicon aMimeType check");
+});
diff --git a/toolkit/components/places/tests/favicons/xpcshell.ini b/toolkit/components/places/tests/favicons/xpcshell.ini
new file mode 100644
index 0000000000..86616c1631
--- /dev/null
+++ b/toolkit/components/places/tests/favicons/xpcshell.ini
@@ -0,0 +1,49 @@
+[DEFAULT]
+head = head_favicons.js
+skip-if = toolkit == 'android'
+support-files =
+  expected-favicon-animated16.png.png
+  expected-favicon-big32.jpg.png
+  expected-favicon-big4.jpg.png
+  expected-favicon-big16.ico.png
+  expected-favicon-big48.ico.png
+  expected-favicon-big64.png.png
+  expected-favicon-scale160x3.jpg.png
+  expected-favicon-scale3x160.jpg.png
+  favicon-animated16.png
+  favicon-big16.ico
+  favicon-big32.jpg
+  favicon-big4.jpg
+  favicon-big48.ico
+  favicon-big64.png
+  favicon-multi.ico
+  favicon-multi-frame16.png
+  favicon-multi-frame32.png
+  favicon-multi-frame64.png
+  favicon-normal16.png
+  favicon-normal32.png
+  favicon-scale160x3.jpg
+  favicon-scale3x160.jpg
+  noise.png
+
+[test_copyFavicons.js]
+[test_expireAllFavicons.js]
+[test_expire_migrated_icons.js]
+[test_expire_on_new_icons.js]
+[test_favicons_conversions.js]
+[test_favicons_protocols_ref.js]
+[test_getFaviconDataForPage.js]
+[test_getFaviconURLForPage.js]
+[test_heavy_favicon.js]
+[test_incremental_vacuum.js]
+[test_moz-anno_favicon_mime_type.js]
+[test_multiple_frames.js]
+[test_page-icon_protocol.js]
+[test_query_result_favicon_changed_on_child.js]
+[test_replaceFaviconData.js]
+[test_replaceFaviconDataFromDataURL.js]
+[test_root_icons.js]
+[test_setAndFetchFaviconForPage.js]
+[test_setAndFetchFaviconForPage_failures.js]
+[test_setAndFetchFaviconForPage_redirects.js]
+[test_svg_favicon.js]
diff --git a/toolkit/components/places/tests/gtest/mock_Link.h b/toolkit/components/places/tests/gtest/mock_Link.h
new file mode 100644
index 0000000000..aa83a2fe7b
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/mock_Link.h
@@ -0,0 +1,72 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This is a mock Link object which can be used in tests.
+ */
+
+#ifndef mock_Link_h__
+#define mock_Link_h__
+
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/dom/Link.h"
+#include "mozilla/StaticPrefs_layout.h"
+
+class mock_Link : public mozilla::dom::Link {
+ public:
+  NS_DECL_ISUPPORTS
+
+  typedef void (*Handler)(State);
+
+  explicit mock_Link(Handler aHandlerFunction, bool aRunNextTest = true)
+      : mozilla::dom::Link(), mRunNextTest(aRunNextTest) {
+    AwaitNewNotification(aHandlerFunction);
+  }
+
+  void VisitedQueryFinished(bool aVisited) final {
+    // Notify our callback function.
+    mHandler(aVisited ? State::Visited : State::Unvisited);
+
+    // Break the cycle so the object can be destroyed.
+    mDeathGrip = nullptr;
+  }
+
+  size_t SizeOfExcludingThis(mozilla::SizeOfState& aState) const final {
+    return 0;  // the value shouldn't matter
+  }
+
+  void NodeInfoChanged(mozilla::dom::Document* aOldDoc) final {}
+
+  bool GotNotified() const { return !mDeathGrip; }
+
+  void AwaitNewNotification(Handler aNewHandler) {
+    MOZ_ASSERT(!mDeathGrip, "Still waiting for a notification");
+    // Create a cyclic ownership, so that the link will be released only
+    // after its status has been updated.  This will ensure that, when it should
+    // run the next test, it will happen at the end of the test function, if
+    // the link status has already been set before.  Indeed the link status is
+    // updated on a separate connection, thus may happen at any time.
+    mDeathGrip = this;
+    mHandler = aNewHandler;
+  }
+
+ protected:
+  ~mock_Link() {
+    // Run the next test if we are supposed to.
+    if (mRunNextTest) {
+      run_next_test();
+    }
+  }
+
+ private:
+  Handler mHandler = nullptr;
+  bool mRunNextTest;
+  RefPtr mDeathGrip;
+};
+
+NS_IMPL_ISUPPORTS(mock_Link, mozilla::dom::Link)
+
+#endif  // mock_Link_h__
diff --git a/toolkit/components/places/tests/gtest/moz.build b/toolkit/components/places/tests/gtest/moz.build
new file mode 100644
index 0000000000..eb7157efc5
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/moz.build
@@ -0,0 +1,12 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+    "test_casing.cpp",
+    "test_IHistory.cpp",
+]
+
+FINAL_LIBRARY = "xul-gtest"
diff --git a/toolkit/components/places/tests/gtest/places_test_harness.h b/toolkit/components/places/tests/gtest/places_test_harness.h
new file mode 100644
index 0000000000..f2d3e06c35
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/places_test_harness.h
@@ -0,0 +1,353 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "nsThreadUtils.h"
+#include "nsDocShellCID.h"
+
+#include "nsToolkitCompsCID.h"
+#include "nsServiceManagerUtils.h"
+#include "nsINavHistoryService.h"
+#include "nsIObserverService.h"
+#include "nsIThread.h"
+#include "nsIURI.h"
+#include "mozilla/IHistory.h"
+#include "mozilla/SpinEventLoopUntil.h"
+#include "mozIStorageConnection.h"
+#include "mozIStorageStatement.h"
+#include "mozIStorageAsyncStatement.h"
+#include "mozIStorageStatementCallback.h"
+#include "mozIStoragePendingStatement.h"
+#include "nsIObserver.h"
+#include "prinrval.h"
+#include "prtime.h"
+#include "mozilla/Attributes.h"
+
+#define WAITFORTOPIC_TIMEOUT_SECONDS 5
+
+#define do_check_true(aCondition) EXPECT_TRUE(aCondition)
+
+#define do_check_false(aCondition) EXPECT_FALSE(aCondition)
+
+#define do_check_success(aResult) do_check_true(NS_SUCCEEDED(aResult))
+
+#define do_check_eq(aExpected, aActual) do_check_true(aExpected == aActual)
+
+struct Test {
+  void (*func)(void);
+  const char* const name;
+};
+#define PTEST(aName) \
+  { aName, #aName }
+
+/**
+ * Runs the next text.
+ */
+void run_next_test();
+
+/**
+ * To be used around asynchronous work.
+ */
+void do_test_pending();
+void do_test_finished();
+
+/**
+ * Spins current thread until a topic is received.
+ */
+class WaitForTopicSpinner final : public nsIObserver {
+ public:
+  NS_DECL_ISUPPORTS
+
+  explicit WaitForTopicSpinner(const char* const aTopic)
+      : mTopicReceived(false), mStartTime(PR_IntervalNow()) {
+    nsCOMPtr observerService =
+        do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+    do_check_true(observerService);
+    (void)observerService->AddObserver(this, aTopic, false);
+  }
+
+  void Spin() {
+    bool timedOut = false;
+    mozilla::SpinEventLoopUntil(
+        "places:WaitForTopicSpinner::Spin"_ns, [&]() -> bool {
+          if (mTopicReceived) {
+            return true;
+          }
+
+          if ((PR_IntervalNow() - mStartTime) >
+              (WAITFORTOPIC_TIMEOUT_SECONDS * PR_USEC_PER_SEC)) {
+            timedOut = true;
+            return true;
+          }
+
+          return false;
+        });
+
+    if (timedOut) {
+      // Timed out waiting for the topic.
+      do_check_true(false);
+    }
+  }
+
+  NS_IMETHOD Observe(nsISupports* aSubject, const char* aTopic,
+                     const char16_t* aData) override {
+    mTopicReceived = true;
+    nsCOMPtr observerService =
+        do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+    do_check_true(observerService);
+    (void)observerService->RemoveObserver(this, aTopic);
+    return NS_OK;
+  }
+
+ private:
+  ~WaitForTopicSpinner() = default;
+
+  bool mTopicReceived;
+  PRIntervalTime mStartTime;
+};
+NS_IMPL_ISUPPORTS(WaitForTopicSpinner, nsIObserver)
+
+/**
+ * Spins current thread until an async statement is executed.
+ */
+class PlacesAsyncStatementSpinner final : public mozIStorageStatementCallback {
+ public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGESTATEMENTCALLBACK
+
+  PlacesAsyncStatementSpinner();
+  void SpinUntilCompleted();
+  uint16_t completionReason;
+
+ protected:
+  ~PlacesAsyncStatementSpinner() = default;
+
+  volatile bool mCompleted;
+};
+
+NS_IMPL_ISUPPORTS(PlacesAsyncStatementSpinner, mozIStorageStatementCallback)
+
+PlacesAsyncStatementSpinner::PlacesAsyncStatementSpinner()
+    : completionReason(0), mCompleted(false) {}
+
+NS_IMETHODIMP
+PlacesAsyncStatementSpinner::HandleResult(mozIStorageResultSet* aResultSet) {
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+PlacesAsyncStatementSpinner::HandleError(mozIStorageError* aError) {
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+PlacesAsyncStatementSpinner::HandleCompletion(uint16_t aReason) {
+  completionReason = aReason;
+  mCompleted = true;
+  return NS_OK;
+}
+
+void PlacesAsyncStatementSpinner::SpinUntilCompleted() {
+  nsCOMPtr thread(::do_GetCurrentThread());
+  nsresult rv = NS_OK;
+  bool processed = true;
+  while (!mCompleted && NS_SUCCEEDED(rv)) {
+    rv = thread->ProcessNextEvent(true, &processed);
+  }
+}
+
+struct PlaceRecord {
+  int64_t id;
+  int32_t hidden;
+  int32_t typed;
+  int32_t visitCount;
+  nsCString guid;
+};
+
+struct VisitRecord {
+  int64_t id;
+  int64_t lastVisitId;
+  int32_t transitionType;
+};
+
+already_AddRefed do_get_IHistory() {
+  nsCOMPtr history = do_GetService(NS_IHISTORY_CONTRACTID);
+  do_check_true(history);
+  return history.forget();
+}
+
+already_AddRefed do_get_NavHistory() {
+  nsCOMPtr serv =
+      do_GetService(NS_NAVHISTORYSERVICE_CONTRACTID);
+  do_check_true(serv);
+  return serv.forget();
+}
+
+already_AddRefed do_get_db() {
+  nsCOMPtr history = do_get_NavHistory();
+  do_check_true(history);
+
+  nsCOMPtr dbConn;
+  nsresult rv = history->GetDBConnection(getter_AddRefs(dbConn));
+  do_check_success(rv);
+  return dbConn.forget();
+}
+
+/**
+ * Get the place record from the database.
+ *
+ * @param aURI The unique URI of the place we are looking up
+ * @param result Out parameter where the result is stored
+ */
+void do_get_place(nsIURI* aURI, PlaceRecord& result) {
+  nsCOMPtr dbConn = do_get_db();
+  nsCOMPtr stmt;
+
+  nsCString spec;
+  nsresult rv = aURI->GetSpec(spec);
+  do_check_success(rv);
+
+  rv = dbConn->CreateStatement(
+      nsLiteralCString(
+          "SELECT id, hidden, typed, visit_count, guid FROM moz_places "
+          "WHERE url_hash = hash(?1) AND url = ?1"),
+      getter_AddRefs(stmt));
+  do_check_success(rv);
+
+  rv = stmt->BindUTF8StringByIndex(0, spec);
+  do_check_success(rv);
+
+  bool hasResults;
+  rv = stmt->ExecuteStep(&hasResults);
+  do_check_success(rv);
+  if (!hasResults) {
+    result.id = 0;
+    return;
+  }
+
+  rv = stmt->GetInt64(0, &result.id);
+  do_check_success(rv);
+  rv = stmt->GetInt32(1, &result.hidden);
+  do_check_success(rv);
+  rv = stmt->GetInt32(2, &result.typed);
+  do_check_success(rv);
+  rv = stmt->GetInt32(3, &result.visitCount);
+  do_check_success(rv);
+  rv = stmt->GetUTF8String(4, result.guid);
+  do_check_success(rv);
+}
+
+/**
+ * Gets the most recent visit to a place.
+ *
+ * @param placeID ID from the moz_places table
+ * @param result Out parameter where visit is stored
+ */
+void do_get_lastVisit(int64_t placeId, VisitRecord& result) {
+  nsCOMPtr dbConn = do_get_db();
+  nsCOMPtr stmt;
+
+  nsresult rv = dbConn->CreateStatement(
+      nsLiteralCString(
+          "SELECT id, from_visit, visit_type FROM moz_historyvisits "
+          "WHERE place_id=?1 "
+          "LIMIT 1"),
+      getter_AddRefs(stmt));
+  do_check_success(rv);
+
+  rv = stmt->BindInt64ByIndex(0, placeId);
+  do_check_success(rv);
+
+  bool hasResults;
+  rv = stmt->ExecuteStep(&hasResults);
+  do_check_success(rv);
+
+  if (!hasResults) {
+    result.id = 0;
+    return;
+  }
+
+  rv = stmt->GetInt64(0, &result.id);
+  do_check_success(rv);
+  rv = stmt->GetInt64(1, &result.lastVisitId);
+  do_check_success(rv);
+  rv = stmt->GetInt32(2, &result.transitionType);
+  do_check_success(rv);
+}
+
+void do_wait_async_updates() {
+  nsCOMPtr db = do_get_db();
+  nsCOMPtr stmt;
+
+  db->CreateAsyncStatement("BEGIN EXCLUSIVE"_ns, getter_AddRefs(stmt));
+  nsCOMPtr pending;
+  (void)stmt->ExecuteAsync(nullptr, getter_AddRefs(pending));
+
+  db->CreateAsyncStatement("COMMIT"_ns, getter_AddRefs(stmt));
+  RefPtr spinner =
+      new PlacesAsyncStatementSpinner();
+  (void)stmt->ExecuteAsync(spinner, getter_AddRefs(pending));
+
+  spinner->SpinUntilCompleted();
+}
+
+/**
+ * Adds a URI to the database.
+ *
+ * @param aURI
+ *        The URI to add to the database.
+ */
+void addURI(nsIURI* aURI) {
+  nsCOMPtr history = do_GetService(NS_IHISTORY_CONTRACTID);
+  do_check_true(history);
+  nsresult rv = history->VisitURI(nullptr, aURI, nullptr,
+                                  mozilla::IHistory::TOP_LEVEL, 0);
+  do_check_success(rv);
+
+  do_wait_async_updates();
+}
+
+static const char TOPIC_PROFILE_CHANGE_QM[] = "profile-before-change-qm";
+static const char TOPIC_PLACES_CONNECTION_CLOSED[] = "places-connection-closed";
+
+class WaitForConnectionClosed final : public nsIObserver {
+  RefPtr mSpinner;
+
+  ~WaitForConnectionClosed() = default;
+
+ public:
+  NS_DECL_ISUPPORTS
+
+  WaitForConnectionClosed() {
+    nsCOMPtr os =
+        do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+    MOZ_ASSERT(os);
+    if (os) {
+      // The places-connection-closed notification happens because of things
+      // that occur during profile-before-change, so we use the stage after that
+      // to wait for it.
+      MOZ_ALWAYS_SUCCEEDS(
+          os->AddObserver(this, TOPIC_PROFILE_CHANGE_QM, false));
+    }
+    mSpinner = new WaitForTopicSpinner(TOPIC_PLACES_CONNECTION_CLOSED);
+  }
+
+  NS_IMETHOD Observe(nsISupports* aSubject, const char* aTopic,
+                     const char16_t* aData) override {
+    nsCOMPtr os =
+        do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+    MOZ_ASSERT(os);
+    if (os) {
+      MOZ_ALWAYS_SUCCEEDS(os->RemoveObserver(this, aTopic));
+    }
+
+    mSpinner->Spin();
+
+    return NS_OK;
+  }
+};
+
+NS_IMPL_ISUPPORTS(WaitForConnectionClosed, nsIObserver)
diff --git a/toolkit/components/places/tests/gtest/places_test_harness_tail.h b/toolkit/components/places/tests/gtest/places_test_harness_tail.h
new file mode 100644
index 0000000000..0464d14e0d
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/places_test_harness_tail.h
@@ -0,0 +1,89 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsWidgetsCID.h"
+#include "nsIUserIdleService.h"
+#include "mozilla/StackWalk.h"
+
+#ifndef TEST_NAME
+#  error "Must #define TEST_NAME before including places_test_harness_tail.h"
+#endif
+
+int gTestsIndex = 0;
+
+#define TEST_INFO_STR "TEST-INFO | "
+
+class RunNextTest : public mozilla::Runnable {
+ public:
+  RunNextTest() : mozilla::Runnable("RunNextTest") {}
+  NS_IMETHOD Run() override {
+    NS_ASSERTION(NS_IsMainThread(), "Not running on the main thread?");
+    if (gTestsIndex < int(mozilla::ArrayLength(gTests))) {
+      do_test_pending();
+      Test& test = gTests[gTestsIndex++];
+      (void)fprintf(stderr, TEST_INFO_STR "Running %s.\n", test.name);
+      test.func();
+    }
+
+    do_test_finished();
+    return NS_OK;
+  }
+};
+
+static const bool kDebugRunNextTest = false;
+
+void run_next_test() {
+  if (kDebugRunNextTest) {
+    printf_stderr("run_next_test()\n");
+    MozWalkTheStack(stderr);
+  }
+  nsCOMPtr event = new RunNextTest();
+  do_check_success(NS_DispatchToCurrentThread(event));
+}
+
+int gPendingTests = 0;
+
+void do_test_pending() {
+  NS_ASSERTION(NS_IsMainThread(), "Not running on the main thread?");
+  if (kDebugRunNextTest) {
+    printf_stderr("do_test_pending()\n");
+    MozWalkTheStack(stderr);
+  }
+  gPendingTests++;
+}
+
+void do_test_finished() {
+  NS_ASSERTION(NS_IsMainThread(), "Not running on the main thread?");
+  NS_ASSERTION(gPendingTests > 0, "Invalid pending test count!");
+  gPendingTests--;
+}
+
+void disable_idle_service() {
+  (void)fprintf(stderr, TEST_INFO_STR "Disabling Idle Service.\n");
+
+  nsCOMPtr idle =
+      do_GetService("@mozilla.org/widget/useridleservice;1");
+  idle->SetDisabled(true);
+}
+
+TEST(IHistory, Test)
+{
+  RefPtr spinClose = new WaitForConnectionClosed();
+
+  // Tinderboxes are constantly on idle.  Since idle tasks can interact with
+  // tests, causing random failures, disable the idle service.
+  disable_idle_service();
+
+  do_test_pending();
+  run_next_test();
+
+  // Spin the event loop until we've run out of tests to run.
+  mozilla::SpinEventLoopUntil("places:TEST(IHistory, Test)"_ns,
+                              [&]() { return !gPendingTests; });
+
+  // And let any other events finish before we quit.
+  (void)NS_ProcessPendingEvents(nullptr);
+}
diff --git a/toolkit/components/places/tests/gtest/test_IHistory.cpp b/toolkit/components/places/tests/gtest/test_IHistory.cpp
new file mode 100644
index 0000000000..1a54c2c401
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/test_IHistory.cpp
@@ -0,0 +1,445 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "places_test_harness.h"
+#include "nsIPrefBranch.h"
+#include "nsIPrefService.h"
+#include "nsString.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/SpinEventLoopUntil.h"
+#include "mozilla/StaticPrefs_layout.h"
+#include "nsNetUtil.h"
+
+#include "mock_Link.h"
+using namespace mozilla;
+using namespace mozilla::dom;
+
+/**
+ * This file tests the IHistory interface.
+ */
+
+////////////////////////////////////////////////////////////////////////////////
+//// Helper Methods
+
+void expect_visit(Link::State aState) {
+  do_check_true(aState == Link::State::Visited);
+}
+
+void expect_no_visit(Link::State aState) {
+  do_check_true(aState == Link::State::Unvisited);
+}
+
+already_AddRefed new_test_uri() {
+  // Create a unique spec.
+  static int32_t specNumber = 0;
+  nsCString spec = "http://mozilla.org/"_ns;
+  spec.AppendInt(specNumber++);
+
+  // Create the URI for the spec.
+  nsCOMPtr testURI;
+  nsresult rv = NS_NewURI(getter_AddRefs(testURI), spec);
+  do_check_success(rv);
+  return testURI.forget();
+}
+
+class VisitURIObserver final : public nsIObserver {
+  ~VisitURIObserver() = default;
+
+ public:
+  NS_DECL_ISUPPORTS
+
+  explicit VisitURIObserver(int aExpectedVisits = 1)
+      : mVisits(0), mExpectedVisits(aExpectedVisits) {
+    nsCOMPtr observerService =
+        do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+    do_check_true(observerService);
+    (void)observerService->AddObserver(this, "uri-visit-saved", false);
+  }
+
+  void WaitForNotification() {
+    SpinEventLoopUntil("places:VisitURIObserver::WaitForNotification"_ns,
+                       [&]() { return mVisits >= mExpectedVisits; });
+  }
+
+  NS_IMETHOD Observe(nsISupports* aSubject, const char* aTopic,
+                     const char16_t* aData) override {
+    mVisits++;
+
+    if (mVisits == mExpectedVisits) {
+      nsCOMPtr observerService =
+          do_GetService(NS_OBSERVERSERVICE_CONTRACTID);
+      (void)observerService->RemoveObserver(this, "uri-visit-saved");
+    }
+
+    return NS_OK;
+  }
+
+ private:
+  int mVisits;
+  int mExpectedVisits;
+};
+NS_IMPL_ISUPPORTS(VisitURIObserver, nsIObserver)
+
+////////////////////////////////////////////////////////////////////////////////
+//// Test Functions
+
+void test_set_places_enabled() {
+  // Ensure places is enabled for everyone.
+  nsresult rv;
+  nsCOMPtr prefBranch =
+      do_GetService(NS_PREFSERVICE_CONTRACTID, &rv);
+  do_check_success(rv);
+
+  rv = prefBranch->SetBoolPref("places.history.enabled", true);
+  do_check_success(rv);
+
+  // Run the next test.
+  run_next_test();
+}
+
+void test_wait_checkpoint() {
+  // This "fake" test is here to wait for the initial WAL checkpoint we force
+  // after creating the database schema, since that may happen at any time,
+  // and cause concurrent readers to access an older checkpoint.
+  nsCOMPtr db = do_get_db();
+  nsCOMPtr stmt;
+  db->CreateAsyncStatement("SELECT 1"_ns, getter_AddRefs(stmt));
+  RefPtr spinner =
+      new PlacesAsyncStatementSpinner();
+  nsCOMPtr pending;
+  (void)stmt->ExecuteAsync(spinner, getter_AddRefs(pending));
+  spinner->SpinUntilCompleted();
+
+  // Run the next test.
+  run_next_test();
+}
+
+// These variables are shared between part 1 and part 2 of the test.  Part 2
+// sets the nsCOMPtr's to nullptr, freeing the reference.
+namespace test_unvisited_does_not_notify {
+nsCOMPtr testURI;
+RefPtr testLink;
+}  // namespace test_unvisited_does_not_notify
+void test_unvisited_does_not_notify_part1() {
+  using namespace test_unvisited_does_not_notify;
+
+  // This test is done in two parts.  The first part registers for a URI that
+  // should not be visited.  We then run another test that will also do a
+  // lookup and will be notified.  Since requests are answered in the order they
+  // are requested (at least as long as the same URI isn't asked for later), we
+  // will know that the Link was not notified.
+
+  // First, we need a test URI.
+  testURI = new_test_uri();
+
+  // Create our test Link.
+  testLink = new mock_Link(expect_no_visit);
+
+  // Now, register our Link to be notified.
+  nsCOMPtr history = do_get_IHistory();
+  history->RegisterVisitedCallback(testURI, testLink);
+
+  // Run the next test.
+  run_next_test();
+}
+
+void test_visited_notifies() {
+  // First, we add our test URI to history.
+  nsCOMPtr testURI = new_test_uri();
+  addURI(testURI);
+
+  // Create our test Link.  The callback function will release the reference we
+  // have on the Link.
+  RefPtr link = new mock_Link(expect_visit);
+
+  // Now, register our Link to be notified.
+  nsCOMPtr history = do_get_IHistory();
+  history->RegisterVisitedCallback(testURI, link);
+
+  // Note: test will continue upon notification.
+}
+
+void test_unvisited_does_not_notify_part2() {
+  using namespace test_unvisited_does_not_notify;
+
+  SpinEventLoopUntil("places:test_unvisited_does_not_notify_part2"_ns,
+                     [&]() { return testLink->GotNotified(); });
+
+  // We would have had a failure at this point had the content node been told it
+  // was visited. Therefore, now we change it so that it expects a visited
+  // notification, and unregisters itself after addURI.
+  testLink->AwaitNewNotification(expect_visit);
+  addURI(testURI);
+
+  // Clear the stored variables now.
+  testURI = nullptr;
+  testLink = nullptr;
+}
+
+void test_same_uri_notifies_both() {
+  // First, we add our test URI to history.
+  nsCOMPtr testURI = new_test_uri();
+  addURI(testURI);
+
+  // Create our two test Links.  The callback function will release the
+  // reference we have on the Links.  Only the second Link should run the next
+  // test!
+  RefPtr link1 = new mock_Link(expect_visit, false);
+  RefPtr link2 = new mock_Link(expect_visit);
+
+  // Now, register our Link to be notified.
+  nsCOMPtr history = do_get_IHistory();
+  history->RegisterVisitedCallback(testURI, link1);
+  history->RegisterVisitedCallback(testURI, link2);
+
+  // Note: test will continue upon notification.
+}
+
+void test_unregistered_visited_does_not_notify() {
+  // This test must have a test that has a successful notification after it.
+  // The Link would have been notified by now if we were buggy and notified
+  // unregistered Links (due to request serialization).
+
+  nsCOMPtr testURI = new_test_uri();
+  RefPtr link = new mock_Link(expect_no_visit, false);
+  nsCOMPtr history(do_get_IHistory());
+  history->RegisterVisitedCallback(testURI, link);
+
+  // Unregister the Link.
+  history->UnregisterVisitedCallback(testURI, link);
+
+  // And finally add a visit for the URI.
+  addURI(testURI);
+
+  // If history tries to notify us, we'll either crash because the Link will
+  // have been deleted (we are the only thing holding a reference to it), or our
+  // expect_no_visit call back will produce a failure.  Either way, the test
+  // will be reported as a failure.
+
+  // Run the next test.
+  run_next_test();
+}
+
+void test_new_visit_notifies_waiting_Link() {
+  // Create our test Link.  The callback function will release the reference we
+  // have on the link.
+  //
+  // Note that this will query the database and we'll get an _unvisited_
+  // notification, then (after we addURI) a _visited_ one.
+  RefPtr link = new mock_Link(expect_no_visit);
+
+  // Now, register our content node to be notified.
+  nsCOMPtr testURI = new_test_uri();
+  nsCOMPtr history = do_get_IHistory();
+  history->RegisterVisitedCallback(testURI, link);
+
+  SpinEventLoopUntil("places:test_new_visit_notifies_waiting_Link"_ns,
+                     [&]() { return link->GotNotified(); });
+
+  link->AwaitNewNotification(expect_visit);
+
+  // Add ourselves to history.
+  addURI(testURI);
+
+  // Note: test will continue upon notification.
+}
+
+void test_RegisterVisitedCallback_returns_before_notifying() {
+  // Add a URI so that it's already in history.
+  nsCOMPtr testURI = new_test_uri();
+  addURI(testURI);
+
+  // Create our test Link.
+  RefPtr link = new mock_Link(expect_no_visit, false);
+
+  // Now, register our content node to be notified.  It should not be notified.
+  nsCOMPtr history = do_get_IHistory();
+  history->RegisterVisitedCallback(testURI, link);
+
+  // Remove ourselves as an observer.  We would have failed if we had been
+  // notified.
+  history->UnregisterVisitedCallback(testURI, link);
+
+  run_next_test();
+}
+
+void test_visituri_inserts() {
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+
+  RefPtr finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  do_get_place(visitedURI, place);
+
+  do_check_true(place.id > 0);
+  do_check_false(place.hidden);
+  do_check_false(place.typed);
+  do_check_eq(place.visitCount, 1);
+
+  run_next_test();
+}
+
+void test_visituri_updates() {
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+  RefPtr finisher;
+
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+  finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+  finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  do_get_place(visitedURI, place);
+
+  do_check_eq(place.visitCount, 2);
+
+  run_next_test();
+}
+
+void test_visituri_preserves_shown_and_typed() {
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+  // this simulates the uri visit happening in a frame.  Normally frame
+  // transitions would be hidden unless it was previously loaded top-level
+  history->VisitURI(nullptr, visitedURI, lastURI, 0, 0);
+
+  RefPtr finisher = new VisitURIObserver(2);
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  do_get_place(visitedURI, place);
+  do_check_false(place.hidden);
+
+  run_next_test();
+}
+
+void test_visituri_creates_visit() {
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+  RefPtr finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  VisitRecord visit;
+  do_get_place(visitedURI, place);
+  do_get_lastVisit(place.id, visit);
+
+  do_check_true(visit.id > 0);
+  do_check_eq(visit.lastVisitId, 0);
+  do_check_eq(visit.transitionType, nsINavHistoryService::TRANSITION_LINK);
+
+  run_next_test();
+}
+
+void test_visituri_transition_typed() {
+  nsCOMPtr navHistory = do_get_NavHistory();
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+
+  navHistory->MarkPageAsTyped(visitedURI);
+  history->VisitURI(nullptr, visitedURI, lastURI, mozilla::IHistory::TOP_LEVEL,
+                    0);
+  RefPtr finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  VisitRecord visit;
+  do_get_place(visitedURI, place);
+  do_get_lastVisit(place.id, visit);
+
+  do_check_true(visit.transitionType == nsINavHistoryService::TRANSITION_TYPED);
+
+  run_next_test();
+}
+
+void test_visituri_transition_embed() {
+  nsCOMPtr history = do_get_IHistory();
+  nsCOMPtr lastURI = new_test_uri();
+  nsCOMPtr visitedURI = new_test_uri();
+
+  history->VisitURI(nullptr, visitedURI, lastURI, 0, 0);
+  RefPtr finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  PlaceRecord place;
+  VisitRecord visit;
+  do_get_place(visitedURI, place);
+  do_get_lastVisit(place.id, visit);
+
+  do_check_eq(place.id, 0);
+  do_check_eq(visit.id, 0);
+
+  run_next_test();
+}
+
+void test_new_visit_adds_place_guid() {
+  // First, add a visit and wait.  This will also add a place.
+  nsCOMPtr visitedURI = new_test_uri();
+  nsCOMPtr history = do_get_IHistory();
+  nsresult rv = history->VisitURI(nullptr, visitedURI, nullptr,
+                                  mozilla::IHistory::TOP_LEVEL, 0);
+  do_check_success(rv);
+  RefPtr finisher = new VisitURIObserver();
+  finisher->WaitForNotification();
+
+  // Check that we have a guid for our visit.
+  PlaceRecord place;
+  do_get_place(visitedURI, place);
+  do_check_eq(place.visitCount, 1);
+  do_check_eq(place.guid.Length(), 12u);
+
+  run_next_test();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//// Test Harness
+
+/**
+ * Note: for tests marked "Order Important!", please see the test for details.
+ */
+Test gTests[] = {
+    PTEST(test_set_places_enabled),               // Must come first!
+    PTEST(test_wait_checkpoint),                  // Must come second!
+    PTEST(test_unvisited_does_not_notify_part1),  // Order Important!
+    PTEST(test_visited_notifies),
+    PTEST(test_unvisited_does_not_notify_part2),  // Order Important!
+    PTEST(test_same_uri_notifies_both),
+    PTEST(test_unregistered_visited_does_not_notify),  // Order Important!
+    PTEST(test_new_visit_notifies_waiting_Link),
+    PTEST(test_RegisterVisitedCallback_returns_before_notifying),
+    PTEST(test_visituri_inserts),
+    PTEST(test_visituri_updates),
+    PTEST(test_visituri_preserves_shown_and_typed),
+    PTEST(test_visituri_creates_visit),
+    PTEST(test_visituri_transition_typed),
+    PTEST(test_visituri_transition_embed),
+    PTEST(test_new_visit_adds_place_guid),
+};
+
+#define TEST_NAME "IHistory"
+#include "places_test_harness_tail.h"
diff --git a/toolkit/components/places/tests/gtest/test_casing.cpp b/toolkit/components/places/tests/gtest/test_casing.cpp
new file mode 100644
index 0000000000..079d64bbd0
--- /dev/null
+++ b/toolkit/components/places/tests/gtest/test_casing.cpp
@@ -0,0 +1,29 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "mozilla/intl/UnicodeProperties.h"
+
+// Verify the assertion in SQLFunctions.cpp / nextSearchCandidate that the
+// only non-ASCII characters that lower-case to ASCII ones are:
+//  * U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE
+//  * U+212A KELVIN SIGN
+TEST(MatchAutocompleteCasing, CaseAssumption)
+{
+  for (uint32_t c = 128; c < 0x110000; c++) {
+    if (c != 304 && c != 8490) {
+      ASSERT_GE(mozilla::intl::UnicodeProperties::ToLower(c), 128U);
+    }
+  }
+}
+
+// Verify the assertion that all ASCII characters lower-case to ASCII.
+TEST(MatchAutocompleteCasing, CaseAssumption2)
+{
+  for (uint32_t c = 0; c < 128; c++) {
+    ASSERT_LT(mozilla::intl::UnicodeProperties::ToLower(c), 128U);
+  }
+}
diff --git a/toolkit/components/places/tests/head_common.js b/toolkit/components/places/tests/head_common.js
new file mode 100644
index 0000000000..9ae5dd4e0f
--- /dev/null
+++ b/toolkit/components/places/tests/head_common.js
@@ -0,0 +1,928 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const NS_APP_USER_PROFILE_50_DIR = "ProfD";
+
+// Shortcuts to transitions type.
+const TRANSITION_LINK = Ci.nsINavHistoryService.TRANSITION_LINK;
+const TRANSITION_TYPED = Ci.nsINavHistoryService.TRANSITION_TYPED;
+const TRANSITION_BOOKMARK = Ci.nsINavHistoryService.TRANSITION_BOOKMARK;
+const TRANSITION_EMBED = Ci.nsINavHistoryService.TRANSITION_EMBED;
+const TRANSITION_FRAMED_LINK = Ci.nsINavHistoryService.TRANSITION_FRAMED_LINK;
+const TRANSITION_REDIRECT_PERMANENT =
+  Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT;
+const TRANSITION_REDIRECT_TEMPORARY =
+  Ci.nsINavHistoryService.TRANSITION_REDIRECT_TEMPORARY;
+const TRANSITION_DOWNLOAD = Ci.nsINavHistoryService.TRANSITION_DOWNLOAD;
+const TRANSITION_RELOAD = Ci.nsINavHistoryService.TRANSITION_RELOAD;
+
+const TITLE_LENGTH_MAX = 4096;
+
+var { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+  "resource://gre/modules/AppConstants.sys.mjs"
+);
+var { PlacesSyncUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesSyncUtils.sys.mjs"
+);
+
+ChromeUtils.defineESModuleGetters(this, {
+  BookmarkHTMLUtils: "resource://gre/modules/BookmarkHTMLUtils.sys.mjs",
+  BookmarkJSONUtils: "resource://gre/modules/BookmarkJSONUtils.sys.mjs",
+  FileUtils: "resource://gre/modules/FileUtils.sys.mjs",
+  PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs",
+  PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.sys.mjs",
+  PlacesTestUtils: "resource://testing-common/PlacesTestUtils.sys.mjs",
+  PlacesTransactions: "resource://gre/modules/PlacesTransactions.sys.mjs",
+  PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+  PromiseUtils: "resource://gre/modules/PromiseUtils.sys.mjs",
+  Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
+  TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs",
+  TestUtils: "resource://testing-common/TestUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  NetUtil: "resource://gre/modules/NetUtil.jsm",
+  ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "SMALLPNG_DATA_URI", function () {
+  return NetUtil.newURI(
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAA" +
+      "AAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg=="
+  );
+});
+const SMALLPNG_DATA_LEN = 67;
+
+XPCOMUtils.defineLazyGetter(this, "SMALLSVG_DATA_URI", function () {
+  return NetUtil.newURI(
+    "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy5" +
+      "3My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxMDAgMTAwIiBmaWxs" +
+      "PSIjNDI0ZTVhIj4NCiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgcj0iN" +
+      "DQiIHN0cm9rZT0iIzQyNGU1YSIgc3Ryb2tlLXdpZHRoPSIxMSIgZmlsbD" +
+      "0ibm9uZSIvPg0KICA8Y2lyY2xlIGN4PSI1MCIgY3k9IjI0LjYiIHI9IjY" +
+      "uNCIvPg0KICA8cmVjdCB4PSI0NSIgeT0iMzkuOSIgd2lkdGg9IjEwLjEi" +
+      "IGhlaWdodD0iNDEuOCIvPg0KPC9zdmc%2BDQo%3D"
+  );
+});
+
+XPCOMUtils.defineLazyGetter(this, "PlacesFrecencyRecalculator", () => {
+  return Cc["@mozilla.org/places/frecency-recalculator;1"].getService(
+    Ci.nsIObserver
+  ).wrappedJSObject;
+});
+
+var gTestDir = do_get_cwd();
+
+// Initialize profile.
+var gProfD = do_get_profile(true);
+
+Services.prefs.setBoolPref("browser.urlbar.usepreloadedtopurls.enabled", false);
+registerCleanupFunction(() =>
+  Services.prefs.clearUserPref("browser.urlbar.usepreloadedtopurls.enabled")
+);
+
+// Remove any old database.
+clearDB();
+
+/**
+ * Shortcut to create a nsIURI.
+ *
+ * @param aSpec
+ *        URLString of the uri.
+ */
+function uri(aSpec) {
+  return NetUtil.newURI(aSpec);
+}
+
+/**
+ * Gets the database connection.  If the Places connection is invalid it will
+ * try to create a new connection.
+ *
+ * @param [optional] aForceNewConnection
+ *        Forces creation of a new connection to the database.  When a
+ *        connection is asyncClosed it cannot anymore schedule async statements,
+ *        though connectionReady will keep returning true (Bug 726990).
+ *
+ * @return The database connection or null if unable to get one.
+ */
+var gDBConn;
+function DBConn(aForceNewConnection) {
+  if (!aForceNewConnection) {
+    let db = PlacesUtils.history.DBConnection;
+    if (db.connectionReady) {
+      return db;
+    }
+  }
+
+  // If the Places database connection has been closed, create a new connection.
+  if (!gDBConn || aForceNewConnection) {
+    let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+    file.append("places.sqlite");
+    let dbConn = (gDBConn = Services.storage.openDatabase(file));
+
+    // Be sure to cleanly close this connection.
+    promiseTopicObserved("profile-before-change").then(() =>
+      dbConn.asyncClose()
+    );
+  }
+
+  return gDBConn.connectionReady ? gDBConn : null;
+}
+
+/**
+ * Reads data from the provided inputstream.
+ *
+ * @return an array of bytes.
+ */
+function readInputStreamData(aStream) {
+  let bistream = Cc["@mozilla.org/binaryinputstream;1"].createInstance(
+    Ci.nsIBinaryInputStream
+  );
+  try {
+    bistream.setInputStream(aStream);
+    let expectedData = [];
+    let avail;
+    while ((avail = bistream.available())) {
+      expectedData = expectedData.concat(bistream.readByteArray(avail));
+    }
+    return expectedData;
+  } finally {
+    bistream.close();
+  }
+}
+
+/**
+ * Reads the data from the specified nsIFile.
+ *
+ * @param aFile
+ *        The nsIFile to read from.
+ * @return an array of bytes.
+ */
+function readFileData(aFile) {
+  let inputStream = Cc[
+    "@mozilla.org/network/file-input-stream;1"
+  ].createInstance(Ci.nsIFileInputStream);
+  // init the stream as RD_ONLY, -1 == default permissions.
+  inputStream.init(aFile, 0x01, -1, null);
+
+  // Check the returned size versus the expected size.
+  let size = inputStream.available();
+  let bytes = readInputStreamData(inputStream);
+  if (size != bytes.length) {
+    throw new Error("Didn't read expected number of bytes");
+  }
+  return bytes;
+}
+
+/**
+ * Reads the data from the named file, verifying the expected file length.
+ *
+ * @param aFileName
+ *        This file should be located in the same folder as the test.
+ * @param aExpectedLength
+ *        Expected length of the file.
+ *
+ * @return The array of bytes read from the file.
+ */
+function readFileOfLength(aFileName, aExpectedLength) {
+  let data = readFileData(do_get_file(aFileName));
+  Assert.equal(data.length, aExpectedLength);
+  return data;
+}
+
+/**
+ * Returns the base64-encoded version of the given string.  This function is
+ * similar to window.btoa, but is available to xpcshell tests also.
+ *
+ * @param aString
+ *        Each character in this string corresponds to a byte, and must be a
+ *        code point in the range 0-255.
+ *
+ * @return The base64-encoded string.
+ */
+function base64EncodeString(aString) {
+  var stream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(
+    Ci.nsIStringInputStream
+  );
+  stream.setData(aString, aString.length);
+  var encoder = Cc["@mozilla.org/scriptablebase64encoder;1"].createInstance(
+    Ci.nsIScriptableBase64Encoder
+  );
+  return encoder.encodeToString(stream, aString.length);
+}
+
+/**
+ * Compares two arrays, and returns true if they are equal.
+ *
+ * @param aArray1
+ *        First array to compare.
+ * @param aArray2
+ *        Second array to compare.
+ */
+function compareArrays(aArray1, aArray2) {
+  if (aArray1.length != aArray2.length) {
+    print("compareArrays: array lengths differ\n");
+    return false;
+  }
+
+  for (let i = 0; i < aArray1.length; i++) {
+    if (aArray1[i] != aArray2[i]) {
+      print(
+        "compareArrays: arrays differ at index " +
+          i +
+          ": " +
+          "(" +
+          aArray1[i] +
+          ") != (" +
+          aArray2[i] +
+          ")\n"
+      );
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Deletes a previously created sqlite file from the profile folder.
+ */
+function clearDB() {
+  try {
+    let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+    file.append("places.sqlite");
+    if (file.exists()) {
+      file.remove(false);
+    }
+  } catch (ex) {
+    dump("Exception: " + ex);
+  }
+}
+
+/**
+ * Dumps the rows of a table out to the console.
+ *
+ * @param aName
+ *        The name of the table or view to output.
+ */
+function dump_table(aName, dbConn) {
+  if (!dbConn) {
+    dbConn = DBConn();
+  }
+  let stmt = dbConn.createStatement("SELECT * FROM " + aName);
+
+  print("\n*** Printing data from " + aName);
+  let count = 0;
+  while (stmt.executeStep()) {
+    let columns = stmt.numEntries;
+
+    if (count == 0) {
+      // Print the column names.
+      for (let i = 0; i < columns; i++) {
+        dump(stmt.getColumnName(i) + "\t");
+      }
+      dump("\n");
+    }
+
+    // Print the rows.
+    for (let i = 0; i < columns; i++) {
+      switch (stmt.getTypeOfIndex(i)) {
+        case Ci.mozIStorageValueArray.VALUE_TYPE_NULL:
+          dump("NULL\t");
+          break;
+        case Ci.mozIStorageValueArray.VALUE_TYPE_INTEGER:
+          dump(stmt.getInt64(i) + "\t");
+          break;
+        case Ci.mozIStorageValueArray.VALUE_TYPE_FLOAT:
+          dump(stmt.getDouble(i) + "\t");
+          break;
+        case Ci.mozIStorageValueArray.VALUE_TYPE_TEXT:
+          dump(stmt.getString(i) + "\t");
+          break;
+      }
+    }
+    dump("\n");
+
+    count++;
+  }
+  print("*** There were a total of " + count + " rows of data.\n");
+
+  stmt.finalize();
+}
+
+/**
+ * Checks if an address is found in the database.
+ * @param aURI
+ *        nsIURI or address to look for.
+ * @return place id of the page or 0 if not found
+ */
+function page_in_database(aURI) {
+  let url = aURI instanceof Ci.nsIURI ? aURI.spec : aURI;
+  let stmt = DBConn().createStatement(
+    "SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url"
+  );
+  stmt.params.url = url;
+  try {
+    if (!stmt.executeStep()) {
+      return 0;
+    }
+    return stmt.getInt64(0);
+  } finally {
+    stmt.finalize();
+  }
+}
+
+/**
+ * Checks how many visits exist for a specified page.
+ * @param aURI
+ *        nsIURI or address to look for.
+ * @return number of visits found.
+ */
+function visits_in_database(aURI) {
+  let url = aURI instanceof Ci.nsIURI ? aURI.spec : aURI;
+  let stmt = DBConn().createStatement(
+    `SELECT count(*) FROM moz_historyvisits v
+     JOIN moz_places h ON h.id = v.place_id
+     WHERE url_hash = hash(:url) AND url = :url`
+  );
+  stmt.params.url = url;
+  try {
+    if (!stmt.executeStep()) {
+      return 0;
+    }
+    return stmt.getInt64(0);
+  } finally {
+    stmt.finalize();
+  }
+}
+
+/**
+ * Allows waiting for an observer notification once.
+ *
+ * @param aTopic
+ *        Notification topic to observe.
+ *
+ * @return {Promise}
+ * @resolves The array [aSubject, aData] from the observed notification.
+ * @rejects Never.
+ */
+function promiseTopicObserved(aTopic) {
+  return new Promise(resolve => {
+    Services.obs.addObserver(function observe(
+      aObsSubject,
+      aObsTopic,
+      aObsData
+    ) {
+      Services.obs.removeObserver(observe, aObsTopic);
+      resolve([aObsSubject, aObsData]);
+    },
+    aTopic);
+  });
+}
+
+/**
+ * Simulates a Places shutdown.
+ */
+var shutdownPlaces = function () {
+  info("shutdownPlaces: starting");
+  let promise = new Promise(resolve => {
+    Services.obs.addObserver(resolve, "places-connection-closed");
+  });
+  let hs = PlacesUtils.history.QueryInterface(Ci.nsIObserver);
+  hs.observe(null, "profile-change-teardown", null);
+  info("shutdownPlaces: sent profile-change-teardown");
+  hs.observe(null, "test-simulate-places-shutdown", null);
+  info("shutdownPlaces: sent test-simulate-places-shutdown");
+  return promise.then(() => {
+    info("shutdownPlaces: complete");
+  });
+};
+
+const FILENAME_BOOKMARKS_HTML = "bookmarks.html";
+const FILENAME_BOOKMARKS_JSON =
+  "bookmarks-" + PlacesBackups.toISODateString(new Date()) + ".json";
+
+/**
+ * Creates a bookmarks.html file in the profile folder from a given source file.
+ *
+ * @param aFilename
+ *        Name of the file to copy to the profile folder.  This file must
+ *        exist in the directory that contains the test files.
+ *
+ * @return nsIFile object for the file.
+ */
+function create_bookmarks_html(aFilename) {
+  if (!aFilename) {
+    do_throw("you must pass a filename to create_bookmarks_html function");
+  }
+  remove_bookmarks_html();
+  let bookmarksHTMLFile = gTestDir.clone();
+  bookmarksHTMLFile.append(aFilename);
+  Assert.ok(bookmarksHTMLFile.exists());
+  bookmarksHTMLFile.copyTo(gProfD, FILENAME_BOOKMARKS_HTML);
+  let profileBookmarksHTMLFile = gProfD.clone();
+  profileBookmarksHTMLFile.append(FILENAME_BOOKMARKS_HTML);
+  Assert.ok(profileBookmarksHTMLFile.exists());
+  return profileBookmarksHTMLFile;
+}
+
+/**
+ * Remove bookmarks.html file from the profile folder.
+ */
+function remove_bookmarks_html() {
+  let profileBookmarksHTMLFile = gProfD.clone();
+  profileBookmarksHTMLFile.append(FILENAME_BOOKMARKS_HTML);
+  if (profileBookmarksHTMLFile.exists()) {
+    profileBookmarksHTMLFile.remove(false);
+    Assert.ok(!profileBookmarksHTMLFile.exists());
+  }
+}
+
+/**
+ * Check bookmarks.html file exists in the profile folder.
+ *
+ * @return nsIFile object for the file.
+ */
+function check_bookmarks_html() {
+  let profileBookmarksHTMLFile = gProfD.clone();
+  profileBookmarksHTMLFile.append(FILENAME_BOOKMARKS_HTML);
+  Assert.ok(profileBookmarksHTMLFile.exists());
+  return profileBookmarksHTMLFile;
+}
+
+/**
+ * Creates a JSON backup in the profile folder folder from a given source file.
+ *
+ * @param aFilename
+ *        Name of the file to copy to the profile folder.  This file must
+ *        exist in the directory that contains the test files.
+ *
+ * @return nsIFile object for the file.
+ */
+function create_JSON_backup(aFilename) {
+  if (!aFilename) {
+    do_throw("you must pass a filename to create_JSON_backup function");
+  }
+  let bookmarksBackupDir = gProfD.clone();
+  bookmarksBackupDir.append("bookmarkbackups");
+  if (!bookmarksBackupDir.exists()) {
+    bookmarksBackupDir.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+    Assert.ok(bookmarksBackupDir.exists());
+  }
+  let profileBookmarksJSONFile = bookmarksBackupDir.clone();
+  profileBookmarksJSONFile.append(FILENAME_BOOKMARKS_JSON);
+  if (profileBookmarksJSONFile.exists()) {
+    profileBookmarksJSONFile.remove();
+  }
+  let bookmarksJSONFile = gTestDir.clone();
+  bookmarksJSONFile.append(aFilename);
+  Assert.ok(bookmarksJSONFile.exists());
+  bookmarksJSONFile.copyTo(bookmarksBackupDir, FILENAME_BOOKMARKS_JSON);
+  profileBookmarksJSONFile = bookmarksBackupDir.clone();
+  profileBookmarksJSONFile.append(FILENAME_BOOKMARKS_JSON);
+  Assert.ok(profileBookmarksJSONFile.exists());
+  return profileBookmarksJSONFile;
+}
+
+/**
+ * Remove bookmarksbackup dir and all backups from the profile folder.
+ */
+function remove_all_JSON_backups() {
+  let bookmarksBackupDir = gProfD.clone();
+  bookmarksBackupDir.append("bookmarkbackups");
+  if (bookmarksBackupDir.exists()) {
+    bookmarksBackupDir.remove(true);
+    Assert.ok(!bookmarksBackupDir.exists());
+  }
+}
+
+/**
+ * Check a JSON backup file for today exists in the profile folder.
+ *
+ * @param aIsAutomaticBackup The boolean indicates whether it's an automatic
+ *        backup.
+ * @return nsIFile object for the file.
+ */
+function check_JSON_backup(aIsAutomaticBackup) {
+  let profileBookmarksJSONFile;
+  if (aIsAutomaticBackup) {
+    let bookmarksBackupDir = gProfD.clone();
+    bookmarksBackupDir.append("bookmarkbackups");
+    let files = bookmarksBackupDir.directoryEntries;
+    while (files.hasMoreElements()) {
+      let entry = files.nextFile;
+      if (PlacesBackups.filenamesRegex.test(entry.leafName)) {
+        profileBookmarksJSONFile = entry;
+        break;
+      }
+    }
+  } else {
+    profileBookmarksJSONFile = gProfD.clone();
+    profileBookmarksJSONFile.append("bookmarkbackups");
+    profileBookmarksJSONFile.append(FILENAME_BOOKMARKS_JSON);
+  }
+  Assert.ok(profileBookmarksJSONFile.exists());
+  return profileBookmarksJSONFile;
+}
+
+/**
+ * Returns the hidden status of a url.
+ *
+ * @param aURI
+ *        The URI or spec to get hidden for.
+ * @return @return true if the url is hidden, false otherwise.
+ */
+function isUrlHidden(aURI) {
+  let url = aURI instanceof Ci.nsIURI ? aURI.spec : aURI;
+  let stmt = DBConn().createStatement(
+    "SELECT hidden FROM moz_places WHERE url_hash = hash(?1) AND url = ?1"
+  );
+  stmt.bindByIndex(0, url);
+  if (!stmt.executeStep()) {
+    throw new Error("No result for hidden.");
+  }
+  let hidden = stmt.getInt32(0);
+  stmt.finalize();
+
+  return !!hidden;
+}
+
+/**
+ * Compares two times in usecs, considering eventual platform timers skews.
+ *
+ * @param aTimeBefore
+ *        The older time in usecs.
+ * @param aTimeAfter
+ *        The newer time in usecs.
+ * @return true if times are ordered, false otherwise.
+ */
+function is_time_ordered(before, after) {
+  // Windows has an estimated 16ms timers precision, since Date.now() and
+  // PR_Now() use different code atm, the results can be unordered by this
+  // amount of time.  See bug 558745 and bug 557406.
+  let isWindows = "@mozilla.org/windows-registry-key;1" in Cc;
+  // Just to be safe we consider 20ms.
+  let skew = isWindows ? 20000000 : 0;
+  return after - before > -skew;
+}
+
+/**
+ * Shutdowns Places, invoking the callback when the connection has been closed.
+ *
+ * @param aCallback
+ *        Function to be called when done.
+ */
+function waitForConnectionClosed(aCallback) {
+  promiseTopicObserved("places-connection-closed").then(aCallback);
+  shutdownPlaces();
+}
+
+/**
+ * Tests if a given guid is valid for use in Places or not.
+ *
+ * @param aGuid
+ *        The guid to test.
+ * @param [optional] aStack
+ *        The stack frame used to report the error.
+ */
+function do_check_valid_places_guid(aGuid) {
+  Assert.ok(/^[a-zA-Z0-9\-_]{12}$/.test(aGuid), "Should be a valid GUID");
+}
+
+/**
+ * Tests that a guid was set in moz_places for a given uri.
+ *
+ * @param aURI
+ *        The uri to check.
+ * @param [optional] aGUID
+ *        The expected guid in the database.
+ */
+async function check_guid_for_uri(aURI, aGUID) {
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: aURI,
+  });
+  if (aGUID) {
+    do_check_valid_places_guid(aGUID);
+    Assert.equal(guid, aGUID, "Should have a guid in moz_places for the URI");
+  }
+}
+
+/**
+ * Tests that a guid was set in moz_places for a given bookmark.
+ *
+ * @param aId
+ *        The bookmark id to check.
+ * @param [optional] aGUID
+ *        The expected guid in the database.
+ */
+async function check_guid_for_bookmark(aId, aGUID) {
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "guid", {
+    id: aId,
+  });
+  if (aGUID) {
+    do_check_valid_places_guid(aGUID);
+    Assert.equal(guid, aGUID, "Should have the correct GUID for the bookmark");
+  }
+}
+
+/**
+ * Compares 2 arrays returning whether they contains the same elements.
+ *
+ * @param a1
+ *        First array to compare.
+ * @param a2
+ *        Second array to compare.
+ * @param [optional] sorted
+ *        Whether the comparison should take in count position of the elements.
+ * @return true if the arrays contain the same elements, false otherwise.
+ */
+function do_compare_arrays(a1, a2, sorted) {
+  if (a1.length != a2.length) {
+    return false;
+  }
+
+  if (sorted) {
+    return a1.every((e, i) => e == a2[i]);
+  }
+  return (
+    !a1.filter(e => !a2.includes(e)).length &&
+    !a2.filter(e => !a1.includes(e)).length
+  );
+}
+
+/**
+ * Generic nsINavHistoryResultObserver that doesn't implement anything, but
+ * provides dummy methods to prevent errors about an object not having a certain
+ * method.
+ */
+function NavHistoryResultObserver() {}
+
+NavHistoryResultObserver.prototype = {
+  batching() {},
+  containerStateChanged() {},
+  invalidateContainer() {},
+  nodeDateAddedChanged() {},
+  nodeHistoryDetailsChanged() {},
+  nodeIconChanged() {},
+  nodeInserted() {},
+  nodeKeywordChanged() {},
+  nodeLastModifiedChanged() {},
+  nodeMoved() {},
+  nodeRemoved() {},
+  nodeTagsChanged() {},
+  nodeTitleChanged() {},
+  nodeURIChanged() {},
+  sortingChanged() {},
+  QueryInterface: ChromeUtils.generateQI(["nsINavHistoryResultObserver"]),
+};
+
+function checkBookmarkObject(info) {
+  do_check_valid_places_guid(info.guid);
+  do_check_valid_places_guid(info.parentGuid);
+  Assert.ok(typeof info.index == "number", "index should be a number");
+  Assert.ok(
+    info.dateAdded.constructor.name == "Date",
+    "dateAdded should be a Date"
+  );
+  Assert.ok(
+    info.lastModified.constructor.name == "Date",
+    "lastModified should be a Date"
+  );
+  Assert.ok(
+    info.lastModified >= info.dateAdded,
+    "lastModified should never be smaller than dateAdded"
+  );
+  Assert.ok(typeof info.type == "number", "type should be a number");
+}
+
+/**
+ * Reads foreign_count value for a given url.
+ */
+async function foreign_count(url) {
+  if (url instanceof Ci.nsIURI) {
+    url = url.spec;
+  }
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.executeCached(
+    `SELECT foreign_count FROM moz_places
+     WHERE url_hash = hash(:url) AND url = :url
+    `,
+    { url }
+  );
+  return !rows.length ? 0 : rows[0].getResultByName("foreign_count");
+}
+
+function compareAscending(a, b) {
+  if (a > b) {
+    return 1;
+  }
+  if (a < b) {
+    return -1;
+  }
+  return 0;
+}
+
+function sortBy(array, prop) {
+  return array.sort((a, b) => compareAscending(a[prop], b[prop]));
+}
+
+/**
+ * Asynchronously set the favicon associated with a page.
+ * @param page
+ *        The page's URL
+ * @param icon
+ *        The URL of the favicon to be set.
+ * @param [optional] forceReload
+ *        Whether to enforce reloading the icon.
+ */
+function setFaviconForPage(page, icon, forceReload = true) {
+  let pageURI =
+    page instanceof Ci.nsIURI ? page : NetUtil.newURI(new URL(page).href);
+  let iconURI =
+    icon instanceof Ci.nsIURI ? icon : NetUtil.newURI(new URL(icon).href);
+  return new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      pageURI,
+      iconURI,
+      forceReload,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      resolve,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+}
+
+function getFaviconUrlForPage(page, width = 0) {
+  let pageURI =
+    page instanceof Ci.nsIURI ? page : NetUtil.newURI(new URL(page).href);
+  return new Promise((resolve, reject) => {
+    PlacesUtils.favicons.getFaviconURLForPage(
+      pageURI,
+      iconURI => {
+        if (iconURI) {
+          resolve(iconURI.spec);
+        } else {
+          reject("Unable to find an icon for " + pageURI.spec);
+        }
+      },
+      width
+    );
+  });
+}
+
+function getFaviconDataForPage(page, width = 0) {
+  let pageURI =
+    page instanceof Ci.nsIURI ? page : NetUtil.newURI(new URL(page).href);
+  return new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      pageURI,
+      (iconUri, len, data, mimeType) => {
+        resolve({ data, mimeType });
+      },
+      width
+    );
+  });
+}
+
+/**
+ * Asynchronously compares contents from 2 favicon urls.
+ */
+async function compareFavicons(icon1, icon2, msg) {
+  icon1 = new URL(icon1 instanceof Ci.nsIURI ? icon1.spec : icon1);
+  icon2 = new URL(icon2 instanceof Ci.nsIURI ? icon2.spec : icon2);
+
+  function getIconData(icon) {
+    return new Promise((resolve, reject) => {
+      NetUtil.asyncFetch(
+        {
+          uri: icon.href,
+          loadUsingSystemPrincipal: true,
+          contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
+        },
+        function (inputStream, status) {
+          if (!Components.isSuccessCode(status)) {
+            reject();
+          }
+          let size = inputStream.available();
+          resolve(NetUtil.readInputStreamToString(inputStream, size));
+        }
+      );
+    });
+  }
+
+  let data1 = await getIconData(icon1);
+  Assert.ok(!!data1.length, "Should fetch icon data");
+  let data2 = await getIconData(icon2);
+  Assert.ok(!!data2.length, "Should fetch icon data");
+  Assert.deepEqual(data1, data2, msg);
+}
+
+/**
+ * Get the internal "root" folder name for an item, specified by its itemGuid.
+ * If the itemGuid does not point to a root folder, null is returned.
+ *
+ * @param itemGuid
+ *        the item guid.
+ * @return the internal-root name for the root folder, if itemGuid points
+ * to such folder, null otherwise.
+ */
+function mapItemGuidToInternalRootName(itemGuid) {
+  switch (itemGuid) {
+    case PlacesUtils.bookmarks.rootGuid:
+      return "placesRoot";
+    case PlacesUtils.bookmarks.menuGuid:
+      return "bookmarksMenuFolder";
+    case PlacesUtils.bookmarks.toolbarGuid:
+      return "toolbarFolder";
+    case PlacesUtils.bookmarks.unfiledGuid:
+      return "unfiledBookmarksFolder";
+    case PlacesUtils.bookmarks.mobileGuid:
+      return "mobileFolder";
+  }
+  return null;
+}
+
+const DB_FILENAME = "places.sqlite";
+
+/**
+ * Sets the database to use for the given test.  This should be the very first
+ * thing in the test, otherwise this database will not be used!
+ *
+ * @param {string|string[]} path
+ *        A filename or path to a database. The database must exist.
+ *        If this is a string, then this is assumed to be a filename in the
+ *        directory where the test calling this is located.
+ *        If this is an array, this is assumed to be a path relative to the
+ *        directory that this file, head_common.js, is located.
+ * @param {string} destFileName
+ *        The destination filename to copy the database to.
+ * @return {Promise} the final path to the database
+ */
+async function setupPlacesDatabase(path, destFileName = DB_FILENAME) {
+  let currentDir = do_get_cwd().path;
+
+  if (typeof path == "string") {
+    path = [path];
+  } else {
+    currentDir = PathUtils.parent(currentDir);
+  }
+  let src = PathUtils.join(currentDir, ...path);
+  Assert.ok(await IOUtils.exists(src), "Database file found");
+
+  // Ensure that our database doesn't already exist.
+  let dest = PathUtils.join(PathUtils.profileDir, destFileName);
+  Assert.ok(
+    !(await IOUtils.exists(dest)),
+    "Database file should not exist yet"
+  );
+
+  await IOUtils.copy(src, dest);
+  return dest;
+}
+
+/**
+ * Gets the URLs of pages that have a particular annotation.
+ *
+ * @param {String} name The name of the annotation to search for.
+ * @return An array of URLs found.
+ */
+function getPagesWithAnnotation(name) {
+  return PlacesUtils.promiseDBConnection().then(async db => {
+    let rows = await db.execute(
+      `
+      SELECT h.url FROM moz_anno_attributes n
+      JOIN moz_annos a ON n.id = a.anno_attribute_id
+      JOIN moz_places h ON h.id = a.place_id
+      WHERE n.name = :name
+    `,
+      { name }
+    );
+
+    return rows.map(row => row.getResultByName("url"));
+  });
+}
+
+/**
+ * Checks there are no orphan page annotations in the database, and no
+ * orphan anno attribute names.
+ */
+async function assertNoOrphanPageAnnotations() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let rows = await db.execute(`
+    SELECT place_id FROM moz_annos
+    WHERE place_id NOT IN (SELECT id FROM moz_places)
+  `);
+
+  Assert.equal(rows.length, 0, "Should not have any orphan page annotations");
+
+  rows = await db.execute(`
+    SELECT id FROM moz_anno_attributes
+    WHERE id NOT IN (SELECT anno_attribute_id FROM moz_annos) AND
+          id NOT IN (SELECT anno_attribute_id FROM moz_items_annos)`);
+}
diff --git a/toolkit/components/places/tests/history/head_history.js b/toolkit/components/places/tests/history/head_history.js
new file mode 100644
index 0000000000..4adce13cce
--- /dev/null
+++ b/toolkit/components/places/tests/history/head_history.js
@@ -0,0 +1,13 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
diff --git a/toolkit/components/places/tests/history/test_async_history_api.js b/toolkit/components/places/tests/history/test_async_history_api.js
new file mode 100644
index 0000000000..d23e947c7e
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_async_history_api.js
@@ -0,0 +1,1343 @@
+/**
+ * This file tests the async history API exposed by mozIAsyncHistory.
+ */
+
+// Globals
+
+XPCOMUtils.defineLazyServiceGetter(
+  this,
+  "asyncHistory",
+  "@mozilla.org/browser/history;1",
+  "mozIAsyncHistory"
+);
+
+const TEST_DOMAIN = "http://mozilla.org/";
+const URI_VISIT_SAVED = "uri-visit-saved";
+const RECENT_EVENT_THRESHOLD = 15 * 60 * 1000000;
+
+// Helpers
+/**
+ * Object that represents a mozIVisitInfo object.
+ *
+ * @param [optional] aTransitionType
+ *        The transition type of the visit.  Defaults to TRANSITION_LINK if not
+ *        provided.
+ * @param [optional] aVisitTime
+ *        The time of the visit.  Defaults to now if not provided.
+ */
+function VisitInfo(aTransitionType, aVisitTime) {
+  this.transitionType =
+    aTransitionType === undefined ? TRANSITION_LINK : aTransitionType;
+  this.visitDate = aVisitTime || Date.now() * 1000;
+}
+
+function promiseUpdatePlaces(aPlaces, aOptions = {}) {
+  return new Promise((resolve, reject) => {
+    asyncHistory.updatePlaces(
+      aPlaces,
+      Object.assign(
+        {
+          _errors: [],
+          _results: [],
+          handleError(aResultCode, aPlace) {
+            this._errors.push({ resultCode: aResultCode, info: aPlace });
+          },
+          handleResult(aPlace) {
+            this._results.push(aPlace);
+          },
+          handleCompletion(resultCount) {
+            resolve({
+              errors: this._errors,
+              results: this._results,
+              resultCount,
+            });
+          },
+        },
+        aOptions
+      )
+    );
+  });
+}
+
+/**
+ * Listens for a title change notification, and calls aCallback when it gets it.
+ */
+class TitleChangedObserver {
+  /**
+   * Constructor.
+   *
+   * @param aURI
+   *        The URI of the page we expect a notification for.
+   * @param aExpectedTitle
+   *        The expected title of the URI we expect a notification for.
+   * @param aCallback
+   *        The method to call when we have gotten the proper notification about
+   *        the title changing.
+   */
+  constructor(aURI, aExpectedTitle, aCallback) {
+    this.uri = aURI;
+    this.expectedTitle = aExpectedTitle;
+    this.callback = aCallback;
+    this.handlePlacesEvent = this.handlePlacesEvent.bind(this);
+    PlacesObservers.addListener(["page-title-changed"], this.handlePlacesEvent);
+  }
+
+  async handlePlacesEvent(aEvents) {
+    info("'page-title-changed'!!!");
+    Assert.equal(aEvents.length, 1, "Right number of title changed notified");
+    Assert.equal(aEvents[0].type, "page-title-changed");
+    if (this.uri.spec !== aEvents[0].url) {
+      return;
+    }
+    Assert.equal(aEvents[0].title, this.expectedTitle);
+    await check_guid_for_uri(this.uri, aEvents[0].pageGuid);
+    this.callback();
+
+    PlacesObservers.removeListener(
+      ["page-title-changed"],
+      this.handlePlacesEvent
+    );
+  }
+}
+
+/**
+ * Listens for a visit notification, and calls aCallback when it gets it.
+ */
+class VisitObserver {
+  constructor(aURI, aGUID, aCallback) {
+    this.uri = aURI;
+    this.guid = aGUID;
+    this.callback = aCallback;
+    this.handlePlacesEvent = this.handlePlacesEvent.bind(this);
+    PlacesObservers.addListener(["page-visited"], this.handlePlacesEvent);
+  }
+
+  handlePlacesEvent(aEvents) {
+    info("'page-visited'!!!");
+    Assert.equal(aEvents.length, 1, "Right number of visits notified");
+    Assert.equal(aEvents[0].type, "page-visited");
+    let {
+      url,
+      visitId,
+      visitTime,
+      referringVisitId,
+      transitionType,
+      pageGuid,
+      hidden,
+      visitCount,
+      typedCount,
+      lastKnownTitle,
+    } = aEvents[0];
+    let args = [
+      visitId,
+      visitTime,
+      referringVisitId,
+      transitionType,
+      pageGuid,
+      hidden,
+      visitCount,
+      typedCount,
+      lastKnownTitle,
+    ];
+    info("'page-visited' (" + url + args.join(", ") + ")");
+    if (this.uri.spec != url || this.guid != pageGuid) {
+      return;
+    }
+    this.callback(visitTime * 1000, transitionType, lastKnownTitle);
+
+    PlacesObservers.removeListener(["page-visited"], this.handlePlacesEvent);
+  }
+}
+
+/**
+ * Tests that a title was set properly in the database.
+ *
+ * @param aURI
+ *        The uri to check.
+ * @param aTitle
+ *        The expected title in the database.
+ */
+function do_check_title_for_uri(aURI, aTitle) {
+  let stmt = DBConn().createStatement(
+    `SELECT title
+     FROM moz_places
+     WHERE url_hash = hash(:url) AND url = :url`
+  );
+  stmt.params.url = aURI.spec;
+  Assert.ok(stmt.executeStep());
+  Assert.equal(stmt.row.title, aTitle);
+  stmt.finalize();
+}
+
+// Test Functions
+
+add_task(async function test_interface_exists() {
+  let history = Cc["@mozilla.org/browser/history;1"].getService(Ci.nsISupports);
+  Assert.ok(history instanceof Ci.mozIAsyncHistory);
+});
+
+add_task(async function test_invalid_uri_throws() {
+  // First, test passing in nothing.
+  let place = {
+    visits: [new VisitInfo()],
+  };
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+
+  // Now, test other bogus things.
+  const TEST_VALUES = [
+    null,
+    undefined,
+    {},
+    [],
+    TEST_DOMAIN + "test_invalid_id_throws",
+  ];
+  for (let i = 0; i < TEST_VALUES.length; i++) {
+    place.uri = TEST_VALUES[i];
+    try {
+      await promiseUpdatePlaces(place);
+      do_throw("Should have thrown!");
+    } catch (e) {
+      Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+    }
+  }
+});
+
+add_task(async function test_invalid_places_throws() {
+  // First, test passing in nothing.
+  try {
+    asyncHistory.updatePlaces();
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_XPC_NOT_ENOUGH_ARGS);
+  }
+
+  // Now, test other bogus things.
+  const TEST_VALUES = [null, undefined, {}, [], ""];
+  for (let i = 0; i < TEST_VALUES.length; i++) {
+    let value = TEST_VALUES[i];
+    try {
+      await promiseUpdatePlaces(value);
+      do_throw("Should have thrown!");
+    } catch (e) {
+      Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+    }
+  }
+});
+
+add_task(async function test_invalid_guid_throws() {
+  // First check invalid length guid.
+  let place = {
+    guid: "BAD_GUID",
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_invalid_guid_throws"),
+    visits: [new VisitInfo()],
+  };
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+
+  // Now check invalid character guid.
+  place.guid = "__BADGUID+__";
+  Assert.equal(place.guid.length, 12);
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+});
+
+add_task(async function test_no_visits_throws() {
+  const TEST_URI = NetUtil.newURI(
+    TEST_DOMAIN + "test_no_id_or_guid_no_visits_throws"
+  );
+  const TEST_GUID = "_RANDOMGUID_";
+
+  let log_test_conditions = function (aPlace) {
+    let str =
+      "Testing place with " +
+      (aPlace.uri ? "uri" : "no uri") +
+      ", " +
+      (aPlace.guid ? "guid" : "no guid") +
+      ", " +
+      (aPlace.visits ? "visits array" : "no visits array");
+    info(str);
+  };
+
+  // Loop through every possible case.  Note that we don't actually care about
+  // the case where we have no uri, place id, or guid (covered by another test),
+  // but it is easier to just make sure it too throws than to exclude it.
+  let place = {};
+  for (let uri = 1; uri >= 0; uri--) {
+    place.uri = uri ? TEST_URI : undefined;
+
+    for (let guid = 1; guid >= 0; guid--) {
+      place.guid = guid ? TEST_GUID : undefined;
+
+      for (let visits = 1; visits >= 0; visits--) {
+        place.visits = visits ? [] : undefined;
+
+        log_test_conditions(place);
+        try {
+          await promiseUpdatePlaces(place);
+          do_throw("Should have thrown!");
+        } catch (e) {
+          Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+        }
+      }
+    }
+  }
+});
+
+add_task(async function test_add_visit_no_date_throws() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_add_visit_no_date_throws"),
+    visits: [new VisitInfo()],
+  };
+  delete place.visits[0].visitDate;
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+});
+
+add_task(async function test_add_visit_no_transitionType_throws() {
+  let place = {
+    uri: NetUtil.newURI(
+      TEST_DOMAIN + "test_add_visit_no_transitionType_throws"
+    ),
+    visits: [new VisitInfo()],
+  };
+  delete place.visits[0].transitionType;
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+});
+
+add_task(async function test_add_visit_invalid_transitionType_throws() {
+  // First, test something that has a transition type lower than the first one.
+  let place = {
+    uri: NetUtil.newURI(
+      TEST_DOMAIN + "test_add_visit_invalid_transitionType_throws"
+    ),
+    visits: [new VisitInfo(TRANSITION_LINK - 1)],
+  };
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+
+  // Now, test something that has a transition type greater than the last one.
+  place.visits[0] = new VisitInfo(TRANSITION_RELOAD + 1);
+  try {
+    await promiseUpdatePlaces(place);
+    do_throw("Should have thrown!");
+  } catch (e) {
+    Assert.equal(e.result, Cr.NS_ERROR_INVALID_ARG);
+  }
+});
+
+add_task(async function test_non_addable_uri_errors() {
+  // Array of protocols that nsINavHistoryService::canAddURI returns false for.
+  const URLS = [
+    "about:config",
+    "imap://cyrus.andrew.cmu.edu/archive.imap",
+    "news://new.mozilla.org/mozilla.dev.apps.firefox",
+    "mailbox:Inbox",
+    "moz-anno:favicon:http://mozilla.org/made-up-favicon",
+    "view-source:http://mozilla.org",
+    "chrome://browser/content/browser.xhtml",
+    "resource://gre-resources/hiddenWindow.html",
+    "data:,Hello%2C%20World!",
+    "javascript:alert('hello wolrd!');",
+    "blob:foo",
+    "moz-extension://f49fb5b3-a1e7-cd41-85e1-d61a3950f5e4/index.html",
+  ];
+  let places = [];
+  URLS.forEach(function (url) {
+    try {
+      let place = {
+        uri: NetUtil.newURI(url),
+        title: "test for " + url,
+        visits: [new VisitInfo()],
+      };
+      places.push(place);
+    } catch (e) {
+      if (e.result != Cr.NS_ERROR_FAILURE) {
+        throw e;
+      }
+      // NetUtil.newURI() can throw if e.g. our app knows about imap://
+      // but the account is not set up and so the URL is invalid for us.
+      // Note this in the log but ignore as it's not the subject of this test.
+      info("Could not construct URI for '" + url + "'; ignoring");
+    }
+  });
+
+  let placesResult = await promiseUpdatePlaces(places);
+  if (placesResult.results.length) {
+    do_throw("Unexpected success.");
+  }
+  for (let place of placesResult.errors) {
+    info("Checking '" + place.info.uri.spec + "'");
+    Assert.equal(place.resultCode, Cr.NS_ERROR_INVALID_ARG);
+    Assert.equal(false, await PlacesUtils.history.hasVisits(place.info.uri));
+  }
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_duplicate_guid_errors() {
+  // This test ensures that trying to add a visit, with a guid already found in
+  // another visit, fails.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_first"),
+    visits: [new VisitInfo()],
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(placeInfo.uri));
+
+  let badPlace = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_second"),
+    visits: [new VisitInfo()],
+    guid: placeInfo.guid,
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(badPlace.uri));
+  placesResult = await promiseUpdatePlaces(badPlace);
+  if (placesResult.results.length) {
+    do_throw("Unexpected success.");
+  }
+  let badPlaceInfo = placesResult.errors[0];
+  Assert.equal(badPlaceInfo.resultCode, Cr.NS_ERROR_STORAGE_CONSTRAINT);
+  Assert.equal(
+    false,
+    await PlacesUtils.history.hasVisits(badPlaceInfo.info.uri)
+  );
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_invalid_referrerURI_ignored() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_invalid_referrerURI_ignored"),
+    visits: [new VisitInfo()],
+  };
+  place.visits[0].referrerURI = NetUtil.newURI(
+    place.uri.spec + "_unvisistedURI"
+  );
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  Assert.equal(
+    false,
+    await PlacesUtils.history.hasVisits(place.visits[0].referrerURI)
+  );
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(placeInfo.uri));
+
+  // Check to make sure we do not visit the invalid referrer.
+  Assert.equal(
+    false,
+    await PlacesUtils.history.hasVisits(place.visits[0].referrerURI)
+  );
+
+  // Check to make sure from_visit is zero in database.
+  let stmt = DBConn().createStatement(
+    `SELECT from_visit
+     FROM moz_historyvisits
+     WHERE id = :visit_id`
+  );
+  stmt.params.visit_id = placeInfo.visits[0].visitId;
+  Assert.ok(stmt.executeStep());
+  Assert.equal(stmt.row.from_visit, 0);
+  stmt.finalize();
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_nonnsIURI_referrerURI_ignored() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_nonnsIURI_referrerURI_ignored"),
+    visits: [new VisitInfo()],
+  };
+  place.visits[0].referrerURI = place.uri.spec + "_nonnsIURI";
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(placeInfo.uri));
+
+  // Check to make sure from_visit is zero in database.
+  let stmt = DBConn().createStatement(
+    `SELECT from_visit
+     FROM moz_historyvisits
+     WHERE id = :visit_id`
+  );
+  stmt.params.visit_id = placeInfo.visits[0].visitId;
+  Assert.ok(stmt.executeStep());
+  Assert.equal(stmt.row.from_visit, 0);
+  stmt.finalize();
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_old_referrer_ignored() {
+  // This tests that a referrer for a visit which is not recent (specifically,
+  // older than 15 minutes as per RECENT_EVENT_THRESHOLD) is not saved by
+  // updatePlaces.
+  let oldTime = Date.now() * 1000 - (RECENT_EVENT_THRESHOLD + 1);
+  let referrerPlace = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_old_referrer_ignored_referrer"),
+    visits: [new VisitInfo(TRANSITION_LINK, oldTime)],
+  };
+
+  // First we must add our referrer to the history so that it is not ignored
+  // as being invalid.
+  Assert.equal(false, await PlacesUtils.history.hasVisits(referrerPlace.uri));
+  let placesResult = await promiseUpdatePlaces(referrerPlace);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+
+  // Now that the referrer is added, we can add a page with a valid
+  // referrer to determine if the recency of the referrer is taken into
+  // account.
+  Assert.ok(await PlacesUtils.history.hasVisits(referrerPlace.uri));
+
+  let visitInfo = new VisitInfo();
+  visitInfo.referrerURI = referrerPlace.uri;
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_old_referrer_ignored_page"),
+    visits: [visitInfo],
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(place.uri));
+
+  // Though the visit will not contain the referrer, we must examine the
+  // database to be sure.
+  Assert.equal(placeInfo.visits[0].referrerURI, null);
+  let stmt = DBConn().createStatement(
+    `SELECT COUNT(1) AS count
+     FROM moz_historyvisits
+     JOIN moz_places h ON h.id = place_id
+     WHERE url_hash = hash(:page_url) AND url = :page_url
+     AND from_visit = 0`
+  );
+  stmt.params.page_url = place.uri.spec;
+  Assert.ok(stmt.executeStep());
+  Assert.equal(stmt.row.count, 1);
+  stmt.finalize();
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_place_id_ignored() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_place_id_ignored_first"),
+    visits: [new VisitInfo()],
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(place.uri));
+
+  let placeId = placeInfo.placeId;
+  Assert.notEqual(placeId, 0);
+
+  let badPlace = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_place_id_ignored_second"),
+    visits: [new VisitInfo()],
+    placeId,
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(badPlace.uri));
+  placesResult = await promiseUpdatePlaces(badPlace);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  placeInfo = placesResult.results[0];
+
+  Assert.notEqual(placeInfo.placeId, placeId);
+  Assert.ok(await PlacesUtils.history.hasVisits(badPlace.uri));
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_handleCompletion_called_when_complete() {
+  // We test a normal visit, and embeded visit, and a uri that would fail
+  // the canAddURI test to make sure that the notification happens after *all*
+  // of them have had a callback.
+  let places = [
+    {
+      uri: NetUtil.newURI(
+        TEST_DOMAIN + "test_handleCompletion_called_when_complete"
+      ),
+      visits: [new VisitInfo(), new VisitInfo(TRANSITION_EMBED)],
+    },
+    {
+      uri: NetUtil.newURI("data:,Hello%2C%20World!"),
+      visits: [new VisitInfo()],
+    },
+  ];
+  Assert.equal(false, await PlacesUtils.history.hasVisits(places[0].uri));
+  Assert.equal(false, await PlacesUtils.history.hasVisits(places[1].uri));
+
+  const EXPECTED_COUNT_SUCCESS = 2;
+  const EXPECTED_COUNT_FAILURE = 1;
+
+  let { results, errors } = await promiseUpdatePlaces(places);
+
+  Assert.equal(results.length, EXPECTED_COUNT_SUCCESS);
+  Assert.equal(errors.length, EXPECTED_COUNT_FAILURE);
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_add_visit() {
+  const VISIT_TIME = Date.now() * 1000;
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_add_visit"),
+    title: "test_add_visit title",
+    visits: [],
+  };
+  for (let t in PlacesUtils.history.TRANSITIONS) {
+    if (t == "EMBED") {
+      continue;
+    }
+    let transitionType = PlacesUtils.history.TRANSITIONS[t];
+    place.visits.push(new VisitInfo(transitionType, VISIT_TIME));
+  }
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let callbackCount = 0;
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  for (let placeInfo of placesResult.results) {
+    Assert.ok(await PlacesUtils.history.hasVisits(place.uri));
+
+    // Check mozIPlaceInfo properties.
+    Assert.ok(place.uri.equals(placeInfo.uri));
+    Assert.equal(placeInfo.frecency, -1); // We don't pass frecency here!
+    Assert.equal(placeInfo.title, place.title);
+
+    // Check mozIVisitInfo properties.
+    let visits = placeInfo.visits;
+    Assert.equal(visits.length, 1);
+    let visit = visits[0];
+    Assert.equal(visit.visitDate, VISIT_TIME);
+    Assert.ok(
+      Object.values(PlacesUtils.history.TRANSITIONS).includes(
+        visit.transitionType
+      )
+    );
+    Assert.ok(visit.referrerURI === null);
+
+    // For TRANSITION_EMBED visits, many properties will always be zero or
+    // undefined.
+    if (visit.transitionType == TRANSITION_EMBED) {
+      // Check mozIPlaceInfo properties.
+      Assert.equal(placeInfo.placeId, 0, "//");
+      Assert.equal(placeInfo.guid, null);
+
+      // Check mozIVisitInfo properties.
+      Assert.equal(visit.visitId, 0);
+    } else {
+      // But they should be valid for non-embed visits.
+      // Check mozIPlaceInfo properties.
+      Assert.ok(placeInfo.placeId > 0);
+      do_check_valid_places_guid(placeInfo.guid);
+
+      // Check mozIVisitInfo properties.
+      Assert.ok(visit.visitId > 0);
+    }
+
+    // If we have had all of our callbacks, continue running tests.
+    if (++callbackCount == place.visits.length) {
+      await PlacesTestUtils.promiseAsyncUpdates();
+    }
+  }
+});
+
+add_task(async function test_properties_saved() {
+  // Check each transition type to make sure it is saved properly.
+  let places = [];
+  for (let t in PlacesUtils.history.TRANSITIONS) {
+    if (t == "EMBED") {
+      continue;
+    }
+    let transitionType = PlacesUtils.history.TRANSITIONS[t];
+    let place = {
+      uri: NetUtil.newURI(
+        TEST_DOMAIN + "test_properties_saved/" + transitionType
+      ),
+      title: "test_properties_saved test",
+      visits: [new VisitInfo(transitionType)],
+    };
+    Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+    places.push(place);
+  }
+
+  let callbackCount = 0;
+  let placesResult = await promiseUpdatePlaces(places);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  for (let placeInfo of placesResult.results) {
+    let uri = placeInfo.uri;
+    Assert.ok(await PlacesUtils.history.hasVisits(uri));
+    let visit = placeInfo.visits[0];
+    print(
+      "TEST-INFO | test_properties_saved | updatePlaces callback for " +
+        "transition type " +
+        visit.transitionType
+    );
+
+    // Note that TRANSITION_EMBED should not be in the database.
+    const EXPECTED_COUNT = visit.transitionType == TRANSITION_EMBED ? 0 : 1;
+
+    // mozIVisitInfo::date
+    let stmt = DBConn().createStatement(
+      `SELECT COUNT(1) AS count
+       FROM moz_places h
+       JOIN moz_historyvisits v
+       ON h.id = v.place_id
+       WHERE h.url_hash = hash(:page_url) AND h.url = :page_url
+       AND v.visit_date = :visit_date`
+    );
+    stmt.params.page_url = uri.spec;
+    stmt.params.visit_date = visit.visitDate;
+    Assert.ok(stmt.executeStep());
+    Assert.equal(stmt.row.count, EXPECTED_COUNT);
+    stmt.finalize();
+
+    // mozIVisitInfo::transitionType
+    stmt = DBConn().createStatement(
+      `SELECT COUNT(1) AS count
+       FROM moz_places h
+       JOIN moz_historyvisits v
+       ON h.id = v.place_id
+       WHERE h.url_hash = hash(:page_url) AND h.url = :page_url
+       AND v.visit_type = :transition_type`
+    );
+    stmt.params.page_url = uri.spec;
+    stmt.params.transition_type = visit.transitionType;
+    Assert.ok(stmt.executeStep());
+    Assert.equal(stmt.row.count, EXPECTED_COUNT);
+    stmt.finalize();
+
+    // mozIPlaceInfo::title
+    stmt = DBConn().createStatement(
+      `SELECT COUNT(1) AS count
+       FROM moz_places h
+       WHERE h.url_hash = hash(:page_url) AND h.url = :page_url
+       AND h.title = :title`
+    );
+    stmt.params.page_url = uri.spec;
+    stmt.params.title = placeInfo.title;
+    Assert.ok(stmt.executeStep());
+    Assert.equal(stmt.row.count, EXPECTED_COUNT);
+    stmt.finalize();
+
+    // If we have had all of our callbacks, continue running tests.
+    if (++callbackCount == places.length) {
+      await PlacesTestUtils.promiseAsyncUpdates();
+    }
+  }
+});
+
+add_task(async function test_guid_saved() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_guid_saved"),
+    guid: "__TESTGUID__",
+    visits: [new VisitInfo()],
+  };
+  do_check_valid_places_guid(place.guid);
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  let uri = placeInfo.uri;
+  Assert.ok(await PlacesUtils.history.hasVisits(uri));
+  Assert.equal(placeInfo.guid, place.guid);
+  await check_guid_for_uri(uri, place.guid);
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_referrer_saved() {
+  let places = [
+    {
+      uri: NetUtil.newURI(TEST_DOMAIN + "test_referrer_saved/referrer"),
+      visits: [new VisitInfo()],
+    },
+    {
+      uri: NetUtil.newURI(TEST_DOMAIN + "test_referrer_saved/test"),
+      visits: [new VisitInfo()],
+    },
+  ];
+  places[1].visits[0].referrerURI = places[0].uri;
+  Assert.equal(false, await PlacesUtils.history.hasVisits(places[0].uri));
+  Assert.equal(false, await PlacesUtils.history.hasVisits(places[1].uri));
+
+  let resultCount = 0;
+  let placesResult = await promiseUpdatePlaces(places);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  for (let placeInfo of placesResult.results) {
+    let uri = placeInfo.uri;
+    Assert.ok(await PlacesUtils.history.hasVisits(uri));
+    let visit = placeInfo.visits[0];
+
+    // We need to insert all of our visits before we can test conditions.
+    if (++resultCount == places.length) {
+      Assert.ok(places[0].uri.equals(visit.referrerURI));
+
+      let stmt = DBConn().createStatement(
+        `SELECT COUNT(1) AS count
+         FROM moz_historyvisits
+         JOIN moz_places h ON h.id = place_id
+         WHERE url_hash = hash(:page_url) AND url = :page_url
+         AND from_visit = (
+           SELECT v.id
+           FROM moz_historyvisits v
+           JOIN moz_places h ON h.id = place_id
+           WHERE url_hash = hash(:referrer) AND url = :referrer
+         )`
+      );
+      stmt.params.page_url = uri.spec;
+      stmt.params.referrer = visit.referrerURI.spec;
+      Assert.ok(stmt.executeStep());
+      Assert.equal(stmt.row.count, 1);
+      stmt.finalize();
+
+      await PlacesTestUtils.promiseAsyncUpdates();
+    }
+  }
+});
+
+add_task(async function test_guid_change_saved() {
+  // First, add a visit for it.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_guid_change_saved"),
+    visits: [new VisitInfo()],
+  };
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  // Then, change the guid with visits.
+  place.guid = "_GUIDCHANGE_";
+  place.visits = [new VisitInfo()];
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  await check_guid_for_uri(place.uri, place.guid);
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_title_change_saved() {
+  // First, add a visit for it.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_title_change_saved"),
+    title: "original title",
+    visits: [new VisitInfo()],
+  };
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+
+  // Now, make sure the empty string clears the title.
+  place.title = "";
+  place.visits = [new VisitInfo()];
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  do_check_title_for_uri(place.uri, null);
+
+  // Then, change the title with visits.
+  place.title = "title change";
+  place.visits = [new VisitInfo()];
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  do_check_title_for_uri(place.uri, place.title);
+
+  // Lastly, check that the title is cleared if we set it to null.
+  place.title = null;
+  place.visits = [new VisitInfo()];
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  do_check_title_for_uri(place.uri, place.title);
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_no_title_does_not_clear_title() {
+  const TITLE = "test title";
+  // First, add a visit for it.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_no_title_does_not_clear_title"),
+    title: TITLE,
+    visits: [new VisitInfo()],
+  };
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  // Now, make sure that not specifying a title does not clear it.
+  delete place.title;
+  place.visits = [new VisitInfo()];
+  placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  do_check_title_for_uri(place.uri, TITLE);
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_title_change_notifies() {
+  // There are three cases to test.  The first case is to make sure we do not
+  // get notified if we do not specify a title.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_title_change_notifies"),
+    visits: [new VisitInfo()],
+  };
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  new TitleChangedObserver(place.uri, "DO NOT WANT", function () {
+    do_throw("unexpected callback!");
+  });
+
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+
+  // The second case to test is that we don't get the notification when we add
+  // it for the first time.  The first case will fail before our callback if it
+  // is busted, so we can do this now.
+  place.uri = NetUtil.newURI(place.uri.spec + "/new-visit-with-title");
+  place.title = "title 1";
+  let expectedNotification = false;
+  let titleChangeObserver;
+  let titleChangePromise = new Promise((resolve, reject) => {
+    titleChangeObserver = new TitleChangedObserver(
+      place.uri,
+      place.title,
+      function () {
+        Assert.ok(
+          expectedNotification,
+          "Should not get notified for " +
+            place.uri.spec +
+            " with title " +
+            place.title
+        );
+        if (expectedNotification) {
+          resolve();
+        }
+      }
+    );
+  });
+
+  let visitPromise = new Promise(resolve => {
+    function onVisits(events) {
+      Assert.equal(events.length, 1, "Should only get notified for one visit.");
+      Assert.equal(events[0].type, "page-visited");
+      let { url } = events[0];
+      Assert.equal(
+        url,
+        place.uri.spec,
+        "Should get notified for visiting the new URI."
+      );
+      PlacesObservers.removeListener(["page-visited"], onVisits);
+      resolve();
+    }
+    PlacesObservers.addListener(["page-visited"], onVisits);
+  });
+  asyncHistory.updatePlaces(place);
+  await visitPromise;
+
+  // The third case to test is to make sure we get a notification when
+  // we change an existing place.
+  expectedNotification = true;
+  titleChangeObserver.expectedTitle = place.title = "title 2";
+  place.visits = [new VisitInfo()];
+  asyncHistory.updatePlaces(place);
+
+  await titleChangePromise;
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_visit_notifies() {
+  // There are two observers we need to see for each visit. One is an
+  // PlacesObservers and the other is the uri-visit-saved observer topic.
+  let place = {
+    guid: "abcdefghijkl",
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_visit_notifies"),
+    visits: [new VisitInfo()],
+  };
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+
+  function promiseVisitObserver(aPlace) {
+    return new Promise((resolve, reject) => {
+      let callbackCount = 0;
+      let finisher = function () {
+        if (++callbackCount == 2) {
+          resolve();
+        }
+      };
+      new VisitObserver(place.uri, place.guid, function (
+        aVisitDate,
+        aTransitionType
+      ) {
+        let visit = place.visits[0];
+        Assert.equal(visit.visitDate, aVisitDate);
+        Assert.equal(visit.transitionType, aTransitionType);
+
+        finisher();
+      });
+      let observer = function (aSubject, aTopic, aData) {
+        info("observe(" + aSubject + ", " + aTopic + ", " + aData + ")");
+        Assert.ok(aSubject instanceof Ci.nsIURI);
+        Assert.ok(aSubject.equals(place.uri));
+
+        Services.obs.removeObserver(observer, URI_VISIT_SAVED);
+        finisher();
+      };
+      Services.obs.addObserver(observer, URI_VISIT_SAVED);
+      asyncHistory.updatePlaces(place);
+    });
+  }
+
+  await promiseVisitObserver(place);
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+// test with empty mozIVisitInfoCallback object
+add_task(async function test_callbacks_not_supplied() {
+  const URLS = [
+    "imap://cyrus.andrew.cmu.edu/archive.imap", // bad URI
+    "http://mozilla.org/", // valid URI
+  ];
+  let places = [];
+  URLS.forEach(function (url) {
+    try {
+      let place = {
+        uri: NetUtil.newURI(url),
+        title: "test for " + url,
+        visits: [new VisitInfo()],
+      };
+      places.push(place);
+    } catch (e) {
+      if (e.result != Cr.NS_ERROR_FAILURE) {
+        throw e;
+      }
+      // NetUtil.newURI() can throw if e.g. our app knows about imap://
+      // but the account is not set up and so the URL is invalid for us.
+      // Note this in the log but ignore as it's not the subject of this test.
+      info("Could not construct URI for '" + url + "'; ignoring");
+    }
+  });
+
+  asyncHistory.updatePlaces(places, {});
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+// Test that we don't wrongly overwrite typed and hidden when adding new visits.
+add_task(async function test_typed_hidden_not_overwritten() {
+  await PlacesUtils.history.clear();
+  let places = [
+    {
+      uri: NetUtil.newURI("http://mozilla.org/"),
+      title: "test",
+      visits: [new VisitInfo(TRANSITION_TYPED), new VisitInfo(TRANSITION_LINK)],
+    },
+    {
+      uri: NetUtil.newURI("http://mozilla.org/"),
+      title: "test",
+      visits: [new VisitInfo(TRANSITION_FRAMED_LINK)],
+    },
+  ];
+  await promiseUpdatePlaces(places);
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(
+    "SELECT hidden, typed FROM moz_places WHERE url_hash = hash(:url) AND url = :url",
+    { url: "http://mozilla.org/" }
+  );
+  Assert.equal(
+    rows[0].getResultByName("typed"),
+    1,
+    "The page should be marked as typed"
+  );
+  Assert.equal(
+    rows[0].getResultByName("hidden"),
+    0,
+    "The page should be marked as not hidden"
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_omit_frecency_notifications() {
+  await PlacesUtils.history.clear();
+  let places = [
+    {
+      uri: NetUtil.newURI("http://mozilla.org/"),
+      title: "test",
+      visits: [new VisitInfo(TRANSITION_TYPED)],
+    },
+    {
+      uri: NetUtil.newURI("http://example.org/"),
+      title: "test",
+      visits: [new VisitInfo(TRANSITION_TYPED)],
+    },
+  ];
+
+  const promiseRankingChanged =
+    PlacesTestUtils.waitForNotification("pages-rank-changed");
+
+  await promiseUpdatePlaces(places);
+  await promiseRankingChanged;
+});
+
+add_task(async function test_ignore_errors() {
+  await PlacesUtils.history.clear();
+  // This test ensures that trying to add a visit, with a guid already found in
+  // another visit, fails - but doesn't report if we told it not to.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_first"),
+    visits: [new VisitInfo()],
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(placeInfo.uri));
+
+  let badPlace = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_second"),
+    visits: [new VisitInfo()],
+    guid: placeInfo.guid,
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(badPlace.uri));
+  placesResult = await promiseUpdatePlaces(badPlace, { ignoreErrors: true });
+  if (placesResult.results.length) {
+    do_throw("Unexpected success.");
+  }
+  Assert.equal(
+    placesResult.errors.length,
+    0,
+    "Should have seen 0 errors because we disabled reporting."
+  );
+  Assert.equal(
+    placesResult.results.length,
+    0,
+    "Should have seen 0 results because there were none."
+  );
+  Assert.equal(
+    placesResult.resultCount,
+    0,
+    "Should know that we updated 0 items from the completion callback."
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_ignore_results() {
+  await PlacesUtils.history.clear();
+  let place = {
+    uri: NetUtil.newURI("http://mozilla.org/"),
+    title: "test",
+    visits: [new VisitInfo()],
+  };
+  let placesResult = await promiseUpdatePlaces(place, { ignoreResults: true });
+  Assert.equal(
+    placesResult.results.length,
+    0,
+    "Should have seen 0 results because we disabled reporting."
+  );
+  Assert.equal(
+    placesResult.errors.length,
+    0,
+    "Should have seen 0 errors because there were none."
+  );
+  Assert.equal(
+    placesResult.resultCount,
+    1,
+    "Should know that we updated 1 item from the completion callback."
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_ignore_results_and_errors() {
+  await PlacesUtils.history.clear();
+  // This test ensures that trying to add a visit, with a guid already found in
+  // another visit, fails - but doesn't report if we told it not to.
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_first"),
+    visits: [new VisitInfo()],
+  };
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(place.uri));
+  let placesResult = await promiseUpdatePlaces(place);
+  if (placesResult.errors.length) {
+    do_throw("Unexpected error.");
+  }
+  let placeInfo = placesResult.results[0];
+  Assert.ok(await PlacesUtils.history.hasVisits(placeInfo.uri));
+
+  let badPlace = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_duplicate_guid_fails_second"),
+    visits: [new VisitInfo()],
+    guid: placeInfo.guid,
+  };
+  let allPlaces = [
+    {
+      uri: NetUtil.newURI(TEST_DOMAIN + "test_other_successful_item"),
+      visits: [new VisitInfo()],
+    },
+    badPlace,
+  ];
+
+  Assert.equal(false, await PlacesUtils.history.hasVisits(badPlace.uri));
+  placesResult = await promiseUpdatePlaces(allPlaces, {
+    ignoreErrors: true,
+    ignoreResults: true,
+  });
+  Assert.equal(
+    placesResult.errors.length,
+    0,
+    "Should have seen 0 errors because we disabled reporting."
+  );
+  Assert.equal(
+    placesResult.results.length,
+    0,
+    "Should have seen 0 results because we disabled reporting."
+  );
+  Assert.equal(
+    placesResult.resultCount,
+    1,
+    "Should know that we updated 1 item from the completion callback."
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_title_on_initial_visit() {
+  let place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_visit_title"),
+    title: "My title",
+    visits: [new VisitInfo()],
+    guid: "mnopqrstuvwx",
+  };
+  let visitPromise = new Promise(resolve => {
+    new VisitObserver(place.uri, place.guid, function (
+      aVisitDate,
+      aTransitionType,
+      aLastKnownTitle
+    ) {
+      Assert.equal(place.title, aLastKnownTitle);
+
+      resolve();
+    });
+  });
+  await promiseUpdatePlaces(place);
+  await visitPromise;
+
+  // Now check an empty title doesn't get reported as null
+  place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_visit_title"),
+    title: "",
+    visits: [new VisitInfo()],
+    guid: "fghijklmnopq",
+  };
+  visitPromise = new Promise(resolve => {
+    new VisitObserver(place.uri, place.guid, function (
+      aVisitDate,
+      aTransitionType,
+      aLastKnownTitle
+    ) {
+      Assert.equal(place.title, aLastKnownTitle);
+
+      resolve();
+    });
+  });
+  await promiseUpdatePlaces(place);
+  await visitPromise;
+
+  // and that a missing title correctly gets reported as null.
+  place = {
+    uri: NetUtil.newURI(TEST_DOMAIN + "test_visit_title"),
+    visits: [new VisitInfo()],
+    guid: "fghijklmnopq",
+  };
+  visitPromise = new Promise(resolve => {
+    new VisitObserver(place.uri, place.guid, function (
+      aVisitDate,
+      aTransitionType,
+      aLastKnownTitle
+    ) {
+      Assert.equal(null, aLastKnownTitle);
+
+      resolve();
+    });
+  });
+  await promiseUpdatePlaces(place);
+  await visitPromise;
+});
diff --git a/toolkit/components/places/tests/history/test_bookmark_unhide.js b/toolkit/components/places/tests/history/test_bookmark_unhide.js
new file mode 100644
index 0000000000..1295c6e8c5
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_bookmark_unhide.js
@@ -0,0 +1,26 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that bookmarking an hidden page unhides it.
+
+"use strict";
+
+add_task(async function test_hidden() {
+  const url = "http://moz.com/";
+  await PlacesTestUtils.addVisits({
+    url,
+    transition: TRANSITION_FRAMED_LINK,
+  });
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { url }),
+    1
+  );
+  await PlacesUtils.bookmarks.insert({
+    url,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "hidden", { url }),
+    0
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_fetch.js b/toolkit/components/places/tests/history/test_fetch.js
new file mode 100644
index 0000000000..899e459403
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_fetch.js
@@ -0,0 +1,270 @@
+/* Any copyright is dedicated to the Public Domain. http://creativecommons.org/publicdomain/zero/1.0/ */
+"use strict";
+
+add_task(async function test_fetch_existent() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  // Populate places and historyvisits.
+  let uriString = `http://mozilla.com/test_browserhistory/test_fetch`;
+  let uri = NetUtil.newURI(uriString);
+  let title = `Test Visit ${Math.random()}`;
+  let dates = [];
+  let visits = [];
+  let transitions = [
+    PlacesUtils.history.TRANSITION_LINK,
+    PlacesUtils.history.TRANSITION_TYPED,
+    PlacesUtils.history.TRANSITION_BOOKMARK,
+    PlacesUtils.history.TRANSITION_REDIRECT_TEMPORARY,
+    PlacesUtils.history.TRANSITION_REDIRECT_PERMANENT,
+    PlacesUtils.history.TRANSITION_DOWNLOAD,
+    PlacesUtils.history.TRANSITION_FRAMED_LINK,
+    PlacesUtils.history.TRANSITION_RELOAD,
+  ];
+  let guid = "";
+  for (let i = 0; i != transitions.length; i++) {
+    dates.push(new Date(Date.now() - i * 10000000));
+    visits.push({
+      uri,
+      title,
+      transition: transitions[i],
+      visitDate: dates[i],
+    });
+  }
+  await PlacesTestUtils.addVisits(visits);
+  Assert.ok(await PlacesTestUtils.isPageInDB(uri));
+  Assert.equal(await PlacesTestUtils.visitsInDB(uri), visits.length);
+
+  // Store guid for further use in testing.
+  guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: uri,
+  });
+  Assert.ok(guid, guid);
+
+  // Initialize the objects to compare against.
+  let idealPageInfo = {
+    url: new URL(uriString),
+    guid,
+    title,
+  };
+  let idealVisits = visits.map(v => {
+    return {
+      date: v.visitDate,
+      transition: v.transition,
+    };
+  });
+
+  // We should check these 4 cases:
+  // 1, 2: visits not included, by URL and guid (same result expected).
+  // 3, 4: visits included, by URL and guid (same result expected).
+  for (let includeVisits of [true, false]) {
+    for (let guidOrURL of [uri, guid]) {
+      let pageInfo = await PlacesUtils.history.fetch(guidOrURL, {
+        includeVisits,
+      });
+      if (includeVisits) {
+        idealPageInfo.visits = idealVisits;
+      } else {
+        // We need to explicitly delete this property since deepEqual looks at
+        // the list of properties as well (`visits in pageInfo` is true here).
+        delete idealPageInfo.visits;
+      }
+
+      // Since idealPageInfo doesn't contain a frecency, check it and delete.
+      Assert.ok(typeof pageInfo.frecency === "number");
+      delete pageInfo.frecency;
+
+      // Visits should be from newer to older.
+      if (includeVisits) {
+        for (let i = 0; i !== pageInfo.visits.length - 1; i++) {
+          Assert.lessOrEqual(
+            pageInfo.visits[i + 1].date.getTime(),
+            pageInfo.visits[i].date.getTime()
+          );
+        }
+      }
+      Assert.deepEqual(idealPageInfo, pageInfo);
+    }
+  }
+});
+
+add_task(async function test_fetch_page_meta_info() {
+  await PlacesUtils.history.clear();
+
+  let TEST_URI = NetUtil.newURI("http://mozilla.com/test_fetch_page_meta_info");
+  await PlacesTestUtils.addVisits(TEST_URI);
+  Assert.ok(page_in_database(TEST_URI));
+
+  // Test fetching the null values
+  let includeMeta = true;
+  let pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeMeta });
+  Assert.strictEqual(
+    null,
+    pageInfo.previewImageURL,
+    "fetch should return a null previewImageURL"
+  );
+  Assert.strictEqual(
+    "",
+    pageInfo.siteName,
+    "fetch should return a null siteName"
+  );
+  Assert.equal(
+    "",
+    pageInfo.description,
+    "fetch should return a empty string description"
+  );
+
+  // Now set the pageMetaInfo for this page
+  let description = "Test description";
+  let siteName = "Mozilla";
+  let previewImageURL = "http://mozilla.com/test_preview_image.png";
+  await PlacesUtils.history.update({
+    url: TEST_URI,
+    description,
+    previewImageURL,
+    siteName,
+  });
+
+  includeMeta = true;
+  pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeMeta });
+  Assert.equal(
+    previewImageURL,
+    pageInfo.previewImageURL.href,
+    "fetch should return a previewImageURL"
+  );
+  Assert.equal(siteName, pageInfo.siteName, "fetch should return a siteName");
+  Assert.equal(
+    description,
+    pageInfo.description,
+    "fetch should return a description"
+  );
+
+  includeMeta = false;
+  pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeMeta });
+  Assert.ok(
+    !("description" in pageInfo),
+    "fetch should not return a description if includeMeta is false"
+  );
+  Assert.ok(
+    !("siteName" in pageInfo),
+    "fetch should not return a siteName if includeMeta is false"
+  );
+  Assert.ok(
+    !("previewImageURL" in pageInfo),
+    "fetch should not return a previewImageURL if includeMeta is false"
+  );
+});
+
+add_task(async function test_fetch_annotations() {
+  await PlacesUtils.history.clear();
+
+  const TEST_URI = "http://mozilla.com/test_fetch_page_meta_info";
+  await PlacesTestUtils.addVisits(TEST_URI);
+  Assert.ok(page_in_database(TEST_URI));
+
+  let includeAnnotations = true;
+  let pageInfo = await PlacesUtils.history.fetch(TEST_URI, {
+    includeAnnotations,
+  });
+  Assert.equal(
+    pageInfo.annotations.size,
+    0,
+    "fetch should return an empty annotation map"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URI,
+    annotations: new Map([["test/annotation", "testContent"]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeAnnotations });
+  Assert.equal(
+    pageInfo.annotations.size,
+    1,
+    "fetch should have only one annotation"
+  );
+
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    "testContent",
+    "fetch should return the expected annotation"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URI,
+    annotations: new Map([["test/annotation2", 123]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeAnnotations });
+  Assert.equal(
+    pageInfo.annotations.size,
+    2,
+    "fetch should have returned two annotations"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    "testContent",
+    "fetch should still have the first annotation"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    123,
+    "fetch should have the second annotation"
+  );
+
+  includeAnnotations = false;
+  pageInfo = await PlacesUtils.history.fetch(TEST_URI, { includeAnnotations });
+  Assert.ok(
+    !("annotations" in pageInfo),
+    "fetch should not return annotations if includeAnnotations is false"
+  );
+});
+
+add_task(async function test_fetch_nonexistent() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  let uri = NetUtil.newURI("http://doesntexist.in.db");
+  let pageInfo = await PlacesUtils.history.fetch(uri);
+  Assert.equal(pageInfo, null);
+});
+
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.fetch("3"),
+    /TypeError: URL constructor: 3 is not a valid /
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetch({ not: "a valid string or guid" }),
+    /TypeError: Invalid url or guid/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetch("http://valid.uri.com", "not an object"),
+    /TypeError: options should be/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetch("http://valid.uri.com", null),
+    /TypeError: options should be/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.fetch("http://valid.uri.come", {
+        includeVisits: "not a boolean",
+      }),
+    /TypeError: includeVisits should be a/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.fetch("http://valid.uri.come", {
+        includeMeta: "not a boolean",
+      }),
+    /TypeError: includeMeta should be a/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.fetch("http://valid.url.com", {
+        includeAnnotations: "not a boolean",
+      }),
+    /TypeError: includeAnnotations should be a/
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_fetchAnnotatedPages.js b/toolkit/components/places/tests/history/test_fetchAnnotatedPages.js
new file mode 100644
index 0000000000..0f487e8090
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_fetchAnnotatedPages.js
@@ -0,0 +1,146 @@
+/* Any copyright is dedicated to the Public Domain. http://creativecommons.org/publicdomain/zero/1.0/ */
+"use strict";
+
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.fetchAnnotatedPages(),
+    /TypeError: annotations should be an Array and not null/,
+    "Should throw an exception for a null parameter"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetchAnnotatedPages("3"),
+    /TypeError: annotations should be an Array and not null/,
+    "Should throw an exception for a parameter of the wrong type"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetchAnnotatedPages([3]),
+    /TypeError: all annotation values should be strings/,
+    "Should throw an exception for a non-string annotation name"
+  );
+});
+
+add_task(async function test_fetchAnnotatedPages_no_matching() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  const TEST_URL = "http://example.com/1";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL),
+    "Should have inserted the page into the database."
+  );
+
+  let result = await PlacesUtils.history.fetchAnnotatedPages(["test/anno"]);
+
+  Assert.equal(result.size, 0, "Should be no items returned.");
+});
+
+add_task(async function test_fetchAnnotatedPages_simple_match() {
+  await PlacesUtils.history.clear();
+
+  const TEST_URL = "http://example.com/1";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL),
+    "Should have inserted the page into the database."
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/anno", "testContent"]]),
+  });
+
+  let result = await PlacesUtils.history.fetchAnnotatedPages(["test/anno"]);
+
+  Assert.equal(
+    result.size,
+    1,
+    "Should have returned one match for the annotation"
+  );
+
+  Assert.deepEqual(
+    result.get("test/anno"),
+    [
+      {
+        uri: new URL(TEST_URL),
+        content: "testContent",
+      },
+    ],
+    "Should have returned the page and its content for the annotation"
+  );
+});
+
+add_task(async function test_fetchAnnotatedPages_multiple_match() {
+  await PlacesUtils.history.clear();
+
+  const TEST_URL1 = "http://example.com/1";
+  const TEST_URL2 = "http://example.com/2";
+  const TEST_URL3 = "http://example.com/3";
+  await PlacesTestUtils.addVisits([
+    { uri: TEST_URL1 },
+    { uri: TEST_URL2 },
+    { uri: TEST_URL3 },
+  ]);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL1),
+    "Should have inserted the first page into the database."
+  );
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL2),
+    "Should have inserted the second page into the database."
+  );
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL3),
+    "Should have inserted the third page into the database."
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL1,
+    annotations: new Map([["test/anno", "testContent1"]]),
+  });
+
+  await PlacesUtils.history.update({
+    url: TEST_URL2,
+    annotations: new Map([
+      ["test/anno", "testContent2"],
+      ["test/anno2", 1234],
+    ]),
+  });
+
+  let result = await PlacesUtils.history.fetchAnnotatedPages([
+    "test/anno",
+    "test/anno2",
+  ]);
+
+  Assert.equal(
+    result.size,
+    2,
+    "Should have returned matches for both annotations"
+  );
+
+  Assert.deepEqual(
+    result.get("test/anno"),
+    [
+      {
+        uri: new URL(TEST_URL1),
+        content: "testContent1",
+      },
+      {
+        uri: new URL(TEST_URL2),
+        content: "testContent2",
+      },
+    ],
+    "Should have returned two pages and their content for the first annotation"
+  );
+
+  Assert.deepEqual(
+    result.get("test/anno2"),
+    [
+      {
+        uri: new URL(TEST_URL2),
+        content: 1234,
+      },
+    ],
+    "Should have returned one page for the second annotation"
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_fetchMany.js b/toolkit/components/places/tests/history/test_fetchMany.js
new file mode 100644
index 0000000000..53c3f6847e
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_fetchMany.js
@@ -0,0 +1,96 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function test_fetchMany() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  let pages = [
+    {
+      url: "https://mozilla.org/test1/",
+      title: "test 1",
+    },
+    {
+      url: "https://mozilla.org/test2/",
+      title: "test 2",
+    },
+    {
+      url: "https://mozilla.org/test3/",
+      title: "test 3",
+    },
+  ];
+  await PlacesTestUtils.addVisits(pages);
+
+  // Add missing page info from the database.
+  for (let page of pages) {
+    page.guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+      url: page.url,
+    });
+    page.frecency = await PlacesTestUtils.getDatabaseValue(
+      "moz_places",
+      "frecency",
+      { url: page.url }
+    );
+  }
+
+  info("Fetch by url");
+  let fetched = await PlacesUtils.history.fetchMany(pages.map(p => p.url));
+  Assert.equal(fetched.size, 3, "Map should contain same number of entries");
+  for (let page of pages) {
+    Assert.deepEqual(page, fetched.get(page.url));
+  }
+  info("Fetch by GUID");
+  fetched = await PlacesUtils.history.fetchMany(pages.map(p => p.guid));
+  Assert.equal(fetched.size, 3, "Map should contain same number of entries");
+  for (let page of pages) {
+    Assert.deepEqual(page, fetched.get(page.guid));
+  }
+  info("Fetch mixed");
+  let keys = pages.map((p, i) => (i % 2 == 0 ? p.guid : p.url));
+  fetched = await PlacesUtils.history.fetchMany(keys);
+  Assert.equal(fetched.size, 3, "Map should contain same number of entries");
+  for (let key of keys) {
+    let page = pages.find(p => p.guid == key || p.url == key);
+    Assert.deepEqual(page, fetched.get(key));
+    Assert.ok(URL.isInstance(fetched.get(key).url));
+  }
+});
+
+add_task(async function test_fetch_empty() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  let fetched = await PlacesUtils.history.fetchMany([]);
+  Assert.equal(fetched.size, 0, "Map should contain no entries");
+});
+
+add_task(async function test_fetch_nonexistent() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  let uri = NetUtil.newURI("http://doesntexist.in.db");
+  let fetched = await PlacesUtils.history.fetchMany([uri]);
+  Assert.equal(fetched.size, 0, "Map should contain no entries");
+});
+
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.fetchMany("3"),
+    /TypeError: Input is not an array/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetchMany([{ not: "a valid string or guid" }]),
+    /TypeError: Invalid url or guid/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.fetchMany(["http://valid.uri.com", "not an object"]),
+    /TypeError: URL constructor/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.fetchMany(["http://valid.uri.com", null]),
+    /TypeError: Invalid url or guid/
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_hasVisits.js b/toolkit/components/places/tests/history/test_hasVisits.js
new file mode 100644
index 0000000000..36fc9fd7be
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_hasVisits.js
@@ -0,0 +1,60 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests for `History.hasVisits` as implemented in History.jsm
+
+"use strict";
+
+add_task(async function test_has_visits_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.hasVisits(),
+    /TypeError: Invalid url or guid: undefined/,
+    "passing a null into History.hasVisits should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.hasVisits(1),
+    /TypeError: Invalid url or guid: 1/,
+    "passing an invalid url into History.hasVisits should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.hasVisits({}),
+    /TypeError: Invalid url or guid: \[object Object\]/,
+    `passing an invalid (not of type URI or nsIURI) object to History.hasVisits
+     should throw a TypeError`
+  );
+});
+
+add_task(async function test_history_has_visits() {
+  const TEST_URL = "http://mozilla.com/";
+  await PlacesUtils.history.clear();
+  Assert.equal(
+    await PlacesUtils.history.hasVisits(TEST_URL),
+    false,
+    "Test Url should not be in history."
+  );
+  Assert.equal(
+    await PlacesUtils.history.hasVisits(Services.io.newURI(TEST_URL)),
+    false,
+    "Test Url should not be in history."
+  );
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.equal(
+    await PlacesUtils.history.hasVisits(TEST_URL),
+    true,
+    "Test Url should be in history."
+  );
+  Assert.equal(
+    await PlacesUtils.history.hasVisits(Services.io.newURI(TEST_URL)),
+    true,
+    "Test Url should be in history."
+  );
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: TEST_URL,
+  });
+  Assert.equal(
+    await PlacesUtils.history.hasVisits(guid),
+    true,
+    "Test Url should be in history."
+  );
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/history/test_insert.js b/toolkit/components/places/tests/history/test_insert.js
new file mode 100644
index 0000000000..a3a820ade9
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_insert.js
@@ -0,0 +1,196 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests for `History.insert` as implemented in History.jsm
+
+"use strict";
+
+add_task(async function test_insert_error_cases() {
+  const TEST_URL = "http://mozilla.com";
+
+  Assert.throws(
+    () => PlacesUtils.history.insert(),
+    /Error: PageInfo: Input should be /,
+    "passing a null into History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert(1),
+    /Error: PageInfo: Input should be/,
+    "passing a non object into History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert({}),
+    /Error: PageInfo: The following properties were expected/,
+    "passing an object without a url to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert({ url: 123 }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing an object with an invalid url to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert({ url: TEST_URL }),
+    /Error: PageInfo: The following properties were expected/,
+    "passing an object without a visits property to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert({ url: TEST_URL, visits: 1 }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing an object with a non-array visits property to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insert({ url: TEST_URL, visits: [] }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing an object with an empty array as the visits property to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.insert({
+        url: TEST_URL,
+        visits: [
+          {
+            transition: TRANSITION_LINK,
+            date: "a",
+          },
+        ],
+      }),
+    /PageInfo: Invalid value for property/,
+    "passing a visit object with an invalid date to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.insert({
+        url: TEST_URL,
+        visits: [
+          {
+            transition: TRANSITION_LINK,
+          },
+          {
+            transition: TRANSITION_LINK,
+            date: "a",
+          },
+        ],
+      }),
+    /PageInfo: Invalid value for property/,
+    "passing a second visit object with an invalid date to History.insert should throw an Error"
+  );
+  let futureDate = new Date();
+  futureDate.setDate(futureDate.getDate() + 1000);
+  Assert.throws(
+    () =>
+      PlacesUtils.history.insert({
+        url: TEST_URL,
+        visits: [
+          {
+            transition: TRANSITION_LINK,
+            date: futureDate,
+          },
+        ],
+      }),
+    /PageInfo: Invalid value for property/,
+    "passing a visit object with a future date to History.insert should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.insert({
+        url: TEST_URL,
+        visits: [{ transition: "a" }],
+      }),
+    /PageInfo: Invalid value for property/,
+    "passing a visit object with an invalid transition to History.insert should throw an Error"
+  );
+});
+
+add_task(async function test_history_insert() {
+  const TEST_URL = "http://mozilla.com/";
+
+  let inserter = async function (name, filter, referrer, date, transition) {
+    info(name);
+    info(
+      `filter: ${filter}, referrer: ${referrer}, date: ${date}, transition: ${transition}`
+    );
+
+    let uri = NetUtil.newURI(TEST_URL + Math.random());
+    let title = "Visit " + Math.random();
+
+    let pageInfo = {
+      title,
+      visits: [{ transition, referrer, date }],
+    };
+
+    pageInfo.url = await filter(uri);
+
+    let result = await PlacesUtils.history.insert(pageInfo);
+
+    Assert.ok(
+      PlacesUtils.isValidGuid(result.guid),
+      "guid for pageInfo object is valid"
+    );
+    Assert.equal(
+      uri.spec,
+      result.url.href,
+      "url is correct for pageInfo object"
+    );
+    Assert.equal(title, result.title, "title is correct for pageInfo object");
+    Assert.equal(
+      TRANSITION_LINK,
+      result.visits[0].transition,
+      "transition is correct for pageInfo object"
+    );
+    if (referrer) {
+      Assert.equal(
+        referrer,
+        result.visits[0].referrer.href,
+        "url of referrer for visit is correct"
+      );
+    } else {
+      Assert.equal(
+        null,
+        result.visits[0].referrer,
+        "url of referrer for visit is correct"
+      );
+    }
+    if (date) {
+      Assert.equal(
+        Number(date),
+        Number(result.visits[0].date),
+        "date of visit is correct"
+      );
+    }
+
+    Assert.ok(await PlacesTestUtils.isPageInDB(uri), "Page was added");
+    Assert.ok(await PlacesTestUtils.visitsInDB(uri), "Visit was added");
+  };
+
+  try {
+    for (let referrer of [TEST_URL, null]) {
+      for (let date of [new Date(), null]) {
+        for (let transition of [TRANSITION_LINK, null]) {
+          await inserter(
+            "Testing History.insert() with an nsIURI",
+            x => x,
+            referrer,
+            date,
+            transition
+          );
+          await inserter(
+            "Testing History.insert() with a string url",
+            x => x.spec,
+            referrer,
+            date,
+            transition
+          );
+          await inserter(
+            "Testing History.insert() with a URL object",
+            x => URL.fromURI(x),
+            referrer,
+            date,
+            transition
+          );
+        }
+      }
+    }
+  } finally {
+    await PlacesUtils.history.clear();
+  }
+});
diff --git a/toolkit/components/places/tests/history/test_insertMany.js b/toolkit/components/places/tests/history/test_insertMany.js
new file mode 100644
index 0000000000..d261d3eaaa
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_insertMany.js
@@ -0,0 +1,247 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests for `History.insertMany` as implemented in History.jsm
+
+"use strict";
+
+add_task(async function test_error_cases() {
+  let validPageInfo = {
+    url: "http://mozilla.com",
+    visits: [{ transition: TRANSITION_LINK }],
+  };
+
+  Assert.throws(
+    () => PlacesUtils.history.insertMany(),
+    /TypeError: pageInfos must be an array/,
+    "passing a null into History.insertMany should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insertMany([]),
+    /TypeError: pageInfos may not be an empty array/,
+    "passing an empty array into History.insertMany should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.insertMany([validPageInfo, {}]),
+    /Error: PageInfo: The following properties were expected/,
+    "passing a second invalid PageInfo object to History.insertMany should throw an Error"
+  );
+});
+
+add_task(async function test_insertMany() {
+  const BAD_URLS = ["about:config", "chrome://browser/content/browser.xhtml"];
+  const GOOD_URLS = [1, 2, 3].map(x => {
+    return `http://mozilla.com/${x}`;
+  });
+
+  let makePageInfos = async function (urls, filter = x => x) {
+    let pageInfos = [];
+    for (let url of urls) {
+      let uri = NetUtil.newURI(url);
+
+      let pageInfo = {
+        title: `Visit to ${url}`,
+        visits: [{ transition: TRANSITION_LINK }],
+      };
+
+      pageInfo.url = await filter(uri);
+      pageInfos.push(pageInfo);
+    }
+    return pageInfos;
+  };
+
+  let inserter = async function (name, filter, useCallbacks) {
+    info(name);
+    info(`filter: ${filter}`);
+    info(`useCallbacks: ${useCallbacks}`);
+    await PlacesUtils.history.clear();
+
+    let result;
+    let allUrls = GOOD_URLS.concat(BAD_URLS);
+    let pageInfos = await makePageInfos(allUrls, filter);
+
+    if (useCallbacks) {
+      let onResultUrls = [];
+      let onErrorUrls = [];
+      result = await PlacesUtils.history.insertMany(
+        pageInfos,
+        pageInfo => {
+          let url = pageInfo.url.href;
+          Assert.ok(
+            GOOD_URLS.includes(url),
+            "onResult callback called for correct url"
+          );
+          onResultUrls.push(url);
+          Assert.equal(
+            `Visit to ${url}`,
+            pageInfo.title,
+            "onResult callback provides the correct title"
+          );
+          Assert.ok(
+            PlacesUtils.isValidGuid(pageInfo.guid),
+            "onResult callback provides a valid guid"
+          );
+        },
+        pageInfo => {
+          let url = pageInfo.url.href;
+          Assert.ok(
+            BAD_URLS.includes(url),
+            "onError callback called for correct uri"
+          );
+          onErrorUrls.push(url);
+          Assert.equal(
+            undefined,
+            pageInfo.title,
+            "onError callback provides the correct title"
+          );
+          Assert.equal(
+            undefined,
+            pageInfo.guid,
+            "onError callback provides the expected guid"
+          );
+        }
+      );
+      Assert.equal(
+        GOOD_URLS.sort().toString(),
+        onResultUrls.sort().toString(),
+        "onResult callback was called for each good url"
+      );
+      Assert.equal(
+        BAD_URLS.sort().toString(),
+        onErrorUrls.sort().toString(),
+        "onError callback was called for each bad url"
+      );
+    } else {
+      const promiseRankingChanged =
+        PlacesTestUtils.waitForNotification("pages-rank-changed");
+      result = await PlacesUtils.history.insertMany(pageInfos);
+      await promiseRankingChanged;
+    }
+
+    Assert.equal(undefined, result, "insertMany returned undefined");
+
+    for (let url of allUrls) {
+      let expected = GOOD_URLS.includes(url);
+      Assert.equal(
+        expected,
+        await PlacesTestUtils.isPageInDB(url),
+        `isPageInDB for ${url} is ${expected}`
+      );
+      Assert.equal(
+        expected,
+        await PlacesTestUtils.visitsInDB(url),
+        `visitsInDB for ${url} is ${expected}`
+      );
+    }
+  };
+
+  try {
+    for (let useCallbacks of [false, true]) {
+      await inserter(
+        "Testing History.insertMany() with an nsIURI",
+        x => x,
+        useCallbacks
+      );
+      await inserter(
+        "Testing History.insertMany() with a string url",
+        x => x.spec,
+        useCallbacks
+      );
+      await inserter(
+        "Testing History.insertMany() with a URL object",
+        x => URL.fromURI(x),
+        useCallbacks
+      );
+    }
+    // Test rejection when no items added
+    let pageInfos = await makePageInfos(BAD_URLS);
+    PlacesUtils.history.insertMany(pageInfos).then(
+      () => {
+        Assert.ok(
+          false,
+          "History.insertMany rejected promise with all bad URLs"
+        );
+      },
+      error => {
+        Assert.equal(
+          "No items were added to history.",
+          error.message,
+          "History.insertMany rejected promise with all bad URLs"
+        );
+      }
+    );
+  } finally {
+    await PlacesUtils.history.clear();
+  }
+});
+
+add_task(async function test_transitions() {
+  const places = Object.keys(PlacesUtils.history.TRANSITIONS).map(
+    transition => {
+      return {
+        url: `http://places.test/${transition}`,
+        visits: [{ transition: PlacesUtils.history.TRANSITIONS[transition] }],
+      };
+    }
+  );
+  // Should not reject.
+  await PlacesUtils.history.insertMany(places);
+  // Check callbacks.
+  let count = 0;
+  await PlacesUtils.history.insertMany(places, pageInfo => {
+    ++count;
+  });
+  Assert.equal(count, Object.keys(PlacesUtils.history.TRANSITIONS).length);
+});
+
+add_task(async function test_guid() {
+  const guidA = "aaaaaaaaaaaa";
+  const guidB = "bbbbbbbbbbbb";
+  const guidC = "cccccccccccc";
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "foo",
+      url: "http://example.com/foo",
+      guid: guidA,
+      visits: [{ transition: TRANSITION_LINK, date: new Date() }],
+    },
+  ]);
+
+  Assert.ok(
+    await PlacesUtils.history.fetch(guidA),
+    "Record is inserted with correct GUID"
+  );
+
+  let expectedGuids = new Set([guidB, guidC]);
+  await PlacesUtils.history.insertMany(
+    [
+      {
+        title: "bar",
+        url: "http://example.com/bar",
+        guid: guidB,
+        visits: [{ transition: TRANSITION_LINK, date: new Date() }],
+      },
+      {
+        title: "baz",
+        url: "http://example.com/baz",
+        guid: guidC,
+        visits: [{ transition: TRANSITION_LINK, date: new Date() }],
+      },
+    ],
+    pageInfo => {
+      Assert.ok(expectedGuids.has(pageInfo.guid));
+      expectedGuids.delete(pageInfo.guid);
+    }
+  );
+  Assert.equal(expectedGuids.size, 0);
+
+  Assert.ok(
+    await PlacesUtils.history.fetch(guidB),
+    "Record B is fetchable after insertMany"
+  );
+  Assert.ok(
+    await PlacesUtils.history.fetch(guidC),
+    "Record C is fetchable after insertMany"
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_insert_null_title.js b/toolkit/components/places/tests/history/test_insert_null_title.js
new file mode 100644
index 0000000000..8cdcddd1e8
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_insert_null_title.js
@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that passing a null title to history insert or update doesn't overwrite
+// an existing title, while an empty string does.
+
+"use strict";
+
+async function fetchTitle(url) {
+  let entry;
+  await TestUtils.waitForCondition(async () => {
+    entry = await PlacesUtils.history.fetch(url);
+    return !!entry;
+  }, "fetch title for entry");
+  return entry.title;
+}
+
+add_task(async function () {
+  const url = "http://mozilla.com";
+  let title = "Mozilla";
+
+  info("Insert a visit with a title");
+  let result = await PlacesUtils.history.insert({
+    url,
+    title,
+    visits: [{ transition: PlacesUtils.history.TRANSITIONS.LINK }],
+  });
+  Assert.equal(title, result.title, "title should be stored");
+  Assert.equal(title, await fetchTitle(url), "title should be stored");
+
+  // This is shared by the next tests.
+  let promiseTitleChange = PlacesTestUtils.waitForNotification(
+    "page-title-changed",
+    () => (notified = true)
+  );
+
+  info("Insert a visit with a null title, should not clear the previous title");
+  let notified = false;
+  result = await PlacesUtils.history.insert({
+    url,
+    title: null,
+    visits: [{ transition: PlacesUtils.history.TRANSITIONS.LINK }],
+  });
+  Assert.equal(title, result.title, "title should be unchanged");
+  Assert.equal(title, await fetchTitle(url), "title should be unchanged");
+  await Promise.race([
+    promiseTitleChange,
+    new Promise(r => do_timeout(1000, r)),
+  ]);
+  Assert.ok(!notified, "A title change should not be notified");
+
+  info(
+    "Insert a visit without specifying a title, should not clear the previous title"
+  );
+  notified = false;
+  result = await PlacesUtils.history.insert({
+    url,
+    visits: [{ transition: PlacesUtils.history.TRANSITIONS.LINK }],
+  });
+  Assert.equal(title, result.title, "title should be unchanged");
+  Assert.equal(title, await fetchTitle(url), "title should be unchanged");
+  await Promise.race([
+    promiseTitleChange,
+    new Promise(r => do_timeout(1000, r)),
+  ]);
+  Assert.ok(!notified, "A title change should not be notified");
+
+  info("Insert a visit with an empty title, should clear the previous title");
+  result = await PlacesUtils.history.insert({
+    url,
+    title: "",
+    visits: [{ transition: PlacesUtils.history.TRANSITIONS.LINK }],
+  });
+  info("Waiting for the title change notification");
+  await promiseTitleChange;
+  Assert.equal("", result.title, "title should be empty");
+  Assert.equal("", await fetchTitle(url), "title should be empty");
+});
diff --git a/toolkit/components/places/tests/history/test_remove.js b/toolkit/components/places/tests/history/test_remove.js
new file mode 100644
index 0000000000..8c5e941fd0
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_remove.js
@@ -0,0 +1,354 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+// Tests for `History.remove`, as implemented in History.jsm
+
+"use strict";
+
+// Test removing a single page
+add_task(async function test_remove_single() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  let WITNESS_URI = NetUtil.newURI(
+    "http://mozilla.com/test_browserhistory/test_remove/" + Math.random()
+  );
+  await PlacesTestUtils.addVisits(WITNESS_URI);
+  Assert.ok(page_in_database(WITNESS_URI));
+
+  let remover = async function (name, filter, options) {
+    info(name);
+    info(JSON.stringify(options));
+    info("Setting up visit");
+
+    let uri = NetUtil.newURI(
+      "http://mozilla.com/test_browserhistory/test_remove/" + Math.random()
+    );
+    let title = "Visit " + Math.random();
+    await PlacesTestUtils.addVisits({ uri, title });
+    Assert.ok(visits_in_database(uri), "History entry created");
+
+    let removeArg = await filter(uri);
+
+    if (options.addBookmark) {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: uri,
+        title: "test bookmark",
+      });
+    }
+
+    let shouldRemove = !options.addBookmark;
+    let placesEventListener;
+    let promiseObserved = new Promise((resolve, reject) => {
+      placesEventListener = events => {
+        for (const event of events) {
+          switch (event.type) {
+            case "page-title-changed": {
+              reject(
+                "Unexpected page-title-changed event happens on " + event.url
+              );
+              break;
+            }
+            case "history-cleared": {
+              reject("Unexpected history-cleared event happens");
+              break;
+            }
+            case "pages-rank-changed": {
+              try {
+                Assert.ok(!shouldRemove, "Observing pages-rank-changed event");
+              } finally {
+                resolve();
+              }
+              break;
+            }
+            case "page-removed": {
+              Assert.equal(
+                event.isRemovedFromStore,
+                shouldRemove,
+                "Observe page-removed event with right removal type"
+              );
+              Assert.equal(
+                event.url,
+                uri.spec,
+                "Observing effect on the right uri"
+              );
+              resolve();
+              break;
+            }
+          }
+        }
+      };
+    });
+    PlacesObservers.addListener(
+      [
+        "page-title-changed",
+        "history-cleared",
+        "pages-rank-changed",
+        "page-removed",
+      ],
+      placesEventListener
+    );
+
+    info("Performing removal");
+    let removed = false;
+    if (options.useCallback) {
+      let onRowCalled = false;
+      let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+        url: uri,
+      });
+      removed = await PlacesUtils.history.remove(removeArg, page => {
+        Assert.equal(onRowCalled, false, "Callback has not been called yet");
+        onRowCalled = true;
+        Assert.equal(
+          page.url.href,
+          uri.spec,
+          "Callback provides the correct url"
+        );
+        Assert.equal(page.guid, guid, "Callback provides the correct guid");
+        Assert.equal(page.title, title, "Callback provides the correct title");
+      });
+      Assert.ok(onRowCalled, "Callback has been called");
+    } else {
+      removed = await PlacesUtils.history.remove(removeArg);
+    }
+
+    await promiseObserved;
+    PlacesObservers.removeListener(
+      [
+        "page-title-changed",
+        "history-cleared",
+        "pages-rank-changed",
+        "page-removed",
+      ],
+      placesEventListener
+    );
+
+    Assert.equal(visits_in_database(uri), 0, "History entry has disappeared");
+    Assert.notEqual(
+      visits_in_database(WITNESS_URI),
+      0,
+      "Witness URI still has visits"
+    );
+    Assert.notEqual(
+      page_in_database(WITNESS_URI),
+      0,
+      "Witness URI is still here"
+    );
+    if (shouldRemove) {
+      Assert.ok(removed, "Something was removed");
+      Assert.equal(page_in_database(uri), 0, "Page has disappeared");
+    } else {
+      Assert.ok(!removed, "The page was not removed, as there was a bookmark");
+      Assert.notEqual(page_in_database(uri), 0, "The page is still present");
+    }
+  };
+
+  try {
+    for (let useCallback of [false, true]) {
+      for (let addBookmark of [false, true]) {
+        let options = { useCallback, addBookmark };
+        await remover(
+          "Testing History.remove() with a single URI",
+          x => x,
+          options
+        );
+        await remover(
+          "Testing History.remove() with a single string url",
+          x => x.spec,
+          options
+        );
+        await remover(
+          "Testing History.remove() with a single string guid",
+          async x =>
+            PlacesTestUtils.getDatabaseValue("moz_places", "guid", { url: x }),
+          options
+        );
+        await remover(
+          "Testing History.remove() with a single URI in an array",
+          x => [x],
+          options
+        );
+        await remover(
+          "Testing History.remove() with a single string url in an array",
+          x => [x.spec],
+          options
+        );
+        await remover(
+          "Testing History.remove() with a single string guid in an array",
+          x =>
+            PlacesTestUtils.getDatabaseValue("moz_places", "guid", { url: x }),
+          options
+        );
+      }
+    }
+  } finally {
+    await PlacesUtils.history.clear();
+  }
+});
+
+add_task(async function cleanup() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+// Test the various error cases
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.remove(),
+    /TypeError: Invalid url/,
+    "History.remove with no argument should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove(null),
+    /TypeError: Invalid url/,
+    "History.remove with `null` should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove(undefined),
+    /TypeError: Invalid url/,
+    "History.remove with `undefined` should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove("not a guid, obviously"),
+    /TypeError: .* is not a valid URL/,
+    "History.remove with an ill-formed guid/url argument should throw a TypeError"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.remove({
+        "not the kind of object we know how to handle": true,
+      }),
+    /TypeError: Invalid url/,
+    "History.remove with an unexpected object should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove([]),
+    /TypeError: Expected at least one page/,
+    "History.remove with an empty array should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove([null]),
+    /TypeError: Invalid url or guid/,
+    "History.remove with an array containing null should throw a TypeError"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.remove([
+        "http://example.org",
+        "not a guid, obviously",
+      ]),
+    /TypeError: .* is not a valid URL/,
+    "History.remove with an array containing an ill-formed guid/url argument should throw a TypeError"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.remove(["0123456789ab" /* valid guid*/, null]),
+    /TypeError: Invalid url or guid: null/,
+    "History.remove with an array containing a guid and a second argument that is null should throw a TypeError"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.remove([
+        "http://example.org",
+        { "not the kind of object we know how to handle": true },
+      ]),
+    /TypeError: Invalid url/,
+    "History.remove with an array containing an unexpected objecgt should throw a TypeError"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.remove(
+        "http://example.org",
+        "not a function, obviously"
+      ),
+    /TypeError: Invalid function/,
+    "History.remove with a second argument that is not a function argument should throw a TypeError"
+  );
+  try {
+    PlacesUtils.history.remove(
+      "http://example.org/I/have/clearly/not/been/added",
+      null
+    );
+    Assert.ok(true, "History.remove should ignore `null` as a second argument");
+  } catch (ex) {
+    Assert.ok(
+      false,
+      "History.remove should ignore `null` as a second argument"
+    );
+  }
+});
+
+add_task(async function test_orphans() {
+  let uri = NetUtil.newURI("http://moz.org/");
+  await PlacesTestUtils.addVisits({ uri });
+
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    uri,
+    SMALLPNG_DATA_URI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+  // Also create a root icon.
+  let faviconURI = Services.io.newURI(uri.spec + "favicon.ico");
+  PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+    faviconURI,
+    SMALLPNG_DATA_URI.spec,
+    0,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    uri,
+    faviconURI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+
+  await PlacesUtils.history.update({
+    url: uri,
+    annotations: new Map([["test", "restval"]]),
+  });
+
+  await PlacesUtils.history.remove(uri);
+  Assert.ok(
+    !(await PlacesTestUtils.isPageInDB(uri)),
+    "Page should have been removed"
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`SELECT (SELECT count(*) FROM moz_annos) +
+                                      (SELECT count(*) FROM moz_icons) +
+                                      (SELECT count(*) FROM moz_pages_w_icons) +
+                                      (SELECT count(*) FROM moz_icons_to_pages) AS count`);
+  Assert.equal(rows[0].getResultByName("count"), 0, "Should not find orphans");
+});
+
+add_task(async function test_remove_backslash() {
+  // Backslash is an escape char in Sqlite, we must take care of that when
+  // removing a url containing a backslash.
+  const url = "https://www.mozilla.org/?test=\u005C";
+  await PlacesTestUtils.addVisits(url);
+  Assert.ok(await PlacesUtils.history.remove(url), "A page should be removed");
+  Assert.deepEqual(
+    await PlacesUtils.history.fetch(url),
+    null,
+    "The page should not be found"
+  );
+});
+
+add_task(async function test_url_with_apices() {
+  // Apices may confuse code and cause injection if mishandled.
+  // The ideal test would be with a javascript url, because it would not be
+  // encoded by URL(), unfortunately it would also not be added to history.
+  const url = `http://mozilla.org/\u0022\u0027`;
+  await PlacesTestUtils.addVisits(url);
+  Assert.ok(await PlacesUtils.history.remove(url), "A page should be removed");
+  Assert.deepEqual(
+    await PlacesUtils.history.fetch(url),
+    null,
+    "The page should not be found"
+  );
+});
diff --git a/toolkit/components/places/tests/history/test_removeByFilter.js b/toolkit/components/places/tests/history/test_removeByFilter.js
new file mode 100644
index 0000000000..fb18bf8e74
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_removeByFilter.js
@@ -0,0 +1,497 @@
+"use strict";
+
+/*
+This test will ideally test the following cases
+(each with and without a callback associated with it)
+  Case A: Tests which should remove pages (Positives)
+    Case A 1: Page has multiple visits both in/out of timeframe, all get deleted
+    Case A 2: Page has single uri, removed by host
+    Case A 3: Page has random subhost, with same host, removed by wildcard
+    Case A 4: Page is localhost and localhost:port, removed by host
+    Case A 5: Page is a `file://` type address, removed by empty host
+    Cases A 1,2,3 will be tried with and without bookmarks added (which prevent page deletion)
+  Case B: Tests in which no pages are removed (Inverses)
+    Case B 1 (inverse): Page has no visits in timeframe, and nothing is deleted
+    Case B 2: Page has single uri, not removed since hostname is different
+    Case B 3: Page has multiple subhosts, not removed since wildcard doesn't match
+  Case C: Combinations tests
+    Case C 1: Single hostname, multiple visits, at least one in timeframe and hostname
+    Case C 2: Random subhosts, multiple visits, at least one in timeframe and hostname-wildcard
+*/
+
+add_task(async function test_removeByFilter() {
+  // Cleanup
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  // Adding a witness URI
+  let witnessURI = NetUtil.newURI(
+    "http://witnessmozilla.org/test_browserhistory/test_removeByFilter" +
+      Math.random()
+  );
+  await PlacesTestUtils.addVisits(witnessURI);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(witnessURI),
+    "Witness URI is in database"
+  );
+
+  let removeByFilterTester = async function (
+    visits,
+    filter,
+    checkBeforeRemove,
+    checkAfterRemove,
+    useCallback,
+    bookmarkedUri
+  ) {
+    // Add visits for URIs
+    await PlacesTestUtils.addVisits(visits);
+    if (
+      bookmarkedUri !== null &&
+      visits.map(v => v.uri).includes(bookmarkedUri)
+    ) {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: bookmarkedUri,
+        title: "test bookmark",
+      });
+    }
+    await checkBeforeRemove();
+
+    // Take care of any observers (due to bookmarks)
+    let { placesEventListener, promiseObserved } =
+      getObserverPromise(bookmarkedUri);
+    if (placesEventListener) {
+      PlacesObservers.addListener(
+        ["page-title-changed", "history-cleared", "page-removed"],
+        placesEventListener
+      );
+    }
+    // Perfom delete operation on database
+    let removed = false;
+    if (useCallback) {
+      // The amount of callbacks will be the unique URIs to remove from the database
+      let netCallbacksRequired = new Set(visits.map(v => v.uri)).size;
+      removed = await PlacesUtils.history.removeByFilter(filter, pageInfo => {
+        Assert.ok(
+          PlacesUtils.validatePageInfo(pageInfo, false),
+          "pageInfo should follow a basic format"
+        );
+        Assert.ok(
+          netCallbacksRequired > 0,
+          "Callback called as many times as required"
+        );
+        netCallbacksRequired--;
+      });
+    } else {
+      removed = await PlacesUtils.history.removeByFilter(filter);
+    }
+    await checkAfterRemove();
+    await promiseObserved;
+    if (placesEventListener) {
+      await PlacesUtils.bookmarks.eraseEverything();
+      PlacesObservers.removeListener(
+        ["page-title-changed", "history-cleared", "page-removed"],
+        placesEventListener
+      );
+    }
+    Assert.ok(
+      await PlacesTestUtils.isPageInDB(witnessURI),
+      "Witness URI is still in database"
+    );
+    return removed;
+  };
+
+  const remoteUriList = [
+    "http://mozilla.org/test_browserhistory/test_removeByFilter/" +
+      Math.random(),
+    "http://subdomain1.mozilla.org/test_browserhistory/test_removeByFilter/" +
+      Math.random(),
+    "http://subdomain2.mozilla.org/test_browserhistory/test_removeByFilter/" +
+      Math.random(),
+  ];
+  const localhostUriList = [
+    "http://localhost:4500/" + Math.random(),
+    "http://localhost/" + Math.random(),
+  ];
+  const fileUriList = ["file:///home/user/files" + Math.random()];
+  const title = "Title " + Math.random();
+  let sameHostVisits = [
+    {
+      uri: remoteUriList[0],
+      title,
+      visitDate: new Date(2005, 1, 1) * 1000,
+    },
+    {
+      uri: remoteUriList[0],
+      title,
+      visitDate: new Date(2005, 3, 3) * 1000,
+    },
+    {
+      uri: remoteUriList[0],
+      title,
+      visitDate: new Date(2007, 1, 1) * 1000,
+    },
+  ];
+  let randomHostVisits = [
+    {
+      uri: remoteUriList[0],
+      title,
+      visitDate: new Date(2005, 1, 1) * 1000,
+    },
+    {
+      uri: remoteUriList[1],
+      title,
+      visitDate: new Date(2005, 3, 3) * 1000,
+    },
+    {
+      uri: remoteUriList[2],
+      title,
+      visitDate: new Date(2007, 1, 1) * 1000,
+    },
+  ];
+  let localhostVisits = [
+    {
+      uri: localhostUriList[0],
+      title,
+    },
+    {
+      uri: localhostUriList[1],
+      title,
+    },
+  ];
+  let fileVisits = [
+    {
+      uri: fileUriList[0],
+      title,
+    },
+  ];
+  let assertInDB = async function (aUri) {
+    Assert.ok(await PlacesTestUtils.isPageInDB(aUri));
+  };
+  let assertNotInDB = async function (aUri) {
+    Assert.ok(!(await PlacesTestUtils.isPageInDB(aUri)));
+  };
+  for (let callbackUse of [true, false]) {
+    // Case A Positives
+    for (let bookmarkUse of [true, false]) {
+      let bookmarkedUri = arr => undefined;
+      let checkableArray = arr => arr;
+      let checkClosure = assertNotInDB;
+      if (bookmarkUse) {
+        bookmarkedUri = arr => arr[0];
+        checkableArray = arr => arr.slice(1);
+        checkClosure = function (aUri) {};
+      }
+      // Case A 1: Dates
+      await removeByFilterTester(
+        sameHostVisits,
+        { beginDate: new Date(2004, 1, 1), endDate: new Date(2006, 1, 1) },
+        () => assertInDB(remoteUriList[0]),
+        () => checkClosure(remoteUriList[0]),
+        callbackUse,
+        bookmarkedUri(remoteUriList)
+      );
+      // Case A 2: Single Sub-host
+      await removeByFilterTester(
+        sameHostVisits,
+        { host: "mozilla.org" },
+        () => assertInDB(remoteUriList[0]),
+        () => checkClosure(remoteUriList[0]),
+        callbackUse,
+        bookmarkedUri(remoteUriList)
+      );
+      // Case A 3: Multiple subhost
+      await removeByFilterTester(
+        randomHostVisits,
+        { host: ".mozilla.org" },
+        async () => {
+          for (let uri of remoteUriList) {
+            await assertInDB(uri);
+          }
+        },
+        async () => {
+          for (let uri of checkableArray(remoteUriList)) {
+            await checkClosure(uri);
+          }
+        },
+        callbackUse,
+        bookmarkedUri(remoteUriList)
+      );
+    }
+
+    // Case A 4: Localhost
+    await removeByFilterTester(
+      localhostVisits,
+      { host: "localhost" },
+      async () => {
+        for (let uri of localhostUriList) {
+          await assertInDB(uri);
+        }
+      },
+      async () => {
+        for (let uri of localhostUriList) {
+          await assertNotInDB(uri);
+        }
+      },
+      callbackUse
+    );
+    // Case A 5: Local Files
+    await removeByFilterTester(
+      fileVisits,
+      { host: "." },
+      async () => {
+        for (let uri of fileUriList) {
+          await assertInDB(uri);
+        }
+      },
+      async () => {
+        for (let uri of fileUriList) {
+          await assertNotInDB(uri);
+        }
+      },
+      callbackUse
+    );
+
+    // Case B: Tests which do not remove anything (inverses)
+    // Case B 1: Date
+    await removeByFilterTester(
+      sameHostVisits,
+      { beginDate: new Date(2001, 1, 1), endDate: new Date(2002, 1, 1) },
+      () => assertInDB(remoteUriList[0]),
+      () => assertInDB(remoteUriList[0]),
+      callbackUse
+    );
+    // Case B 2 : Single subhost
+    await removeByFilterTester(
+      sameHostVisits,
+      { host: "notthere.org" },
+      () => assertInDB(remoteUriList[0]),
+      () => assertInDB(remoteUriList[0]),
+      callbackUse
+    );
+    // Case B 3 : Multiple subhosts
+    await removeByFilterTester(
+      randomHostVisits,
+      { host: ".notthere.org" },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertInDB(uri);
+        }
+      },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertInDB(uri);
+        }
+      },
+      callbackUse
+    );
+    // Case B 4 : invalid local subhost
+    await removeByFilterTester(
+      randomHostVisits,
+      { host: ".org" },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertInDB(uri);
+        }
+      },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertInDB(uri);
+        }
+      },
+      callbackUse
+    );
+
+    // Case C: Combination Cases
+    // Case C 1: single subhost
+    await removeByFilterTester(
+      sameHostVisits,
+      {
+        host: "mozilla.org",
+        beginDate: new Date(2004, 1, 1),
+        endDate: new Date(2006, 1, 1),
+      },
+      () => assertInDB(remoteUriList[0]),
+      () => assertNotInDB(remoteUriList[0]),
+      callbackUse
+    );
+    // Case C 2: multiple subhost
+    await removeByFilterTester(
+      randomHostVisits,
+      {
+        host: ".mozilla.org",
+        beginDate: new Date(2005, 1, 1),
+        endDate: new Date(2017, 1, 1),
+      },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertInDB(uri);
+        }
+      },
+      async () => {
+        for (let uri of remoteUriList) {
+          await assertNotInDB(uri);
+        }
+      },
+      callbackUse
+    );
+  }
+});
+
+// Test various error cases
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter(),
+    /TypeError: Expected a filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter("obviously, not a filter"),
+    /TypeError: Expected a filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({}),
+    /TypeError: Expected a non-empty filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ beginDate: "now" }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ beginDate: Date.now() }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ beginDate: new Date(NaN) }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeByFilter(
+        { beginDate: new Date() },
+        "obviously, not a callback"
+      ),
+    /TypeError: Invalid function/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeByFilter({
+        beginDate: new Date(1000),
+        endDate: new Date(0),
+      }),
+    /TypeError: `beginDate` should be at least as old/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "#" }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "www..org" }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: {} }),
+    /TypeError: `host` should be a string/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "*.mozilla.org" }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "*" }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "local.host." }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "(local files)" }),
+    /TypeError: Expected well formed hostname string for/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeByFilter({ host: "" }),
+    /TypeError: Expected a non-empty filter/
+  );
+});
+
+add_task(async function test_chunking() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  info("Insert many visited pages");
+  let pages = [];
+  for (let i = 1; i <= 1500; i++) {
+    let visits = [
+      {
+        date: new Date(Date.now() - (86400 + i) * 1000),
+        transition: PlacesUtils.history.TRANSITIONS.TYPED,
+      },
+    ];
+    pages.push(
+      {
+        url: `http://example.com/${i}`,
+        title: `Page ${i}`,
+        visits,
+      },
+      {
+        url: `http://subdomain.example.com/${i}`,
+        title: `Subdomain ${i}`,
+        visits,
+      }
+    );
+  }
+  await PlacesUtils.history.insertMany(pages);
+
+  info("Remove all visited pages");
+  await PlacesUtils.history.removeByFilter({
+    host: ".example.com",
+  });
+});
+
+// Helper functions
+
+function getObserverPromise(bookmarkedUri) {
+  if (!bookmarkedUri) {
+    return { promiseObserved: Promise.resolve() };
+  }
+  let placesEventListener;
+  let promiseObserved = new Promise((resolve, reject) => {
+    placesEventListener = events => {
+      for (const event of events) {
+        switch (event.type) {
+          case "page-title-changed": {
+            reject(new Error("Unexpected page-title-changed event happens"));
+            break;
+          }
+          case "history-cleared": {
+            reject(new Error("Unexpected history-cleared event happens"));
+            break;
+          }
+          case "page-removed": {
+            if (event.isRemovedFromStore) {
+              Assert.notEqual(
+                event.url,
+                bookmarkedUri,
+                "Bookmarked URI should not be deleted"
+              );
+            } else {
+              Assert.equal(
+                event.isPartialVisistsRemoval,
+                false,
+                "Observing page-removed deletes all visits"
+              );
+              Assert.equal(
+                event.url,
+                bookmarkedUri,
+                "Bookmarked URI should have all visits removed but not the page itself"
+              );
+            }
+            resolve();
+            break;
+          }
+        }
+      }
+    };
+  });
+  return { placesEventListener, promiseObserved };
+}
diff --git a/toolkit/components/places/tests/history/test_removeMany.js b/toolkit/components/places/tests/history/test_removeMany.js
new file mode 100644
index 0000000000..ff8c3a21ee
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_removeMany.js
@@ -0,0 +1,206 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+// Tests for `History.remove` with removing many urls, as implemented in
+// History.jsm.
+
+"use strict";
+
+// Test removing a list of pages
+add_task(async function test_remove_many() {
+  // This is set so that we are guaranteed to trigger REMOVE_PAGES_CHUNKLEN.
+  const SIZE = 310;
+
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  info("Adding a witness page");
+  let WITNESS_URI = NetUtil.newURI(
+    "http://mozilla.com/test_browserhistory/test_remove/" + Math.random()
+  );
+  await PlacesTestUtils.addVisits(WITNESS_URI);
+  Assert.ok(page_in_database(WITNESS_URI), "Witness page added");
+
+  info("Generating samples");
+  let pages = [];
+  for (let i = 0; i < SIZE; ++i) {
+    let uri = NetUtil.newURI(
+      "http://mozilla.com/test_browserhistory/test_remove?sample=" +
+        i +
+        "&salt=" +
+        Math.random()
+    );
+    let title = "Visit " + i + ", " + Math.random();
+    let hasBookmark = i % 3 == 0;
+    let page = {
+      uri,
+      title,
+      hasBookmark,
+      // `true` once `onResult` has been called for this page
+      onResultCalled: false,
+      // `true` once page-removed for store has been fired for this page
+      pageRemovedFromStore: false,
+      // `true` once page-removed for all visits has been fired for this page
+      pageRemovedAllVisits: false,
+    };
+    info("Pushing: " + uri.spec);
+    pages.push(page);
+
+    await PlacesTestUtils.addVisits(page);
+    page.guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+      url: uri,
+    });
+    if (hasBookmark) {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: uri,
+        title: "test bookmark " + i,
+      });
+    }
+    Assert.ok(page_in_database(uri), "Page added");
+  }
+
+  info("Mixing key types and introducing dangling keys");
+  let keys = [];
+  for (let i = 0; i < SIZE; ++i) {
+    if (i % 4 == 0) {
+      keys.push(pages[i].uri);
+      keys.push(NetUtil.newURI("http://example.org/dangling/nsIURI/" + i));
+    } else if (i % 4 == 1) {
+      keys.push(new URL(pages[i].uri.spec));
+      keys.push(new URL("http://example.org/dangling/URL/" + i));
+    } else if (i % 4 == 2) {
+      keys.push(pages[i].uri.spec);
+      keys.push("http://example.org/dangling/stringuri/" + i);
+    } else {
+      keys.push(pages[i].guid);
+      keys.push(("guid_" + i + "_01234567890").substr(0, 12));
+    }
+  }
+
+  let onPageRankingChanged = false;
+  const placesEventListener = events => {
+    for (const event of events) {
+      switch (event.type) {
+        case "page-title-changed": {
+          Assert.ok(
+            false,
+            "Unexpected page-title-changed event happens on " + event.url
+          );
+          break;
+        }
+        case "history-cleared": {
+          Assert.ok(false, "Unexpected history-cleared event happens");
+          break;
+        }
+        case "pages-rank-changed": {
+          onPageRankingChanged = true;
+          break;
+        }
+        case "page-removed": {
+          const origin = pages.find(x => x.uri.spec === event.url);
+          Assert.ok(origin);
+
+          if (event.isRemovedFromStore) {
+            Assert.ok(
+              !origin.hasBookmark,
+              "Observing page-removed event on a page without a bookmark"
+            );
+            Assert.ok(
+              !origin.pageRemovedFromStore,
+              "Observing page-removed for store for the first time"
+            );
+            origin.pageRemovedFromStore = true;
+          } else {
+            Assert.ok(
+              !origin.pageRemovedAllVisits,
+              "Observing page-removed for all visits for the first time"
+            );
+            origin.pageRemovedAllVisits = true;
+          }
+          break;
+        }
+      }
+    }
+  };
+
+  PlacesObservers.addListener(
+    [
+      "page-title-changed",
+      "history-cleared",
+      "pages-rank-changed",
+      "page-removed",
+    ],
+    placesEventListener
+  );
+
+  info("Removing the pages and checking the callbacks");
+
+  let removed = await PlacesUtils.history.remove(keys, page => {
+    let origin = pages.find(candidate => candidate.uri.spec == page.url.href);
+
+    Assert.ok(origin, "onResult has a valid page");
+    Assert.ok(!origin.onResultCalled, "onResult has not seen this page yet");
+    origin.onResultCalled = true;
+    Assert.equal(page.guid, origin.guid, "onResult has the right guid");
+    Assert.equal(page.title, origin.title, "onResult has the right title");
+  });
+  Assert.ok(removed, "Something was removed");
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  PlacesObservers.removeListener(
+    [
+      "page-title-changed",
+      "history-cleared",
+      "pages-rank-changed",
+      "page-removed",
+    ],
+    placesEventListener
+  );
+
+  info("Checking out results");
+  // By now the observers should have been called.
+  for (let i = 0; i < pages.length; ++i) {
+    let page = pages[i];
+    Assert.ok(
+      page.onResultCalled,
+      `We have reached the page #${i} from the callback`
+    );
+    Assert.ok(
+      visits_in_database(page.uri) == 0,
+      "History entry has disappeared"
+    );
+    Assert.equal(
+      page_in_database(page.uri) != 0,
+      page.hasBookmark,
+      "Page is present only if it also has bookmarks"
+    );
+    Assert.notEqual(
+      page.pageRemovedFromStore,
+      page.pageRemovedAllVisits,
+      "Either only page-removed event for store or all visits should be called"
+    );
+  }
+
+  Assert.equal(
+    onPageRankingChanged,
+    pages.some(p => p.pageRemovedFromStore || p.pageRemovedAllVisits),
+    "page-rank-changed was fired"
+  );
+
+  Assert.notEqual(
+    visits_in_database(WITNESS_URI),
+    0,
+    "Witness URI still has visits"
+  );
+  Assert.notEqual(
+    page_in_database(WITNESS_URI),
+    0,
+    "Witness URI is still here"
+  );
+});
+
+add_task(async function cleanup() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
diff --git a/toolkit/components/places/tests/history/test_removeVisits.js b/toolkit/components/places/tests/history/test_removeVisits.js
new file mode 100644
index 0000000000..3a82132bd8
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_removeVisits.js
@@ -0,0 +1,376 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const JS_NOW = Date.now();
+const DB_NOW = JS_NOW * 1000;
+const TEST_URI = uri("http://example.com/");
+
+async function cleanup() {
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+  // This is needed to remove place: entries.
+  DBConn().executeSimpleSQL("DELETE FROM moz_places");
+}
+
+add_task(async function remove_visits_outside_unbookmarked_uri() {
+  info(
+    "*** TEST: Remove some visits outside valid timeframe from an unbookmarked URI"
+  );
+
+  info("Add 10 visits for the URI from way in the past.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - 100000 - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+
+  info("Remove visits using timerange outside the URI's visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 10),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+
+  info("Run a history query and check that all visits still exist.");
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 10);
+  for (let i = 0; i < root.childCount; i++) {
+    let visitTime = root.getChild(i).time;
+    Assert.equal(visitTime, DB_NOW - 100000 - i * 1000);
+  }
+  root.containerOpen = false;
+
+  Assert.ok(
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist"
+  );
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Frecency should be positive.");
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_visits_outside_bookmarked_uri() {
+  info(
+    "*** TEST: Remove some visits outside valid timeframe from a bookmarked URI"
+  );
+
+  info("Add 10 visits for the URI from way in the past.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - 100000 - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+  info("Bookmark the URI.");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: TEST_URI,
+    title: "bookmark title",
+  });
+
+  info("Remove visits using timerange outside the URI's visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 10),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+
+  info("Run a history query and check that all visits still exist.");
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 10);
+  for (let i = 0; i < root.childCount; i++) {
+    let visitTime = root.getChild(i).time;
+    Assert.equal(visitTime, DB_NOW - 100000 - i * 1000);
+  }
+  root.containerOpen = false;
+
+  Assert.ok(
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist"
+  );
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Frecency should be positive.");
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_visits_unbookmarked_uri() {
+  info("*** TEST: Remove some visits from an unbookmarked URI");
+
+  info("Add 10 visits for the URI from now to 9 usecs in the past.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+
+  info("Remove the 5 most recent visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 4),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+
+  info(
+    "Run a history query and check that only the older 5 visits still exist."
+  );
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 5);
+  for (let i = 0; i < root.childCount; i++) {
+    let visitTime = root.getChild(i).time;
+    Assert.equal(visitTime, DB_NOW - i * 1000 - 5000);
+  }
+  root.containerOpen = false;
+
+  Assert.ok(
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist"
+  );
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Frecency should be positive.");
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_visits_bookmarked_uri() {
+  info("*** TEST: Remove some visits from a bookmarked URI");
+
+  info("Add 10 visits for the URI from now to 9 usecs in the past.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+  info("Bookmark the URI.");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: TEST_URI,
+    title: "bookmark title",
+  });
+
+  info("Remove the 5 most recent visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 4),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+
+  info(
+    "Run a history query and check that only the older 5 visits still exist."
+  );
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 5);
+  for (let i = 0; i < root.childCount; i++) {
+    let visitTime = root.getChild(i).time;
+    Assert.equal(visitTime, DB_NOW - i * 1000 - 5000);
+  }
+  root.containerOpen = false;
+
+  Assert.ok(
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should exist"
+  );
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Frecency should be positive.");
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_all_visits_unbookmarked_uri() {
+  info("*** TEST: Remove all visits from an unbookmarked URI");
+
+  info("Add some visits for the URI.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+
+  info("Remove all visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 10),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should no longer exist in moz_places.");
+  Assert.ok(!page_in_database(TEST_URI.spec));
+
+  info("Run a history query and check that no visits exist.");
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+
+  Assert.equal(
+    false,
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should not exist"
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_all_visits_bookmarked_uri() {
+  info("*** TEST: Remove all visits from a bookmarked URI");
+
+  info("Add some visits for the URI.");
+  let visits = [];
+  for (let i = 0; i < 10; i++) {
+    visits.push({ uri: TEST_URI, visitDate: DB_NOW - i * 1000 });
+  }
+  await PlacesTestUtils.addVisits(visits);
+  info("Bookmark the URI.");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: TEST_URI,
+    title: "bookmark title",
+  });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  let initialFrecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    { url: TEST_URI }
+  );
+
+  info("Remove all visits.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 10),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+
+  info("Run a history query and check that no visits exist.");
+  let query = PlacesUtils.history.getNewQuery();
+  let opts = PlacesUtils.history.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_VISIT;
+  opts.sortingMode = opts.SORT_BY_DATE_DESCENDING;
+  let root = PlacesUtils.history.executeQuery(query, opts).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+
+  Assert.equal(
+    false,
+    await PlacesUtils.history.hasVisits(TEST_URI),
+    "visit should not exist"
+  );
+
+  info("URI should be bookmarked");
+  Assert.ok(await PlacesUtils.bookmarks.fetch({ url: TEST_URI }));
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Frecency should be smaller.");
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) < initialFrecency
+  );
+
+  await cleanup();
+});
+
+add_task(async function remove_all_visits_bookmarked_uri() {
+  info(
+    "*** TEST: Remove some visits from a zero frecency URI retains zero frecency"
+  );
+
+  info("Add some visits for the URI.");
+  await PlacesTestUtils.addVisits([
+    {
+      uri: TEST_URI,
+      transition: TRANSITION_FRAMED_LINK,
+      visitDate: DB_NOW - 86400000000000,
+    },
+    { uri: TEST_URI, transition: TRANSITION_FRAMED_LINK, visitDate: DB_NOW },
+  ]);
+
+  info("Remove newer visit.");
+  let filter = {
+    beginDate: new Date(JS_NOW - 10),
+    endDate: new Date(JS_NOW),
+  };
+  await PlacesUtils.history.removeVisitsByFilter(filter);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI should still exist in moz_places.");
+  Assert.ok(page_in_database(TEST_URI.spec));
+  info("Frecency should be zero.");
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await cleanup();
+});
diff --git a/toolkit/components/places/tests/history/test_removeVisitsByFilter.js b/toolkit/components/places/tests/history/test_removeVisitsByFilter.js
new file mode 100644
index 0000000000..3d71c7348a
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_removeVisitsByFilter.js
@@ -0,0 +1,409 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+// Tests for `History.removeVisitsByFilter`, as implemented in History.jsm
+
+"use strict";
+
+const { PromiseUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PromiseUtils.sys.mjs"
+);
+
+add_task(async function test_removeVisitsByFilter() {
+  let referenceDate = new Date(1999, 9, 9, 9, 9);
+
+  // Populate a database with 20 entries, remove a subset of entries,
+  // ensure consistency.
+  let remover = async function (options) {
+    info("Remover with options " + JSON.stringify(options));
+    let SAMPLE_SIZE = options.sampleSize;
+
+    await PlacesUtils.history.clear();
+    await PlacesUtils.bookmarks.eraseEverything();
+
+    // Populate the database.
+    // Create `SAMPLE_SIZE` visits, from the oldest to the newest.
+
+    let bookmarkIndices = new Set(options.bookmarks);
+    let visits = [];
+    let rankingChangePromises = [];
+    let uriDeletePromises = new Map();
+    let getURL = options.url
+      ? i =>
+          "http://mozilla.com/test_browserhistory/test_removeVisitsByFilter/removeme/byurl/" +
+          Math.floor(i / (SAMPLE_SIZE / 5)) +
+          "/"
+      : i =>
+          "http://mozilla.com/test_browserhistory/test_removeVisitsByFilter/removeme/" +
+          i +
+          "/" +
+          Math.random();
+    for (let i = 0; i < SAMPLE_SIZE; ++i) {
+      let spec = getURL(i);
+      let uri = NetUtil.newURI(spec);
+      let jsDate = new Date(Number(referenceDate) + 3600 * 1000 * i);
+      let dbDate = jsDate * 1000;
+      let hasBookmark = bookmarkIndices.has(i);
+      let hasOwnBookmark = hasBookmark;
+      if (!hasOwnBookmark && options.url) {
+        // Also mark as bookmarked if one of the earlier bookmarked items has the same URL.
+        hasBookmark = options.bookmarks
+          .filter(n => n < i)
+          .some(n => visits[n].uri.spec == spec && visits[n].test.hasBookmark);
+      }
+      info("Generating " + uri.spec + ", " + dbDate);
+      let visit = {
+        uri,
+        title: "visit " + i,
+        visitDate: dbDate,
+        test: {
+          // `visitDate`, as a Date
+          jsDate,
+          // `true` if we expect that the visit will be removed
+          toRemove: false,
+          // `true` if `onRow` informed of the removal of this visit
+          announcedByOnRow: false,
+          // `true` if there is a bookmark for this URI, i.e. of the page
+          // should not be entirely removed.
+          hasBookmark,
+        },
+      };
+      visits.push(visit);
+      if (hasOwnBookmark) {
+        info("Adding a bookmark to visit " + i);
+        await PlacesUtils.bookmarks.insert({
+          url: uri,
+          parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+          title: "test bookmark",
+        });
+        info("Bookmark added");
+      }
+    }
+
+    info("Adding visits");
+    await PlacesTestUtils.addVisits(visits);
+
+    info("Preparing filters");
+    let filter = {};
+    let beginIndex = 0;
+    let endIndex = visits.length - 1;
+    if ("begin" in options) {
+      let ms = Number(visits[options.begin].test.jsDate) - 1000;
+      filter.beginDate = new Date(ms);
+      beginIndex = options.begin;
+    }
+    if ("end" in options) {
+      let ms = Number(visits[options.end].test.jsDate) + 1000;
+      filter.endDate = new Date(ms);
+      endIndex = options.end;
+    }
+    if ("limit" in options) {
+      endIndex = beginIndex + options.limit - 1; // -1 because the start index is inclusive.
+      filter.limit = options.limit;
+    }
+    let removedItems = visits.slice(beginIndex);
+    endIndex -= beginIndex;
+    if (options.url) {
+      let rawURL = "";
+      switch (options.url) {
+        case 1:
+          filter.url = new URL(removedItems[0].uri.spec);
+          rawURL = filter.url.href;
+          break;
+        case 2:
+          filter.url = removedItems[0].uri;
+          rawURL = filter.url.spec;
+          break;
+        case 3:
+          filter.url = removedItems[0].uri.spec;
+          rawURL = filter.url;
+          break;
+      }
+      endIndex = Math.min(
+        endIndex,
+        removedItems.findIndex((v, index) => v.uri.spec != rawURL) - 1
+      );
+    }
+    removedItems.splice(endIndex + 1);
+    let remainingItems = visits.filter(v => !removedItems.includes(v));
+    for (let i = 0; i < removedItems.length; i++) {
+      let test = removedItems[i].test;
+      info("Marking visit " + (beginIndex + i) + " as expecting removal");
+      test.toRemove = true;
+      if (
+        test.hasBookmark ||
+        (options.url &&
+          remainingItems.some(v => v.uri.spec == removedItems[i].uri.spec))
+      ) {
+        rankingChangePromises.push(PromiseUtils.defer());
+      } else if (!options.url || i == 0) {
+        uriDeletePromises.set(removedItems[i].uri.spec, PromiseUtils.defer());
+      }
+    }
+
+    const placesEventListener = events => {
+      for (const event of events) {
+        switch (event.type) {
+          case "page-title-changed": {
+            this.deferred.reject(
+              "Unexpected page-title-changed event happens on " + event.url
+            );
+            break;
+          }
+          case "history-cleared": {
+            info("history-cleared");
+            this.deferred.reject("Unexpected history-cleared event happens");
+            break;
+          }
+          case "pages-rank-changed": {
+            info("pages-rank-changed");
+            for (const deferred of rankingChangePromises) {
+              deferred.resolve();
+            }
+            break;
+          }
+        }
+      }
+    };
+    PlacesObservers.addListener(
+      ["page-title-changed", "history-cleared", "pages-rank-changed"],
+      placesEventListener
+    );
+
+    let cbarg;
+    if (options.useCallback) {
+      info("Setting up callback");
+      cbarg = [
+        info => {
+          for (let visit of visits) {
+            info("Comparing " + info.date + " and " + visit.test.jsDate);
+            if (Math.abs(visit.test.jsDate - info.date) < 100) {
+              // Assume rounding errors
+              Assert.ok(
+                !visit.test.announcedByOnRow,
+                "This is the first time we announce the removal of this visit"
+              );
+              Assert.ok(
+                visit.test.toRemove,
+                "This is a visit we intended to remove"
+              );
+              visit.test.announcedByOnRow = true;
+              return;
+            }
+          }
+          Assert.ok(false, "Could not find the visit we attempt to remove");
+        },
+      ];
+    } else {
+      info("No callback");
+      cbarg = [];
+    }
+    let result = await PlacesUtils.history.removeVisitsByFilter(
+      filter,
+      ...cbarg
+    );
+
+    Assert.ok(result, "Removal succeeded");
+
+    // Make sure that we have eliminated exactly the entries we expected
+    // to eliminate.
+    for (let i = 0; i < visits.length; ++i) {
+      let visit = visits[i];
+      info("Controlling the results on visit " + i);
+      let remainingVisitsForURI = remainingItems.filter(
+        v => visit.uri.spec == v.uri.spec
+      ).length;
+      Assert.equal(
+        visits_in_database(visit.uri),
+        remainingVisitsForURI,
+        "Visit is still present iff expected"
+      );
+      if (options.useCallback) {
+        Assert.equal(
+          visit.test.toRemove,
+          visit.test.announcedByOnRow,
+          "Visit removal has been announced by onResult iff expected"
+        );
+      }
+      if (visit.test.hasBookmark || remainingVisitsForURI) {
+        Assert.notEqual(
+          page_in_database(visit.uri),
+          0,
+          "The page should still appear in the db"
+        );
+      } else {
+        Assert.equal(
+          page_in_database(visit.uri),
+          0,
+          "The page should have been removed from the db"
+        );
+      }
+    }
+
+    // Make sure that the observer has been called wherever applicable.
+    info("Checking URI delete promises.");
+    await Promise.all(Array.from(uriDeletePromises.values()));
+    info("Checking frecency change promises.");
+    await Promise.all(rankingChangePromises);
+    PlacesObservers.removeListener(
+      ["page-title-changed", "history-cleared", "pages-rank-changed"],
+      placesEventListener
+    );
+  };
+
+  let size = 20;
+  for (let range of [
+    { begin: 0 },
+    { end: 19 },
+    { begin: 0, end: 10 },
+    { begin: 3, end: 4 },
+    { begin: 5, end: 8, limit: 2 },
+    { begin: 10, end: 18, limit: 5 },
+  ]) {
+    for (let bookmarks of [[], [5, 6]]) {
+      let options = {
+        sampleSize: size,
+        bookmarks,
+      };
+      if ("begin" in range) {
+        options.begin = range.begin;
+      }
+      if ("end" in range) {
+        options.end = range.end;
+      }
+      if ("limit" in range) {
+        options.limit = range.limit;
+      }
+      await remover(options);
+      options.url = 1;
+      await remover(options);
+      options.url = 2;
+      await remover(options);
+      options.url = 3;
+      await remover(options);
+    }
+  }
+  await PlacesUtils.history.clear();
+});
+
+// Test the various error cases
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter(),
+    /TypeError: Expected a filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter("obviously, not a filter"),
+    /TypeError: Expected a filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({}),
+    /TypeError: Expected a non-empty filter/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ beginDate: "now" }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ beginDate: Date.now() }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeVisitsByFilter({ beginDate: new Date(NaN) }),
+    /TypeError: Expected a valid Date/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeVisitsByFilter(
+        { beginDate: new Date() },
+        "obviously, not a callback"
+      ),
+    /TypeError: Invalid function/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeVisitsByFilter({
+        beginDate: new Date(1000),
+        endDate: new Date(0),
+      }),
+    /TypeError: `beginDate` should be at least as old/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ limit: {} }),
+    /Expected a non-zero positive integer as a limit/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ limit: -1 }),
+    /Expected a non-zero positive integer as a limit/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ limit: 0.1 }),
+    /Expected a non-zero positive integer as a limit/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ limit: Infinity }),
+    /Expected a non-zero positive integer as a limit/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ url: {} }),
+    /Expected a valid URL for `url`/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ url: 0 }),
+    /Expected a valid URL for `url`/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeVisitsByFilter({
+        beginDate: new Date(1000),
+        endDate: new Date(0),
+      }),
+    /TypeError: `beginDate` should be at least as old/
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.removeVisitsByFilter({
+        beginDate: new Date(1000),
+        endDate: new Date(0),
+      }),
+    /TypeError: `beginDate` should be at least as old/
+  );
+  Assert.throws(
+    () => PlacesUtils.history.removeVisitsByFilter({ transition: -1 }),
+    /TypeError: `transition` should be valid/
+  );
+});
+
+add_task(async function test_orphans() {
+  let uri = NetUtil.newURI("http://moz.org/");
+  await PlacesTestUtils.addVisits({ uri });
+
+  PlacesUtils.favicons.setAndFetchFaviconForPage(
+    uri,
+    SMALLPNG_DATA_URI,
+    true,
+    PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+    null,
+    Services.scriptSecurityManager.getSystemPrincipal()
+  );
+  await PlacesUtils.history.update({
+    url: uri,
+    annotations: new Map([["test", "restval"]]),
+  });
+
+  await PlacesUtils.history.removeVisitsByFilter({
+    beginDate: new Date(1999, 9, 9, 9, 9),
+    endDate: new Date(),
+  });
+  Assert.ok(
+    !(await PlacesTestUtils.isPageInDB(uri)),
+    "Page should have been removed"
+  );
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`SELECT (SELECT count(*) FROM moz_annos) +
+                                      (SELECT count(*) FROM moz_icons) +
+                                      (SELECT count(*) FROM moz_pages_w_icons) +
+                                      (SELECT count(*) FROM moz_icons_to_pages) AS count`);
+  Assert.equal(rows[0].getResultByName("count"), 0, "Should not find orphans");
+});
diff --git a/toolkit/components/places/tests/history/test_sameUri_titleChanged.js b/toolkit/components/places/tests/history/test_sameUri_titleChanged.js
new file mode 100644
index 0000000000..87fb3f455c
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_sameUri_titleChanged.js
@@ -0,0 +1,54 @@
+// Test that repeated additions of the same URI to history, properly
+// update from_visit and notify titleChanged.
+
+add_task(async function test() {
+  let uri = "http://test.com/";
+
+  const promiseTitleChangedNotifications =
+    PlacesTestUtils.waitForNotification("page-title-changed");
+
+  // This repeats the url on purpose, don't merge it into a single place entry.
+  await PlacesTestUtils.addVisits([
+    { uri, title: "test" },
+    { uri, referrer: uri, title: "test2" },
+  ]);
+
+  const events = await promiseTitleChangedNotifications;
+  Assert.equal(events.length, 1, "Right number of title changed notified");
+  Assert.equal(events[0].url, uri, "Should notify the proper url");
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let query = PlacesUtils.history.getNewQuery();
+  query.uri = NetUtil.newURI(uri);
+  options.resultType = options.RESULTS_AS_VISIT;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  Assert.equal(root.childCount, 2);
+
+  let child = root.getChild(0);
+  Assert.equal(
+    child.visitType,
+    TRANSITION_LINK,
+    "Visit type should be TRANSITION_LINK"
+  );
+  Assert.equal(child.visitId, 1, "Visit ID should be 1");
+  Assert.equal(child.fromVisitId, -1, "Should have no referrer visit ID");
+  Assert.equal(child.title, "test2", "Should have the correct title");
+
+  child = root.getChild(1);
+  Assert.equal(
+    child.visitType,
+    TRANSITION_LINK,
+    "Visit type should be TRANSITION_LINK"
+  );
+  Assert.equal(child.visitId, 2, "Visit ID should be 2");
+  Assert.equal(
+    child.fromVisitId,
+    1,
+    "First visit should be the referring visit"
+  );
+  Assert.equal(child.title, "test2", "Should have the correct title");
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/history/test_update.js b/toolkit/components/places/tests/history/test_update.js
new file mode 100644
index 0000000000..d7beafd368
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_update.js
@@ -0,0 +1,700 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests for `History.update` as implemented in History.jsm
+
+"use strict";
+
+add_task(async function test_error_cases() {
+  Assert.throws(
+    () => PlacesUtils.history.update("not an object"),
+    /Error: PageInfo: Input should be a valid object/,
+    "passing a string as pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.update(null),
+    /Error: PageInfo: Input should be/,
+    "passing a null as pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        description: "Test description",
+      }),
+    /Error: PageInfo: The following properties were expected: url, guid/,
+    "not included a url or a guid should throw"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.update({ url: "not a valid url string" }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing an invalid url should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        description: 123,
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a non-string description in pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        guid: "invalid guid",
+        description: "Test description",
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a invalid guid in pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        previewImageURL: "not a valid url string",
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing an invlid preview image url in pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () => {
+      let imageName = "a-very-long-string".repeat(10000);
+      let previewImageURL = `http://valid.uri.com/${imageName}.png`;
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        previewImageURL,
+      });
+    },
+    /Error: PageInfo: Invalid value for property/,
+    "passing an oversized previewImageURL in pageInfo should throw an Error"
+  );
+  Assert.throws(
+    () => PlacesUtils.history.update({ url: "http://valid.uri.com" }),
+    /TypeError: pageInfo object must at least/,
+    "passing a pageInfo with neither description, previewImageURL, nor annotations should throw a TypeError"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        annotations: "asd",
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a pageInfo with incorrect annotations type should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        annotations: new Map(),
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a pageInfo with an empty annotations type should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        annotations: new Map([[1234, "value"]]),
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a pageInfo with an invalid key type should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        annotations: new Map([["test", ["myarray"]]]),
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a pageInfo with an invalid key type should throw an Error"
+  );
+  Assert.throws(
+    () =>
+      PlacesUtils.history.update({
+        url: "http://valid.uri.com",
+        annotations: new Map([["test", { anno: "value" }]]),
+      }),
+    /Error: PageInfo: Invalid value for property/,
+    "passing a pageInfo with an invalid key type should throw an Error"
+  );
+});
+
+add_task(async function test_description_change_saved() {
+  await PlacesUtils.history.clear();
+
+  let TEST_URL = "http://mozilla.org/test_description_change_saved";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(await PlacesTestUtils.isPageInDB(TEST_URL));
+
+  let description = "Test description";
+  await PlacesUtils.history.update({ url: TEST_URL, description });
+  let descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    { url: TEST_URL }
+  );
+  Assert.equal(
+    description,
+    descriptionInDB,
+    "description should be updated via URL as expected"
+  );
+
+  description = "";
+  await PlacesUtils.history.update({ url: TEST_URL, description });
+  descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    { url: TEST_URL }
+  );
+  Assert.strictEqual(
+    null,
+    descriptionInDB,
+    "an empty description should set it to null in the database"
+  );
+
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: TEST_URL,
+  });
+  description = "Test description";
+  await PlacesUtils.history.update({ url: TEST_URL, guid, description });
+  descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    { url: TEST_URL }
+  );
+  Assert.equal(
+    description,
+    descriptionInDB,
+    "description should be updated via GUID as expected"
+  );
+
+  description = "Test descipriton".repeat(1000);
+  await PlacesUtils.history.update({ url: TEST_URL, description });
+  descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    { url: TEST_URL }
+  );
+  Assert.ok(
+    !!descriptionInDB.length < description.length,
+    "a long description should be truncated"
+  );
+
+  description = null;
+  await PlacesUtils.history.update({ url: TEST_URL, description });
+  descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    { url: TEST_URL }
+  );
+  Assert.strictEqual(
+    description,
+    descriptionInDB,
+    "a null description should set it to null in the database"
+  );
+});
+
+add_task(async function test_siteName_change_saved() {
+  await PlacesUtils.history.clear();
+
+  let TEST_URL = "http://mozilla.org/test_siteName_change_saved";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(await PlacesTestUtils.isPageInDB(TEST_URL));
+
+  let siteName = "Test site name";
+  await PlacesUtils.history.update({ url: TEST_URL, siteName });
+  let siteNameInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "site_name",
+    { url: TEST_URL }
+  );
+  Assert.equal(
+    siteName,
+    siteNameInDB,
+    "siteName should be updated via URL as expected"
+  );
+
+  siteName = "";
+  await PlacesUtils.history.update({ url: TEST_URL, siteName });
+  siteNameInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "site_name",
+    { url: TEST_URL }
+  );
+  Assert.strictEqual(
+    null,
+    siteNameInDB,
+    "an empty siteName should set it to null in the database"
+  );
+
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: TEST_URL,
+  });
+  siteName = "Test site name";
+  await PlacesUtils.history.update({ url: TEST_URL, guid, siteName });
+  siteNameInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "site_name",
+    { url: TEST_URL }
+  );
+  Assert.equal(
+    siteName,
+    siteNameInDB,
+    "siteName should be updated via GUID as expected"
+  );
+
+  siteName = "Test site name".repeat(1000);
+  await PlacesUtils.history.update({ url: TEST_URL, siteName });
+  siteNameInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "site_name",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.ok(
+    !!siteNameInDB.length < siteName.length,
+    "a long siteName should be truncated"
+  );
+
+  siteName = null;
+  await PlacesUtils.history.update({ url: TEST_URL, siteName });
+  siteNameInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "site_name",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.strictEqual(
+    siteName,
+    siteNameInDB,
+    "a null siteName should set it to null in the database"
+  );
+});
+
+add_task(async function test_previewImageURL_change_saved() {
+  await PlacesUtils.history.clear();
+
+  let TEST_URL = "http://mozilla.org/test_previewImageURL_change_saved";
+  let IMAGE_URL = "http://mozilla.org/test_preview_image.png";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(await PlacesTestUtils.isPageInDB(TEST_URL));
+
+  let previewImageURL = IMAGE_URL;
+  await PlacesUtils.history.update({ url: TEST_URL, previewImageURL });
+  let previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(
+    previewImageURL,
+    previewImageURLInDB,
+    "previewImageURL should be updated via URL as expected"
+  );
+
+  previewImageURL = null;
+  await PlacesUtils.history.update({ url: TEST_URL, previewImageURL });
+  previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.strictEqual(
+    null,
+    previewImageURLInDB,
+    "a null previewImageURL should set it to null in the database"
+  );
+
+  let guid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: TEST_URL,
+  });
+  previewImageURL = IMAGE_URL;
+  await PlacesUtils.history.update({ guid, previewImageURL });
+  previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(
+    previewImageURL,
+    previewImageURLInDB,
+    "previewImageURL should be updated via GUID as expected"
+  );
+
+  previewImageURL = "";
+  await PlacesUtils.history.update({ url: TEST_URL, previewImageURL });
+  previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.strictEqual(
+    null,
+    previewImageURLInDB,
+    "an empty previewImageURL should set it to null in the database"
+  );
+});
+
+add_task(async function test_change_description_and_preview_saved() {
+  await PlacesUtils.history.clear();
+
+  let TEST_URL = "http://mozilla.org/test_change_both_saved";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(await PlacesTestUtils.isPageInDB(TEST_URL));
+
+  let description = "Test description";
+  let previewImageURL = "http://mozilla.org/test_preview_image.png";
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    description,
+    previewImageURL,
+  });
+  let descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    {
+      url: TEST_URL,
+    }
+  );
+  let previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(
+    description,
+    descriptionInDB,
+    "description should be updated via URL as expected"
+  );
+  Assert.equal(
+    previewImageURL,
+    previewImageURLInDB,
+    "previewImageURL should be updated via URL as expected"
+  );
+
+  // Update description should not touch other fields
+  description = null;
+  await PlacesUtils.history.update({ url: TEST_URL, description });
+  descriptionInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "description",
+    {
+      url: TEST_URL,
+    }
+  );
+  previewImageURLInDB = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "preview_image_url",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.strictEqual(
+    description,
+    descriptionInDB,
+    "description should be updated via URL as expected"
+  );
+  Assert.equal(
+    previewImageURL,
+    previewImageURLInDB,
+    "previewImageURL should not be updated"
+  );
+});
+
+/**
+ * Gets annotation information from the database for the specified URL and
+ * annotation name.
+ *
+ * @param {String} pageUrl The URL to search for.
+ * @param {String} annoName The name of the annotation to search for.
+ * @return {Array} An array of objects containing the annotations found.
+ */
+async function getAnnotationInfoFromDB(pageUrl, annoName) {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let rows = await db.execute(
+    `
+    SELECT a.content, a.flags, a.expiration, a.type FROM moz_anno_attributes n
+    JOIN moz_annos a ON n.id = a.anno_attribute_id
+    JOIN moz_places h ON h.id = a.place_id
+    WHERE h.url_hash = hash(:pageUrl) AND h.url = :pageUrl
+      AND n.name = :annoName
+  `,
+    { annoName, pageUrl }
+  );
+
+  let result = rows.map(row => {
+    return {
+      content: row.getResultByName("content"),
+      flags: row.getResultByName("flags"),
+      expiration: row.getResultByName("expiration"),
+      type: row.getResultByName("type"),
+    };
+  });
+
+  return result;
+}
+
+add_task(async function test_simple_change_annotations() {
+  await PlacesUtils.history.clear();
+
+  const TEST_URL = "http://mozilla.org/test_change_both_saved";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL),
+    "Should have inserted the page into the database."
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/annotation", "testContent"]]),
+  });
+
+  let pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    1,
+    "Should have one annotation for the page"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    "testContent",
+    "Should have the correct annotation"
+  );
+
+  let annotationInfo = await getAnnotationInfoFromDB(
+    TEST_URL,
+    "test/annotation"
+  );
+  Assert.deepEqual(
+    {
+      content: "testContent",
+      flags: 0,
+      type: PlacesUtils.history.ANNOTATION_TYPE_STRING,
+      expiration: PlacesUtils.history.ANNOTATION_EXPIRE_NEVER,
+    },
+    annotationInfo[0],
+    "Should have stored the correct annotation data in the db"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/annotation2", "testAnno"]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    2,
+    "Should have two annotations for the page"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    "testContent",
+    "Should have the correct value for the first annotation"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    "testAnno",
+    "Should have the correct value for the second annotation"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/annotation", 1234]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    2,
+    "Should still have two annotations for the page"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    1234,
+    "Should have the updated the first annotation value"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    "testAnno",
+    "Should have kept the value for the second annotation"
+  );
+
+  annotationInfo = await getAnnotationInfoFromDB(TEST_URL, "test/annotation");
+  Assert.deepEqual(
+    {
+      content: 1234,
+      flags: 0,
+      type: PlacesUtils.history.ANNOTATION_TYPE_INT64,
+      expiration: PlacesUtils.history.ANNOTATION_EXPIRE_NEVER,
+    },
+    annotationInfo[0],
+    "Should have updated the annotation data in the db"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/annotation", null]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    1,
+    "Should have removed only the first annotation"
+  );
+  Assert.strictEqual(
+    pageInfo.annotations.get("test/annotation"),
+    undefined,
+    "Should have removed only the first annotation"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    "testAnno",
+    "Should have kept the value for the second annotation"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([["test/annotation2", null]]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(pageInfo.annotations.size, 0, "Should have no annotations left");
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT * FROM moz_annos
+  `);
+  Assert.equal(rows.length, 0, "Should be no annotations left in the db");
+});
+
+add_task(async function test_change_multiple_annotations() {
+  await PlacesUtils.history.clear();
+
+  const TEST_URL = "http://mozilla.org/test_change_both_saved";
+  await PlacesTestUtils.addVisits(TEST_URL);
+  Assert.ok(
+    await PlacesTestUtils.isPageInDB(TEST_URL),
+    "Should have inserted the page into the database."
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([
+      ["test/annotation", "testContent"],
+      ["test/annotation2", "testAnno"],
+    ]),
+  });
+
+  let pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    2,
+    "Should have inserted the two annotations for the page."
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    "testContent",
+    "Should have the correct value for the first annotation"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    "testAnno",
+    "Should have the correct value for the second annotation"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([
+      ["test/annotation", 123456],
+      ["test/annotation2", 135246],
+    ]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.size,
+    2,
+    "Should have two annotations for the page"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation"),
+    123456,
+    "Should have the correct value for the first annotation"
+  );
+  Assert.equal(
+    pageInfo.annotations.get("test/annotation2"),
+    135246,
+    "Should have the correct value for the second annotation"
+  );
+
+  await PlacesUtils.history.update({
+    url: TEST_URL,
+    annotations: new Map([
+      ["test/annotation", null],
+      ["test/annotation2", null],
+    ]),
+  });
+
+  pageInfo = await PlacesUtils.history.fetch(TEST_URL, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(pageInfo.annotations.size, 0, "Should have no annotations left");
+});
+
+add_task(async function test_annotations_nonexisting_page() {
+  info("Adding annotations to a non existing page should be silent");
+  await PlacesUtils.history.update({
+    url: "http://nonexisting.moz/",
+    annotations: new Map([["test/annotation", null]]),
+  });
+});
+
+add_task(async function test_annotations_nonexisting_page() {
+  info("Adding annotations to a non existing page should be silent");
+  await PlacesUtils.history.update({
+    url: "http://nonexisting.moz/",
+    annotations: new Map([["test/annotation", null]]),
+  });
+});
diff --git a/toolkit/components/places/tests/history/test_updatePlaces_embed.js b/toolkit/components/places/tests/history/test_updatePlaces_embed.js
new file mode 100644
index 0000000000..a2831f2f58
--- /dev/null
+++ b/toolkit/components/places/tests/history/test_updatePlaces_embed.js
@@ -0,0 +1,81 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that updatePlaces properly handled callbacks for embed visits.
+
+"use strict";
+
+XPCOMUtils.defineLazyServiceGetter(
+  this,
+  "asyncHistory",
+  "@mozilla.org/browser/history;1",
+  "mozIAsyncHistory"
+);
+
+add_task(async function test_embed_visit() {
+  let place = {
+    uri: NetUtil.newURI("http://places.test/"),
+    visits: [
+      {
+        transitionType: PlacesUtils.history.TRANSITIONS.EMBED,
+        visitDate: PlacesUtils.toPRTime(new Date()),
+      },
+    ],
+  };
+  let errors = 0;
+  let results = 0;
+  let updated = await new Promise(resolve => {
+    asyncHistory.updatePlaces(place, {
+      ignoreErrors: true,
+      ignoreResults: true,
+      handleError(aResultCode, aPlace) {
+        errors++;
+      },
+      handleResult(aPlace) {
+        results++;
+      },
+      handleCompletion(resultCount) {
+        resolve(resultCount);
+      },
+    });
+  });
+  Assert.equal(errors, 0, "There should be no error callback");
+  Assert.equal(results, 0, "There should be no result callback");
+  Assert.equal(updated, 1, "The visit should have been added");
+});
+
+add_task(async function test_misc_visits() {
+  let place = {
+    uri: NetUtil.newURI("http://places.test/"),
+    visits: [
+      {
+        transitionType: PlacesUtils.history.TRANSITIONS.EMBED,
+        visitDate: PlacesUtils.toPRTime(new Date()),
+      },
+      {
+        transitionType: PlacesUtils.history.TRANSITIONS.LINK,
+        visitDate: PlacesUtils.toPRTime(new Date()),
+      },
+    ],
+  };
+  let errors = 0;
+  let results = 0;
+  let updated = await new Promise(resolve => {
+    asyncHistory.updatePlaces(place, {
+      ignoreErrors: true,
+      ignoreResults: true,
+      handleError(aResultCode, aPlace) {
+        errors++;
+      },
+      handleResult(aPlace) {
+        results++;
+      },
+      handleCompletion(resultCount) {
+        resolve(resultCount);
+      },
+    });
+  });
+  Assert.equal(errors, 0, "There should be no error callback");
+  Assert.equal(results, 0, "There should be no result callback");
+  Assert.equal(updated, 2, "The visit should have been added");
+});
diff --git a/toolkit/components/places/tests/history/xpcshell.ini b/toolkit/components/places/tests/history/xpcshell.ini
new file mode 100644
index 0000000000..b4a017e15d
--- /dev/null
+++ b/toolkit/components/places/tests/history/xpcshell.ini
@@ -0,0 +1,20 @@
+[DEFAULT]
+head = head_history.js
+
+[test_async_history_api.js]
+[test_bookmark_unhide.js]
+[test_fetch.js]
+[test_fetchAnnotatedPages.js]
+[test_fetchMany.js]
+[test_hasVisits.js]
+[test_insert.js]
+[test_insert_null_title.js]
+[test_insertMany.js]
+[test_remove.js]
+[test_removeMany.js]
+[test_removeVisits.js]
+[test_removeByFilter.js]
+[test_removeVisitsByFilter.js]
+[test_sameUri_titleChanged.js]
+[test_update.js]
+[test_updatePlaces_embed.js]
diff --git a/toolkit/components/places/tests/legacy/head_legacy.js b/toolkit/components/places/tests/legacy/head_legacy.js
new file mode 100644
index 0000000000..06e7fda560
--- /dev/null
+++ b/toolkit/components/places/tests/legacy/head_legacy.js
@@ -0,0 +1,14 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
diff --git a/toolkit/components/places/tests/legacy/test_bookmarks.js b/toolkit/components/places/tests/legacy/test_bookmarks.js
new file mode 100644
index 0000000000..3f331b56cb
--- /dev/null
+++ b/toolkit/components/places/tests/legacy/test_bookmarks.js
@@ -0,0 +1,519 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var bs = PlacesUtils.bookmarks;
+var hs = PlacesUtils.history;
+var os = PlacesUtils.observers;
+
+var bookmarksObserver = {
+  handlePlacesEvents(events) {
+    Assert.equal(events.length, 1);
+    let event = events[0];
+    switch (event.type) {
+      case "bookmark-added":
+        bookmarksObserver._itemAddedId = event.id;
+        bookmarksObserver._itemAddedParent = event.parentId;
+        bookmarksObserver._itemAddedIndex = event.index;
+        bookmarksObserver._itemAddedURI = event.url
+          ? Services.io.newURI(event.url)
+          : null;
+        bookmarksObserver._itemAddedTitle = event.title;
+
+        // Ensure that we've created a guid for this item.
+        let stmt = DBConn().createStatement(
+          `SELECT guid
+           FROM moz_bookmarks
+           WHERE id = :item_id`
+        );
+        stmt.params.item_id = event.id;
+        Assert.ok(stmt.executeStep());
+        Assert.ok(!stmt.getIsNull(0));
+        do_check_valid_places_guid(stmt.row.guid);
+        Assert.equal(stmt.row.guid, event.guid);
+        stmt.finalize();
+        break;
+      case "bookmark-removed":
+        bookmarksObserver._itemRemovedId = event.id;
+        bookmarksObserver._itemRemovedFolder = event.parentId;
+        bookmarksObserver._itemRemovedIndex = event.index;
+        break;
+      case "bookmark-title-changed":
+        bookmarksObserver._itemTitleChangedId = event.id;
+        bookmarksObserver._itemTitleChangedTitle = event.title;
+        break;
+    }
+  },
+};
+
+var root;
+// Index at which items should begin.
+var bmStartIndex = 0;
+
+add_task(async function setup() {
+  // Get bookmarks menu folder id.
+  root = await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.menuGuid);
+});
+
+add_task(async function test_bookmarks() {
+  os.addListener(
+    ["bookmark-added", "bookmark-removed", "bookmark-title-changed"],
+    bookmarksObserver.handlePlacesEvents
+  );
+
+  // test special folders
+  Assert.ok(bs.tagsFolder > 0);
+
+  // create a folder to hold all the tests
+  // this makes the tests more tolerant of changes to default_places.html
+  let testRoot = bs.createFolder(
+    root,
+    "places bookmarks xpcshell tests",
+    bs.DEFAULT_INDEX
+  );
+  let testRootGuid = await PlacesUtils.promiseItemGuid(testRoot);
+  Assert.equal(bookmarksObserver._itemAddedId, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedParent, root);
+  Assert.equal(bookmarksObserver._itemAddedIndex, bmStartIndex);
+  Assert.equal(bookmarksObserver._itemAddedURI, null);
+  let testStartIndex = 0;
+
+  // insert a bookmark.
+  // the time before we insert, in microseconds
+  let beforeInsert = Date.now() * 1000;
+  Assert.ok(beforeInsert > 0);
+
+  let newId = bs.insertBookmark(
+    testRoot,
+    uri("http://google.com/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId);
+  Assert.equal(bookmarksObserver._itemAddedParent, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedIndex, testStartIndex);
+  Assert.ok(bookmarksObserver._itemAddedURI.equals(uri("http://google.com/")));
+
+  // after just inserting, modified should not be set
+  let lastModified = PlacesUtils.toPRTime(
+    (
+      await PlacesUtils.bookmarks.fetch(
+        await PlacesUtils.promiseItemGuid(newId)
+      )
+    ).lastModified
+  );
+
+  // The time before we set the title, in microseconds.
+  let beforeSetTitle = Date.now() * 1000;
+  Assert.ok(beforeSetTitle >= beforeInsert);
+
+  // Workaround possible VM timers issues moving lastModified and dateAdded
+  // to the past.
+  lastModified -= 1000;
+  bs.setItemLastModified(newId, lastModified);
+
+  // set bookmark title
+  bs.setItemTitle(newId, "Google");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "Google");
+
+  // check lastModified after we set the title
+  let lastModified2 = PlacesUtils.toPRTime(
+    (
+      await PlacesUtils.bookmarks.fetch(
+        await PlacesUtils.promiseItemGuid(newId)
+      )
+    ).lastModified
+  );
+  info("test setItemTitle");
+  info("beforeSetTitle = " + beforeSetTitle);
+  info("lastModified = " + lastModified);
+  info("lastModified2 = " + lastModified2);
+  Assert.ok(is_time_ordered(lastModified, lastModified2));
+
+  // get item title
+  let title = bs.getItemTitle(newId);
+  Assert.equal(title, "Google");
+
+  // get item title bad input
+  try {
+    bs.getItemTitle(-3);
+    do_throw("getItemTitle accepted bad input");
+  } catch (ex) {}
+
+  // create a folder at a specific index
+  let workFolder = bs.createFolder(testRoot, "Work", 0);
+  Assert.equal(bookmarksObserver._itemAddedId, workFolder);
+  Assert.equal(bookmarksObserver._itemAddedParent, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 0);
+  Assert.equal(bookmarksObserver._itemAddedURI, null);
+
+  Assert.equal(bs.getItemTitle(workFolder), "Work");
+  bs.setItemTitle(workFolder, "Work #");
+  Assert.equal(bs.getItemTitle(workFolder), "Work #");
+
+  // add item into subfolder, specifying index
+  let newId2 = bs.insertBookmark(
+    workFolder,
+    uri("http://developer.mozilla.org/"),
+    0,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId2);
+  Assert.equal(bookmarksObserver._itemAddedParent, workFolder);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 0);
+
+  // change item
+  bs.setItemTitle(newId2, "DevMo");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId2);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "DevMo");
+
+  // insert item into subfolder
+  let newId3 = bs.insertBookmark(
+    workFolder,
+    uri("http://msdn.microsoft.com/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId3);
+  Assert.equal(bookmarksObserver._itemAddedParent, workFolder);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 1);
+
+  // change item
+  bs.setItemTitle(newId3, "MSDN");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId3);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "MSDN");
+
+  // remove item
+  bs.removeItem(newId2);
+  Assert.equal(bookmarksObserver._itemRemovedId, newId2);
+  Assert.equal(bookmarksObserver._itemRemovedFolder, workFolder);
+  Assert.equal(bookmarksObserver._itemRemovedIndex, 0);
+
+  // insert item into subfolder
+  let newId4 = bs.insertBookmark(
+    workFolder,
+    uri("http://developer.mozilla.org/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId4);
+  Assert.equal(bookmarksObserver._itemAddedParent, workFolder);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 1);
+
+  // create folder
+  let homeFolder = bs.createFolder(testRoot, "Home", bs.DEFAULT_INDEX);
+  Assert.equal(bookmarksObserver._itemAddedId, homeFolder);
+  Assert.equal(bookmarksObserver._itemAddedParent, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 2);
+
+  // insert item
+  let newId5 = bs.insertBookmark(
+    homeFolder,
+    uri("http://espn.com/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId5);
+  Assert.equal(bookmarksObserver._itemAddedParent, homeFolder);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 0);
+
+  // change item
+  bs.setItemTitle(newId5, "ESPN");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId5);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "ESPN");
+
+  // insert query item
+  let uri6 = uri(
+    "place:domain=google.com&type=" +
+      Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY
+  );
+  let newId6 = bs.insertBookmark(testRoot, uri6, bs.DEFAULT_INDEX, "");
+  Assert.equal(bookmarksObserver._itemAddedParent, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 3);
+
+  // change item
+  bs.setItemTitle(newId6, "Google Sites");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId6);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "Google Sites");
+
+  // test bookmark id in query output
+  try {
+    let options = hs.getNewQueryOptions();
+    let query = hs.getNewQuery();
+    query.setParents([testRootGuid]);
+    let result = hs.executeQuery(query, options);
+    let rootNode = result.root;
+    rootNode.containerOpen = true;
+    let cc = rootNode.childCount;
+    info("bookmark itemId test: CC = " + cc);
+    Assert.ok(cc > 0);
+    for (let i = 0; i < cc; ++i) {
+      let node = rootNode.getChild(i);
+      if (
+        node.type == node.RESULT_TYPE_FOLDER ||
+        node.type == node.RESULT_TYPE_URI ||
+        node.type == node.RESULT_TYPE_SEPARATOR ||
+        node.type == node.RESULT_TYPE_QUERY
+      ) {
+        Assert.ok(node.itemId > 0);
+      } else {
+        Assert.equal(node.itemId, -1);
+      }
+    }
+    rootNode.containerOpen = false;
+  } catch (ex) {
+    do_throw("bookmarks query: " + ex);
+  }
+
+  // test that multiple bookmarks with same URI show up right in bookmark
+  // folder queries, todo: also to do for complex folder queries
+  try {
+    // test uri
+    let mURI = uri("http://multiple.uris.in.query");
+
+    let testFolder = bs.createFolder(testRoot, "test Folder", bs.DEFAULT_INDEX);
+    let testFolderGuid = await PlacesUtils.promiseItemGuid(testFolder);
+    // add 2 bookmarks
+    bs.insertBookmark(testFolder, mURI, bs.DEFAULT_INDEX, "title 1");
+    bs.insertBookmark(testFolder, mURI, bs.DEFAULT_INDEX, "title 2");
+
+    // query
+    let options = hs.getNewQueryOptions();
+    let query = hs.getNewQuery();
+    query.setParents([testFolderGuid]);
+    let result = hs.executeQuery(query, options);
+    let rootNode = result.root;
+    rootNode.containerOpen = true;
+    let cc = rootNode.childCount;
+    Assert.equal(cc, 2);
+    Assert.equal(rootNode.getChild(0).title, "title 1");
+    Assert.equal(rootNode.getChild(1).title, "title 2");
+    rootNode.containerOpen = false;
+  } catch (ex) {
+    do_throw("bookmarks query: " + ex);
+  }
+
+  // test change bookmark uri
+  let newId10 = bs.insertBookmark(
+    testRoot,
+    uri("http://foo10.com/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+
+  // Workaround possible VM timers issues moving lastModified and dateAdded
+  // to the past.
+  lastModified -= 1000;
+  bs.setItemLastModified(newId10, lastModified);
+
+  // insert a bookmark with title ZZZXXXYYY and then search for it.
+  // this test confirms that we can find bookmarks that we haven't visited
+  // (which are "hidden") and that we can find by title.
+  // see bug #369887 for more details
+  let newId13 = bs.insertBookmark(
+    testRoot,
+    uri("http://foobarcheese.com/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  Assert.equal(bookmarksObserver._itemAddedId, newId13);
+  Assert.equal(bookmarksObserver._itemAddedParent, testRoot);
+  Assert.equal(bookmarksObserver._itemAddedIndex, 6);
+
+  // set bookmark title
+  bs.setItemTitle(newId13, "ZZZXXXYYY");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId13);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, "ZZZXXXYYY");
+
+  // test search on bookmark title ZZZXXXYYY
+  try {
+    let options = hs.getNewQueryOptions();
+    options.excludeQueries = 1;
+    options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+    let query = hs.getNewQuery();
+    query.searchTerms = "ZZZXXXYYY";
+    let result = hs.executeQuery(query, options);
+    let rootNode = result.root;
+    rootNode.containerOpen = true;
+    let cc = rootNode.childCount;
+    Assert.equal(cc, 1);
+    let node = rootNode.getChild(0);
+    Assert.equal(node.title, "ZZZXXXYYY");
+    Assert.ok(node.itemId > 0);
+    rootNode.containerOpen = false;
+  } catch (ex) {
+    do_throw("bookmarks query: " + ex);
+  }
+
+  // test dateAdded and lastModified properties
+  // for a search query
+  try {
+    let options = hs.getNewQueryOptions();
+    options.excludeQueries = 1;
+    options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+    let query = hs.getNewQuery();
+    query.searchTerms = "ZZZXXXYYY";
+    let result = hs.executeQuery(query, options);
+    let rootNode = result.root;
+    rootNode.containerOpen = true;
+    let cc = rootNode.childCount;
+    Assert.equal(cc, 1);
+    let node = rootNode.getChild(0);
+
+    Assert.equal(typeof node.dateAdded, "number");
+    Assert.ok(node.dateAdded > 0);
+
+    Assert.equal(typeof node.lastModified, "number");
+    Assert.ok(node.lastModified > 0);
+
+    rootNode.containerOpen = false;
+  } catch (ex) {
+    do_throw("bookmarks query: " + ex);
+  }
+
+  // test dateAdded and lastModified properties
+  // for a folder query
+  try {
+    let options = hs.getNewQueryOptions();
+    let query = hs.getNewQuery();
+    query.setParents([testRootGuid]);
+    let result = hs.executeQuery(query, options);
+    let rootNode = result.root;
+    rootNode.containerOpen = true;
+    let cc = rootNode.childCount;
+    Assert.ok(cc > 0);
+    for (let i = 0; i < cc; i++) {
+      let node = rootNode.getChild(i);
+
+      if (node.type == node.RESULT_TYPE_URI) {
+        Assert.equal(typeof node.dateAdded, "number");
+        Assert.ok(node.dateAdded > 0);
+
+        Assert.equal(typeof node.lastModified, "number");
+        Assert.ok(node.lastModified > 0);
+        break;
+      }
+    }
+    rootNode.containerOpen = false;
+  } catch (ex) {
+    do_throw("bookmarks query: " + ex);
+  }
+
+  // check setItemLastModified()
+  let newId14 = bs.insertBookmark(
+    testRoot,
+    uri("http://bar.tld/"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  bs.setItemLastModified(newId14, 1234000000000000);
+  let fakeLastModified = PlacesUtils.toPRTime(
+    (
+      await PlacesUtils.bookmarks.fetch(
+        await PlacesUtils.promiseItemGuid(newId14)
+      )
+    ).lastModified
+  );
+  Assert.equal(fakeLastModified, 1234000000000000);
+
+  // bug 378820
+  let uri1 = uri("http://foo.tld/a");
+  bs.insertBookmark(testRoot, uri1, bs.DEFAULT_INDEX, "");
+  await PlacesTestUtils.addVisits(uri1);
+
+  // bug 646993 - test bookmark titles longer than the maximum allowed length
+  let title15 = Array(TITLE_LENGTH_MAX + 5).join("X");
+  let title15expected = title15.substring(0, TITLE_LENGTH_MAX);
+  let newId15 = bs.insertBookmark(
+    testRoot,
+    uri("http://evil.com/"),
+    bs.DEFAULT_INDEX,
+    title15
+  );
+
+  Assert.equal(bs.getItemTitle(newId15).length, title15expected.length);
+  Assert.equal(bookmarksObserver._itemAddedTitle, title15expected);
+  // test title length after updates
+  bs.setItemTitle(newId15, title15 + " updated");
+  Assert.equal(bs.getItemTitle(newId15).length, title15expected.length);
+  Assert.equal(bookmarksObserver._itemTitleChangedId, newId15);
+  Assert.equal(bookmarksObserver._itemTitleChangedTitle, title15expected);
+
+  await testSimpleFolderResult();
+});
+
+async function testSimpleFolderResult() {
+  // the time before we create a folder, in microseconds
+  // Workaround possible VM timers issues subtracting 1us.
+  let beforeCreate = Date.now() * 1000 - 1;
+  Assert.ok(beforeCreate > 0);
+
+  // create a folder
+  let parent = bs.createFolder(root, "test", bs.DEFAULT_INDEX);
+  let parentGuid = await PlacesUtils.promiseItemGuid(parent);
+
+  // the time before we insert, in microseconds
+  // Workaround possible VM timers issues subtracting 1ms.
+  let beforeInsert = Date.now() * 1000 - 1;
+  Assert.ok(beforeInsert > 0);
+
+  // re-set item title separately so can test nodes' last modified
+  let item = bs.insertBookmark(
+    parent,
+    uri("about:blank"),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  bs.setItemTitle(item, "test bookmark");
+
+  // see above
+  let folder = bs.createFolder(parent, "test folder", bs.DEFAULT_INDEX);
+  bs.setItemTitle(folder, "test folder");
+
+  let longName = Array(TITLE_LENGTH_MAX + 5).join("A");
+  let folderLongName = bs.createFolder(parent, longName, bs.DEFAULT_INDEX);
+  Assert.equal(
+    bookmarksObserver._itemAddedTitle,
+    longName.substring(0, TITLE_LENGTH_MAX)
+  );
+
+  let options = hs.getNewQueryOptions();
+  let query = hs.getNewQuery();
+  query.setParents([parentGuid]);
+  let result = hs.executeQuery(query, options);
+  let rootNode = result.root;
+  rootNode.containerOpen = true;
+  Assert.equal(rootNode.childCount, 3);
+
+  let node = rootNode.getChild(0);
+  Assert.equal(node.itemId, item);
+  Assert.ok(node.dateAdded > 0);
+  Assert.ok(node.lastModified > 0);
+  Assert.equal(node.title, "test bookmark");
+  node = rootNode.getChild(1);
+  Assert.equal(node.itemId, folder);
+  Assert.equal(node.title, "test folder");
+  Assert.ok(node.dateAdded > 0);
+  Assert.ok(node.lastModified > 0);
+  node = rootNode.getChild(2);
+  Assert.equal(node.itemId, folderLongName);
+  Assert.equal(node.title, longName.substring(0, TITLE_LENGTH_MAX));
+  Assert.ok(node.dateAdded > 0);
+  Assert.ok(node.lastModified > 0);
+
+  // update with another long title
+  bs.setItemTitle(folderLongName, longName + " updated");
+  Assert.equal(bookmarksObserver._itemTitleChangedId, folderLongName);
+  Assert.equal(
+    bookmarksObserver._itemTitleChangedTitle,
+    longName.substring(0, TITLE_LENGTH_MAX)
+  );
+
+  node = rootNode.getChild(2);
+  Assert.equal(node.title, longName.substring(0, TITLE_LENGTH_MAX));
+
+  rootNode.containerOpen = false;
+}
diff --git a/toolkit/components/places/tests/legacy/test_bookmarks_setNullTitle.js b/toolkit/components/places/tests/legacy/test_bookmarks_setNullTitle.js
new file mode 100644
index 0000000000..ff224c3402
--- /dev/null
+++ b/toolkit/components/places/tests/legacy/test_bookmarks_setNullTitle.js
@@ -0,0 +1,50 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Both setItemTitle and insertBookmark should default to the empty string
+ * for null titles.
+ */
+
+const bs = Cc["@mozilla.org/browser/nav-bookmarks-service;1"].getService(
+  Ci.nsINavBookmarksService
+);
+
+const TEST_URL = "http://www.mozilla.org";
+
+function run_test() {
+  // Insert a bookmark with an empty title.
+  var itemId = bs.insertBookmark(
+    bs.tagsFolder,
+    uri(TEST_URL),
+    bs.DEFAULT_INDEX,
+    ""
+  );
+  // Check returned title is an empty string.
+  Assert.equal(bs.getItemTitle(itemId), "");
+  // Set title to null.
+  bs.setItemTitle(itemId, null);
+  // Check returned title defaults to an empty string.
+  Assert.equal(bs.getItemTitle(itemId), "");
+  // Cleanup.
+  bs.removeItem(itemId);
+
+  // Insert a bookmark with a null title.
+  itemId = bs.insertBookmark(
+    bs.tagsFolder,
+    uri(TEST_URL),
+    bs.DEFAULT_INDEX,
+    null
+  );
+  // Check returned title defaults to an empty string.
+  Assert.equal(bs.getItemTitle(itemId), "");
+  // Set title to an empty string.
+  bs.setItemTitle(itemId, "");
+  // Check returned title is an empty string.
+  Assert.equal(bs.getItemTitle(itemId), "");
+  // Cleanup.
+  bs.removeItem(itemId);
+}
diff --git a/toolkit/components/places/tests/legacy/test_protectRoots.js b/toolkit/components/places/tests/legacy/test_protectRoots.js
new file mode 100644
index 0000000000..7c3c9d31dc
--- /dev/null
+++ b/toolkit/components/places/tests/legacy/test_protectRoots.js
@@ -0,0 +1,21 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async () => {
+  const ROOTS = [
+    PlacesUtils.bookmarks.rootGuid,
+    ...PlacesUtils.bookmarks.userContentRoots,
+    PlacesUtils.bookmarks.tagsGuid,
+  ];
+
+  for (let guid of ROOTS) {
+    Assert.ok(PlacesUtils.isRootItem(guid));
+
+    let id = await PlacesUtils.promiseItemId(guid);
+
+    try {
+      PlacesUtils.bookmarks.removeItem(id);
+      do_throw("Trying to remove a root should throw");
+    } catch (ex) {}
+  }
+});
diff --git a/toolkit/components/places/tests/legacy/xpcshell.ini b/toolkit/components/places/tests/legacy/xpcshell.ini
new file mode 100644
index 0000000000..54c9c6d3f5
--- /dev/null
+++ b/toolkit/components/places/tests/legacy/xpcshell.ini
@@ -0,0 +1,10 @@
+# This directory is for tests for the legacy, sync APIs as somewhere to put them
+# until we remove the APIs themselves.
+
+[DEFAULT]
+head = head_legacy.js
+firefox-appdir = browser
+
+[test_bookmarks.js]
+[test_bookmarks_setNullTitle.js]
+[test_protectRoots.js]
diff --git a/toolkit/components/places/tests/maintenance/corruptDB.sqlite b/toolkit/components/places/tests/maintenance/corruptDB.sqlite
new file mode 100644
index 0000000000..b234246cac
Binary files /dev/null and b/toolkit/components/places/tests/maintenance/corruptDB.sqlite differ
diff --git a/toolkit/components/places/tests/maintenance/corruptPayload.sqlite b/toolkit/components/places/tests/maintenance/corruptPayload.sqlite
new file mode 100644
index 0000000000..16717bda80
Binary files /dev/null and b/toolkit/components/places/tests/maintenance/corruptPayload.sqlite differ
diff --git a/toolkit/components/places/tests/maintenance/head.js b/toolkit/components/places/tests/maintenance/head.js
new file mode 100644
index 0000000000..3117ab323d
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/head.js
@@ -0,0 +1,119 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+ChromeUtils.defineESModuleGetters(this, {
+  PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.sys.mjs",
+});
+
+async function createCorruptDb(filename) {
+  let path = PathUtils.join(PathUtils.profileDir, filename);
+  await IOUtils.remove(path, { ignoreAbsent: true });
+  // Create a corrupt database.
+  let dir = do_get_cwd().path;
+  let src = PathUtils.join(dir, "corruptDB.sqlite");
+  await IOUtils.copy(src, path);
+}
+
+/**
+ * Used in _replaceOnStartup_ tests as common test code. It checks whether we
+ * are properly cloning or replacing a corrupt database.
+ *
+ * @param {string[]} src
+ *        Array of strings which form a path to a test database, relative to
+ *        the parent of this test folder.
+ * @param {string} filename
+ *        Database file name
+ * @param {boolean} shouldClone
+ *        Whether we expect the database to be cloned
+ * @param {boolean} dbStatus
+ *        The expected final database status
+ */
+async function test_database_replacement(src, filename, shouldClone, dbStatus) {
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("places.database.cloneOnCorruption");
+  });
+  Services.prefs.setBoolPref("places.database.cloneOnCorruption", shouldClone);
+
+  // Only the main database file (places.sqlite) will be cloned, because
+  // attached databased would break due to OS file lockings.
+  let willClone = shouldClone && filename == DB_FILENAME;
+
+  // Ensure that our databases don't exist yet.
+  let dest = PathUtils.join(PathUtils.profileDir, filename);
+  Assert.ok(
+    !(await IOUtils.exists(dest)),
+    `"${filename} should not exist initially`
+  );
+  let corrupt = PathUtils.join(PathUtils.profileDir, `${filename}.corrupt`);
+  Assert.ok(
+    !(await IOUtils.exists(corrupt)),
+    `${filename}.corrupt should not exist initially`
+  );
+
+  let dir = PathUtils.parent(do_get_cwd().path);
+  src = PathUtils.join(dir, ...src);
+  await IOUtils.copy(src, dest);
+
+  // Create some unique stuff to check later.
+  let db = await Sqlite.openConnection({ path: dest });
+  await db.execute(`CREATE TABLE moz_cloned (id INTEGER PRIMARY KEY)`);
+  await db.execute(`CREATE TABLE not_cloned (id INTEGER PRIMARY KEY)`);
+  await db.execute(`DELETE FROM moz_cloned`); // Shouldn't throw.
+  await db.close();
+
+  // Open the database with Places.
+  Services.prefs.setCharPref(
+    "places.database.replaceDatabaseOnStartup",
+    filename
+  );
+  Assert.equal(PlacesUtils.history.databaseStatus, dbStatus);
+
+  Assert.ok(await IOUtils.exists(dest), "The database should exist");
+
+  // Check the new database still contains our special data.
+  db = await Sqlite.openConnection({ path: dest });
+  if (willClone) {
+    await db.execute(`DELETE FROM moz_cloned`); // Shouldn't throw.
+  }
+
+  // Check the new database is really a new one.
+  await Assert.rejects(
+    db.execute(`DELETE FROM not_cloned`),
+    /no such table/,
+    "The database should have been replaced"
+  );
+  await db.close();
+
+  if (willClone) {
+    Assert.ok(
+      !(await IOUtils.exists(corrupt)),
+      "The corrupt db should not exist"
+    );
+    Assert.ok(
+      !(await IOUtils.exists(corrupt + "-wal")),
+      "The corrupt db wal should not exist"
+    );
+    Assert.ok(
+      !(await IOUtils.exists(corrupt + "-shm")),
+      "The corrupt db shm should not exist"
+    );
+  } else {
+    Assert.ok(await IOUtils.exists(corrupt), "The corrupt db should exist");
+  }
+
+  Assert.equal(
+    Services.prefs.getCharPref("places.database.replaceDatabaseOnStartup", ""),
+    "",
+    "The replaceDatabaseOnStartup pref should have been unset"
+  );
+}
diff --git a/toolkit/components/places/tests/maintenance/test_corrupt_favicons.js b/toolkit/components/places/tests/maintenance/test_corrupt_favicons.js
new file mode 100644
index 0000000000..2021428a62
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_corrupt_favicons.js
@@ -0,0 +1,16 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a corrupt favicons file
+// that can't be opened.
+
+add_task(async function () {
+  await createCorruptDb("favicons.sqlite");
+
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CREATE
+  );
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute("SELECT * FROM moz_icons"); // Should not fail.
+});
diff --git a/toolkit/components/places/tests/maintenance/test_corrupt_favicons_schema.js b/toolkit/components/places/tests/maintenance/test_corrupt_favicons_schema.js
new file mode 100644
index 0000000000..299bbca65d
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_corrupt_favicons_schema.js
@@ -0,0 +1,23 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a corrupt places schema.
+
+add_task(async function () {
+  let path = await setupPlacesDatabase(["migration", "favicons_v41.sqlite"]);
+
+  // Ensure the database will go through a migration that depends on moz_places
+  // and break the schema by dropping that table.
+  let db = await Sqlite.openConnection({ path });
+  await db.setSchemaVersion(38);
+  await db.execute("DROP TABLE moz_icons");
+  await db.close();
+
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CORRUPT
+  );
+  db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute("SELECT 1 FROM moz_icons");
+  Assert.equal(rows.length, 0, "Found no icons");
+});
diff --git a/toolkit/components/places/tests/maintenance/test_corrupt_places_schema.js b/toolkit/components/places/tests/maintenance/test_corrupt_places_schema.js
new file mode 100644
index 0000000000..9af7863ca2
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_corrupt_places_schema.js
@@ -0,0 +1,22 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a corrupt places schema.
+
+add_task(async function () {
+  let path = await setupPlacesDatabase(["migration", "places_v43.sqlite"]);
+
+  // Ensure the database will go through a migration that depends on moz_places
+  // and break the schema by dropping that table.
+  let db = await Sqlite.openConnection({ path });
+  await db.setSchemaVersion(43);
+  await db.execute("DROP TABLE moz_places");
+  await db.close();
+
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CORRUPT
+  );
+  db = await PlacesUtils.promiseDBConnection();
+  await db.execute("SELECT * FROM moz_places LIMIT 1"); // Should not fail.
+});
diff --git a/toolkit/components/places/tests/maintenance/test_corrupt_telemetry.js b/toolkit/components/places/tests/maintenance/test_corrupt_telemetry.js
new file mode 100644
index 0000000000..d6659267da
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_corrupt_telemetry.js
@@ -0,0 +1,24 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a request to forcibly
+// replace the current database.
+
+add_task(async function () {
+  await createCorruptDb("places.sqlite");
+
+  let count = Services.telemetry
+    .getHistogramById("PLACES_DATABASE_CORRUPTION_HANDLING_STAGE")
+    .snapshot().values[3];
+  Assert.equal(count, undefined, "There should be no telemetry");
+
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CORRUPT
+  );
+
+  count = Services.telemetry
+    .getHistogramById("PLACES_DATABASE_CORRUPTION_HANDLING_STAGE")
+    .snapshot().values[3];
+  Assert.equal(count, 1, "Telemetry should have been added");
+});
diff --git a/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup.js b/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup.js
new file mode 100644
index 0000000000..d48b32f5d6
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup.js
@@ -0,0 +1,14 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a request to forcibly
+// replace the current database.
+
+add_task(async function () {
+  await test_database_replacement(
+    ["migration", "favicons_v41.sqlite"],
+    "favicons.sqlite",
+    false,
+    PlacesUtils.history.DATABASE_STATUS_CREATE
+  );
+});
diff --git a/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup_clone.js b/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup_clone.js
new file mode 100644
index 0000000000..f6ff2379a0
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_favicons_replaceOnStartup_clone.js
@@ -0,0 +1,16 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a request to forcibly
+// replace the current database.
+
+add_task(async function () {
+  // In reality, this won't try to clone the database, because attached
+  // databases cannot be supported when cloning. This test also verifies that.
+  await test_database_replacement(
+    ["migration", "favicons_v41.sqlite"],
+    "favicons.sqlite",
+    true,
+    PlacesUtils.history.DATABASE_STATUS_CREATE
+  );
+});
diff --git a/toolkit/components/places/tests/maintenance/test_integrity_replacement.js b/toolkit/components/places/tests/maintenance/test_integrity_replacement.js
new file mode 100644
index 0000000000..dde8fd16a3
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_integrity_replacement.js
@@ -0,0 +1,17 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that integrity check will replace a corrupt database.
+
+add_task(async function () {
+  await setupPlacesDatabase("corruptPayload.sqlite");
+  await Assert.rejects(
+    PlacesDBUtils.checkIntegrity(),
+    /will be replaced on next startup/,
+    "Should reject on corruption"
+  );
+  Assert.equal(
+    Services.prefs.getCharPref("places.database.replaceDatabaseOnStartup"),
+    DB_FILENAME
+  );
+});
diff --git a/toolkit/components/places/tests/maintenance/test_places_purge_caches.js b/toolkit/components/places/tests/maintenance/test_places_purge_caches.js
new file mode 100644
index 0000000000..dc3e8452f1
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_places_purge_caches.js
@@ -0,0 +1,31 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Test whether purge-caches event works collectry when maintenance the places.
+
+add_task(async function test_history() {
+  await PlacesTestUtils.addVisits({ uri: "http://example.com/" });
+  await assertPurgingCaches();
+});
+
+add_task(async function test_bookmark() {
+  await PlacesTestUtils.addBookmarkWithDetails({ uri: "http://example.com/" });
+  await assertPurgingCaches();
+});
+
+async function assertPurgingCaches() {
+  const query = PlacesUtils.history.getNewQuery();
+  const options = PlacesUtils.history.getNewQueryOptions();
+  const result = PlacesUtils.history.executeQuery(query, options);
+  result.root.containerOpen = true;
+
+  const onInvalidateContainer = new Promise(resolve => {
+    const resultObserver = new NavHistoryResultObserver();
+    resultObserver.invalidateContainer = resolve;
+    result.addObserver(resultObserver, false);
+  });
+
+  await PlacesDBUtils.maintenanceOnIdle();
+  await onInvalidateContainer;
+  ok(true, "InvalidateContainer is called");
+}
diff --git a/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup.js b/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup.js
new file mode 100644
index 0000000000..dae5154df4
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup.js
@@ -0,0 +1,14 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a request to forcibly
+// replace the current database.
+
+add_task(async function () {
+  await test_database_replacement(
+    ["migration", "places_v43.sqlite"],
+    "places.sqlite",
+    false,
+    PlacesUtils.history.DATABASE_STATUS_CORRUPT
+  );
+});
diff --git a/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup_clone.js b/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup_clone.js
new file mode 100644
index 0000000000..d2ef1374e9
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_places_replaceOnStartup_clone.js
@@ -0,0 +1,14 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that history initialization correctly handles a request to forcibly
+// replace the current database.
+
+add_task(async function () {
+  await test_database_replacement(
+    ["migration", "places_v43.sqlite"],
+    "places.sqlite",
+    true,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+});
diff --git a/toolkit/components/places/tests/maintenance/test_preventive_maintenance.js b/toolkit/components/places/tests/maintenance/test_preventive_maintenance.js
new file mode 100644
index 0000000000..a1ae830d8e
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_preventive_maintenance.js
@@ -0,0 +1,2823 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Test preventive maintenance
+ * For every maintenance query create an uncoherent db and check that we take
+ * correct fix steps, without polluting valid data.
+ */
+
+// ------------------------------------------------------------------------------
+// Helpers
+
+var defaultBookmarksMaxId = 0;
+async function cleanDatabase() {
+  // First clear any bookmarks the "proper way" to ensure caches like GuidHelper
+  // are properly cleared.
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.withConnectionWrapper("cleanDatabase", async db => {
+    await db.executeTransaction(async () => {
+      await db.executeCached("DELETE FROM moz_places");
+      await db.executeCached("DELETE FROM moz_origins");
+      await db.executeCached("DELETE FROM moz_historyvisits");
+      await db.executeCached("DELETE FROM moz_anno_attributes");
+      await db.executeCached("DELETE FROM moz_annos");
+      await db.executeCached("DELETE FROM moz_inputhistory");
+      await db.executeCached("DELETE FROM moz_keywords");
+      await db.executeCached("DELETE FROM moz_icons");
+      await db.executeCached("DELETE FROM moz_pages_w_icons");
+      await db.executeCached(
+        "DELETE FROM moz_bookmarks WHERE id > " + defaultBookmarksMaxId
+      );
+      await db.executeCached("DELETE FROM moz_bookmarks_deleted");
+      await db.executeCached("DELETE FROM moz_places_metadata_search_queries");
+    });
+  });
+  // Since we're doing raw deletes, we must invalidate the guids cache.
+  await PlacesUtils.invalidateCachedGuids();
+}
+
+async function addPlace(
+  aUrl,
+  aFavicon,
+  aGuid = PlacesUtils.history.makeGuid(),
+  aHash = null
+) {
+  let href = new URL(
+    aUrl || `http://www.mozilla.org/${encodeURIComponent(aGuid)}`
+  ).href;
+  let id;
+  await PlacesUtils.withConnectionWrapper("cleanDatabase", async db => {
+    await db.executeTransaction(async () => {
+      id = (
+        await db.executeCached(
+          `INSERT INTO moz_places (url, url_hash, guid)
+           VALUES (:url, IFNULL(:hash, hash(:url)), :guid)
+           RETURNING id`,
+          {
+            url: href,
+            hash: aHash,
+            guid: aGuid,
+          }
+        )
+      )[0].getResultByIndex(0);
+      await db.executeCached("DELETE FROM moz_updateoriginsinsert_temp");
+
+      if (aFavicon) {
+        await db.executeCached(
+          `INSERT INTO moz_pages_w_icons (page_url, page_url_hash)
+           VALUES (:url, IFNULL(:hash, hash(:url)))`,
+          {
+            url: href,
+            hash: aHash,
+          }
+        );
+        await db.executeCached(
+          `INSERT INTO moz_icons_to_pages (page_id, icon_id)
+           VALUES (
+            (SELECT id FROM moz_pages_w_icons WHERE page_url_hash = IFNULL(:hash, hash(:url))),
+            :favicon
+           )`,
+          {
+            url: href,
+            hash: aHash,
+            favicon: aFavicon,
+          }
+        );
+      }
+    });
+  });
+  return id;
+}
+
+async function addBookmark(
+  aPlaceId,
+  aType,
+  aParentGuid = PlacesUtils.bookmarks.unfiledGuid,
+  aKeywordId,
+  aTitle,
+  aGuid = PlacesUtils.history.makeGuid(),
+  aSyncStatus = PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+  aSyncChangeCounter = 0
+) {
+  return PlacesUtils.withConnectionWrapper("addBookmark", async db => {
+    return (
+      await db.executeCached(
+        `INSERT INTO moz_bookmarks (fk, type, parent, keyword_id,
+                                  title, guid, syncStatus, syncChangeCounter)
+         VALUES (:place_id, :type,
+                 (SELECT id FROM moz_bookmarks WHERE guid = :parent), :keyword_id,
+                  :title, :guid, :sync_status, :change_counter)
+         RETURNING id`,
+        {
+          place_id: aPlaceId || null,
+          type: aType || null,
+          parent: aParentGuid,
+          keyword_id: aKeywordId || null,
+          title: typeof aTitle == "string" ? aTitle : null,
+          guid: aGuid,
+          sync_status: aSyncStatus,
+          change_counter: aSyncChangeCounter,
+        }
+      )
+    )[0].getResultByIndex(0);
+  });
+}
+
+// ------------------------------------------------------------------------------
+// Tests
+
+var tests = [];
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "A.1",
+  desc: "Remove obsolete annotations from moz_annos",
+
+  _obsoleteWeaveAttribute: "weave/test",
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid.
+    this._placeId = await addPlace();
+    // Add an obsolete attribute.
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeTransaction(async () => {
+        db.executeCached(
+          "INSERT INTO moz_anno_attributes (name) VALUES (:anno)",
+          { anno: this._obsoleteWeaveAttribute }
+        );
+
+        db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+           VALUES (:place_id,
+             (SELECT id FROM moz_anno_attributes WHERE name = :anno)
+           )`,
+          {
+            place_id: this._placeId,
+            anno: this._obsoleteWeaveAttribute,
+          }
+        );
+      });
+    });
+  },
+
+  async check() {
+    // Check that the obsolete annotation has been removed.
+    Assert.strictEqual(
+      await PlacesTestUtils.getDatabaseValue("moz_anno_attributes", "id", {
+        name: this._obsoleteWeaveAttribute,
+      }),
+      undefined
+    );
+  },
+});
+
+tests.push({
+  name: "A.3",
+  desc: "Remove unused attributes",
+
+  _usedPageAttribute: "usedPage",
+  _unusedAttribute: "unused",
+  _placeId: null,
+  _bookmarkId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // add a bookmark
+    this._bookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeTransaction(async () => {
+        // Add a used attribute and an unused one.
+        await db.executeCached(
+          `INSERT INTO moz_anno_attributes (name)
+           VALUES (:anno1), (:anno2)`,
+          {
+            anno1: this._usedPageAttribute,
+            anno2: this._unusedAttribute,
+          }
+        );
+        await db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+           VALUES(:place_id, (SELECT id FROM moz_anno_attributes WHERE name = :anno))`,
+          {
+            place_id: this._placeId,
+            anno: this._usedPageAttribute,
+          }
+        );
+      });
+    });
+  },
+
+  async check() {
+    // Check that used attributes are still there
+    let value = await PlacesTestUtils.getDatabaseValue(
+      "moz_anno_attributes",
+      "id",
+      {
+        name: this._usedPageAttribute,
+      }
+    );
+    Assert.notStrictEqual(value, undefined);
+    // Check that unused attribute has been removed
+    value = await PlacesTestUtils.getDatabaseValue(
+      "moz_anno_attributes",
+      "id",
+      {
+        name: this._unusedAttribute,
+      }
+    );
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "B.1",
+  desc: "Remove annotations with an invalid attribute",
+
+  _usedPageAttribute: "usedPage",
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeTransaction(async () => {
+        // Add a used attribute.
+        await db.executeCached(
+          "INSERT INTO moz_anno_attributes (name) VALUES (:anno)",
+          { anno: this._usedPageAttribute }
+        );
+        await db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+           VALUES(:place_id, (SELECT id FROM moz_anno_attributes WHERE name = :anno))`,
+          {
+            place_id: this._placeId,
+            anno: this._usedPageAttribute,
+          }
+        );
+        // Add an annotation with a nonexistent attribute
+        await db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+           VALUES(:place_id, 1337)`,
+          { place_id: this._placeId }
+        );
+      });
+    });
+  },
+
+  async check() {
+    // Check that used attribute is still there
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      "SELECT id FROM moz_anno_attributes WHERE name = :anno",
+      { anno: this._usedPageAttribute }
+    );
+    Assert.equal(rows.length, 1);
+    // check that annotation with valid attribute is still there
+    rows = await db.executeCached(
+      `SELECT id FROM moz_annos
+       WHERE anno_attribute_id = (SELECT id FROM moz_anno_attributes WHERE name = :anno)`,
+      { anno: this._usedPageAttribute }
+    );
+    Assert.equal(rows.length, 1);
+    // Check that annotation with bogus attribute has been removed
+    let value = await PlacesTestUtils.getDatabaseValue("moz_annos", "id", {
+      anno_attribute_id: 1337,
+    });
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "B.2",
+  desc: "Remove orphan page annotations",
+
+  _usedPageAttribute: "usedPage",
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeTransaction(async () => {
+        // Add a used attribute.
+        await db.executeCached(
+          "INSERT INTO moz_anno_attributes (name) VALUES (:anno)",
+          { anno: this._usedPageAttribute }
+        );
+        await db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+           VALUES(:place_id, (SELECT id FROM moz_anno_attributes WHERE name = :anno))`,
+          { place_id: this._placeId, anno: this._usedPageAttribute }
+        );
+        // Add an annotation to a nonexistent page
+        await db.executeCached(
+          `INSERT INTO moz_annos (place_id, anno_attribute_id)
+          VALUES(:place_id, (SELECT id FROM moz_anno_attributes WHERE name = :anno))`,
+          { place_id: 1337, anno: this._usedPageAttribute }
+        );
+      });
+    });
+  },
+
+  async check() {
+    // Check that used attribute is still there
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      "SELECT id FROM moz_anno_attributes WHERE name = :anno",
+      { anno: this._usedPageAttribute }
+    );
+    Assert.equal(rows.length, 1);
+    // check that annotation with valid attribute is still there
+    rows = await db.executeCached(
+      `SELECT id FROM moz_annos
+       WHERE anno_attribute_id = (SELECT id FROM moz_anno_attributes WHERE name = :anno)`,
+      { anno: this._usedPageAttribute }
+    );
+    Assert.equal(rows.length, 1);
+    // Check that an annotation to a nonexistent page has been removed
+    let value = await PlacesTestUtils.getDatabaseValue("moz_annos", "id", {
+      place_id: 1337,
+    });
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.9",
+  desc: "Remove items without a valid place",
+
+  _validItemId: null,
+  _invalidItemId: null,
+  _invalidSyncedItemId: null,
+  placeId: null,
+
+  _changeCounterStmt: null,
+  _menuChangeCounter: -1,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this.placeId = await addPlace();
+    // Insert a valid bookmark
+    this._validItemId = await addBookmark(
+      this.placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    // Insert a bookmark with an invalid place
+    this._invalidItemId = await addBookmark(
+      1337,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    // Insert a synced bookmark with an invalid place. We should write a
+    // tombstone when we remove it.
+    this._invalidSyncedItemId = await addBookmark(
+      1337,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.menuGuid,
+      null,
+      null,
+      "bookmarkAAAA",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+    // Insert a folder referencing a nonexistent place ID. D.5 should convert
+    // it to a bookmark; D.9 should remove it.
+    this._invalidWrongTypeItemId = await addBookmark(
+      1337,
+      PlacesUtils.bookmarks.TYPE_FOLDER
+    );
+
+    let value = await PlacesTestUtils.getDatabaseValue(
+      "moz_bookmarks",
+      "syncChangeCounter",
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+      }
+    );
+    Assert.equal(value, 0);
+    this._menuChangeCounter = value;
+  },
+
+  async check() {
+    // Check that valid bookmark is still there
+    let value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+      id: this._validItemId,
+    });
+    Assert.notStrictEqual(value, undefined);
+    // Check that invalid bookmarks have been removed
+    for (let id of [
+      this._invalidItemId,
+      this._invalidSyncedItemId,
+      this._invalidWrongTypeItemId,
+    ]) {
+      value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+        id,
+      });
+      Assert.strictEqual(value, undefined);
+    }
+
+    value = await PlacesTestUtils.getDatabaseValue(
+      "moz_bookmarks",
+      "syncChangeCounter",
+      { guid: PlacesUtils.bookmarks.menuGuid }
+    );
+    Assert.equal(value, 1);
+    Assert.equal(value, this._menuChangeCounter + 1);
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["bookmarkAAAA"]
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.1",
+  desc: "Remove items that are not uri bookmarks from tag containers",
+
+  _tagId: null,
+  _bookmarkId: null,
+  _separatorId: null,
+  _folderId: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Create a tag
+    this._tagId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.tagsGuid
+    );
+    let tagGuid = await PlacesUtils.promiseItemGuid(this._tagId);
+    // Insert a bookmark in the tag
+    this._bookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      tagGuid
+    );
+    // Insert a separator in the tag
+    this._separatorId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      tagGuid
+    );
+    // Insert a folder in the tag
+    this._folderId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      tagGuid
+    );
+  },
+
+  async check() {
+    // Check that valid bookmark is still there
+    let value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      parent: this._tagId,
+    });
+    Assert.notStrictEqual(value, undefined);
+    // Check that separator is no more there
+    value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+      type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      parent: this._tagId,
+    });
+    Assert.equal(value, undefined);
+    // Check that folder is no more there
+    value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      parent: this._tagId,
+    });
+    Assert.equal(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.2",
+  desc: "Remove empty tags",
+
+  _tagId: null,
+  _bookmarkId: null,
+  _emptyTagId: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Create a tag
+    this._tagId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.tagsGuid
+    );
+    let tagGuid = await PlacesUtils.promiseItemGuid(this._tagId);
+    // Insert a bookmark in the tag
+    this._bookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      tagGuid
+    );
+    // Create another tag (empty)
+    this._emptyTagId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.tagsGuid
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    // Check that valid bookmark is still there
+    let value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "id", {
+      id: this._bookmarkId,
+      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      parent: this._tagId,
+    });
+    Assert.notStrictEqual(value, undefined);
+    let rows = await db.executeCached(
+      `SELECT b.id FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      WHERE b.id = :id AND b.type = :type AND p.guid = :parent`,
+      {
+        id: this._tagId,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        parent: PlacesUtils.bookmarks.tagsGuid,
+      }
+    );
+    Assert.equal(rows.length, 1);
+    rows = await db.executeCached(
+      `SELECT b.id FROM moz_bookmarks b
+      JOIN moz_bookmarks p ON p.id = b.parent
+      WHERE b.id = :id AND b.type = :type AND p.guid = :parent`,
+      {
+        id: this._emptyTagId,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        parent: PlacesUtils.bookmarks.tagsGuid,
+      }
+    );
+    Assert.equal(rows.length, 0);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.3",
+  desc: "Move orphan items to unsorted folder",
+
+  _orphanBookmarkId: null,
+  _orphanSeparatorId: null,
+  _orphanFolderId: null,
+  _bookmarkId: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Insert an orphan bookmark
+    this._orphanBookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      8888
+    );
+    // Insert an orphan separator
+    this._orphanSeparatorId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      8888
+    );
+    // Insert a orphan folder
+    this._orphanFolderId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      8888
+    );
+    let folderGuid = await PlacesUtils.promiseItemGuid(this._orphanFolderId);
+    // Create a child of the last created folder
+    this._bookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      folderGuid
+    );
+  },
+
+  async check() {
+    // Check that bookmarks are now children of a real folder (unfiled)
+    let expectedInfos = [
+      {
+        id: this._orphanBookmarkId,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._orphanSeparatorId,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._orphanFolderId,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._bookmarkId,
+        parent: await PlacesUtils.promiseItemGuid(this._orphanFolderId),
+        syncChangeCounter: 0,
+      },
+      {
+        id: await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.unfiledGuid),
+        parent: PlacesUtils.bookmarks.rootGuid,
+        syncChangeCounter: 3,
+      },
+    ];
+    let db = await PlacesUtils.promiseDBConnection();
+    for (let { id, parent, syncChangeCounter } of expectedInfos) {
+      let rows = await db.executeCached(
+        `
+        SELECT b.id, b.syncChangeCounter
+        FROM moz_bookmarks b
+        JOIN moz_bookmarks p ON b.parent = p.id
+        WHERE b.id = :item_id
+          AND p.guid = :parent`,
+        { item_id: id, parent }
+      );
+      Assert.equal(rows.length, 1);
+
+      let actualChangeCounter = rows[0].getResultByName("syncChangeCounter");
+      Assert.equal(actualChangeCounter, syncChangeCounter);
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.5",
+  desc: "Fix wrong item types | folders and separators",
+
+  _separatorId: null,
+  _separatorGuid: null,
+  _folderId: null,
+  _folderGuid: null,
+  _syncedFolderId: null,
+  _syncedFolderGuid: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Add a separator with a fk
+    this._separatorId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR
+    );
+    this._separatorGuid = await PlacesUtils.promiseItemGuid(this._separatorId);
+    // Add a folder with a fk
+    this._folderId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_FOLDER
+    );
+    this._folderGuid = await PlacesUtils.promiseItemGuid(this._folderId);
+    // Add a synced folder with a fk
+    this._syncedFolderId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      null,
+      "itemAAAAAAAA",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+    this._syncedFolderGuid = await PlacesUtils.promiseItemGuid(
+      this._syncedFolderId
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    // Check that items with an fk have been converted to bookmarks
+    let rows = await db.executeCached(
+      `SELECT id, guid, syncChangeCounter
+       FROM moz_bookmarks
+       WHERE id = :item_id AND type = :type`,
+      { item_id: this._separatorId, type: PlacesUtils.bookmarks.TYPE_BOOKMARK }
+    );
+    Assert.equal(rows.length, 1);
+    let expected = [
+      {
+        id: this._folderId,
+        oldGuid: this._folderGuid,
+      },
+      {
+        id: this._syncedFolderId,
+        oldGuid: this._syncedFolderGuid,
+      },
+    ];
+    for (let { id, oldGuid } of expected) {
+      rows = await db.executeCached(
+        `SELECT id, guid, syncChangeCounter
+         FROM moz_bookmarks
+         WHERE id = :item_id AND type = :type`,
+        { item_id: id, type: PlacesUtils.bookmarks.TYPE_BOOKMARK }
+      );
+      Assert.equal(rows.length, 1);
+      Assert.notEqual(rows[0].getResultByName("guid"), oldGuid);
+      Assert.equal(rows[0].getResultByName("syncChangeCounter"), 1);
+      await Assert.rejects(
+        PlacesUtils.promiseItemId(oldGuid),
+        /no item found for the given GUID/
+      );
+    }
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["itemAAAAAAAA"]
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.6",
+  desc: "Fix wrong item types | bookmarks",
+
+  _validBookmarkId: null,
+  _validBookmarkGuid: null,
+  _invalidBookmarkId: null,
+  _invalidBookmarkGuid: null,
+  _invalidSyncedBookmarkId: null,
+  _invalidSyncedBookmarkGuid: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Add a bookmark with a valid place id
+    this._validBookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    this._validBookmarkGuid = await PlacesUtils.promiseItemGuid(
+      this._validBookmarkId
+    );
+    // Add a bookmark with a null place id
+    this._invalidBookmarkId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    this._invalidBookmarkGuid = await PlacesUtils.promiseItemGuid(
+      this._invalidBookmarkId
+    );
+    // Add a synced bookmark with a null place id
+    this._invalidSyncedBookmarkId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      null,
+      "bookmarkAAAA",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+    this._invalidSyncedBookmarkGuid = await PlacesUtils.promiseItemGuid(
+      this._invalidSyncedBookmarkId
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    // Check valid bookmark
+    let rows = await db.executeCached(
+      `SELECT id, guid, syncChangeCounter
+       FROM moz_bookmarks
+       WHERE id = :item_id AND type = :type`,
+      {
+        item_id: this._validBookmarkId,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      }
+    );
+    Assert.equal(rows.length, 1);
+    Assert.equal(rows[0].getResultByName("syncChangeCounter"), 0);
+    Assert.equal(
+      await PlacesUtils.promiseItemId(this._validBookmarkGuid),
+      this._validBookmarkId
+    );
+
+    // Check invalid bookmarks have been converted to folders
+    let expected = [
+      {
+        id: this._invalidBookmarkId,
+        oldGuid: this._invalidBookmarkGuid,
+      },
+      {
+        id: this._invalidSyncedBookmarkId,
+        oldGuid: this._invalidSyncedBookmarkGuid,
+      },
+    ];
+    for (let { id, oldGuid } of expected) {
+      rows = await db.executeCached(
+        `SELECT id, guid, syncChangeCounter
+         FROM moz_bookmarks
+         WHERE id = :item_id AND type = :type`,
+        { item_id: id, type: PlacesUtils.bookmarks.TYPE_FOLDER }
+      );
+      Assert.equal(rows.length, 1);
+      Assert.notEqual(rows[0].getResultByName("guid"), oldGuid);
+      Assert.equal(rows[0].getResultByName("syncChangeCounter"), 1);
+      await Assert.rejects(
+        PlacesUtils.promiseItemId(oldGuid),
+        /no item found for the given GUID/
+      );
+    }
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["bookmarkAAAA"]
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.7",
+  desc: "Fix missing item types",
+
+  _placeId: null,
+  _bookmarkId: null,
+  _bookmarkGuid: null,
+  _syncedBookmarkId: null,
+  _syncedBookmarkGuid: null,
+  _folderId: null,
+  _folderGuid: null,
+  _syncedFolderId: null,
+  _syncedFolderGuid: null,
+
+  async setup() {
+    // Item without a type but with a place ID; should be converted to a
+    // bookmark. The synced bookmark should be handled the same way, but with
+    // a tombstone.
+    this._placeId = await addPlace();
+    this._bookmarkId = await addBookmark(this._placeId);
+    this._bookmarkGuid = await PlacesUtils.promiseItemGuid(this._bookmarkId);
+    this._syncedBookmarkId = await addBookmark(
+      this._placeId,
+      null,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      null,
+      "bookmarkAAAA",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+    this._syncedBookmarkGuid = await PlacesUtils.promiseItemGuid(
+      this._syncedBookmarkId
+    );
+
+    // Item without a type and without a place ID; should be converted to a
+    // folder.
+    this._folderId = await addBookmark();
+    this._folderGuid = await PlacesUtils.promiseItemGuid(this._folderId);
+    this._syncedFolderId = await addBookmark(
+      null,
+      null,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      null,
+      "folderBBBBBB",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+    this._syncedFolderGuid = await PlacesUtils.promiseItemGuid(
+      this._syncedFolderId
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let expected = [
+      {
+        id: this._bookmarkId,
+        oldGuid: this._bookmarkGuid,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._syncedBookmarkId,
+        oldGuid: this._syncedBookmarkGuid,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._folderId,
+        oldGuid: this._folderGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._syncedFolderId,
+        oldGuid: this._syncedFolderGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        syncChangeCounter: 1,
+      },
+    ];
+    for (let { id, oldGuid, type, syncChangeCounter } of expected) {
+      let rows = await db.executeCached(
+        `SELECT id, guid, type, syncChangeCounter
+         FROM moz_bookmarks
+         WHERE id = :item_id`,
+        { item_id: id }
+      );
+      Assert.equal(rows.length, 1);
+      Assert.equal(rows[0].getResultByName("type"), type);
+      Assert.equal(
+        rows[0].getResultByName("syncChangeCounter"),
+        syncChangeCounter
+      );
+      Assert.notEqual(rows[0].getResultByName("guid"), oldGuid);
+      await Assert.rejects(
+        PlacesUtils.promiseItemId(oldGuid),
+        /no item found for the given GUID/
+      );
+    }
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["bookmarkAAAA", "folderBBBBBB"]
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.8",
+  desc: "Fix wrong parents",
+
+  _bookmarkId: null,
+  _separatorId: null,
+  _bookmarkId1: null,
+  _bookmarkId2: null,
+  _placeId: null,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Insert a bookmark
+    this._bookmarkId = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK
+    );
+    // Insert a separator
+    this._separatorId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR
+    );
+    // Create 3 children of these items
+    let bookmarkGuid = await PlacesUtils.promiseItemGuid(this._bookmarkId);
+    this._bookmarkId1 = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      bookmarkGuid
+    );
+    let separatorGuid = await PlacesUtils.promiseItemGuid(this._separatorId);
+    this._bookmarkId2 = await addBookmark(
+      this._placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      separatorGuid
+    );
+  },
+
+  async check() {
+    // Check that bookmarks are now children of a real folder (unfiled)
+    let expectedInfos = [
+      {
+        id: this._bookmarkId1,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        syncChangeCounter: 1,
+      },
+      {
+        id: this._bookmarkId2,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        syncChangeCounter: 1,
+      },
+      {
+        id: await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.unfiledGuid),
+        parent: PlacesUtils.bookmarks.rootGuid,
+        syncChangeCounter: 2,
+      },
+    ];
+    let db = await PlacesUtils.promiseDBConnection();
+    for (let { id, parent, syncChangeCounter } of expectedInfos) {
+      let rows = await db.executeCached(
+        `
+        SELECT b.id, b.syncChangeCounter
+        FROM moz_bookmarks b
+        JOIN moz_bookmarks p ON b.parent = p.id
+        WHERE b.id = :item_id AND p.guid = :parent`,
+        { item_id: id, parent }
+      );
+      Assert.equal(rows.length, 1);
+
+      let actualChangeCounter = rows[0].getResultByName("syncChangeCounter");
+      Assert.equal(actualChangeCounter, syncChangeCounter);
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.10",
+  desc: "Recalculate positions",
+
+  _unfiledBookmarks: [],
+  _toolbarBookmarks: [],
+
+  async setup() {
+    const NUM_BOOKMARKS = 20;
+    let children = [];
+    for (let i = 0; i < NUM_BOOKMARKS; i++) {
+      children.push({
+        title: "testbookmark",
+        url: "http://example.com",
+      });
+    }
+
+    // Add bookmarks to two folders to better perturbate the table.
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      children,
+      source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    });
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      children,
+      source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    });
+
+    async function randomize_positions(aParent, aResultArray) {
+      await PlacesUtils.withConnectionWrapper("setup", async db => {
+        await db.executeTransaction(async () => {
+          for (let i = 0; i < NUM_BOOKMARKS / 2; i++) {
+            await db.executeCached(
+              `UPDATE moz_bookmarks SET position = :rand
+               WHERE id IN (
+                 SELECT b.id FROM moz_bookmarks b
+                 JOIN moz_bookmarks p ON b.parent = p.id
+                 WHERE p.guid = :parent
+                 ORDER BY RANDOM() LIMIT 1
+               )`,
+              {
+                parent: aParent,
+                rand: Math.round(Math.random() * (NUM_BOOKMARKS - 1)),
+              }
+            );
+          }
+
+          // Build the expected ordered list of bookmarks.
+          let rows = await db.executeCached(
+            `SELECT b.id
+             FROM moz_bookmarks b
+             JOIN moz_bookmarks p ON b.parent = p.id
+             WHERE p.guid = :parent
+             ORDER BY b.position ASC, b.ROWID ASC`,
+            { parent: aParent }
+          );
+          rows.forEach(r => {
+            aResultArray.push(r.getResultByName("id"));
+          });
+          await PlacesTestUtils.dumpTable({
+            db,
+            table: "moz_bookmarks",
+            columns: ["id", "parent", "position"],
+          });
+        });
+      });
+    }
+
+    // Set random positions for the added bookmarks.
+    await randomize_positions(
+      PlacesUtils.bookmarks.unfiledGuid,
+      this._unfiledBookmarks
+    );
+    await randomize_positions(
+      PlacesUtils.bookmarks.toolbarGuid,
+      this._toolbarBookmarks
+    );
+
+    let syncInfos = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.unfiledGuid,
+      PlacesUtils.bookmarks.toolbarGuid
+    );
+    Assert.ok(syncInfos.every(info => info.syncChangeCounter === 0));
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+
+    async function check_order(aParent, aResultArray) {
+      // Build the expected ordered list of bookmarks.
+      let childRows = await db.executeCached(
+        `SELECT b.id, b.position, b.syncChangeCounter
+         FROM moz_bookmarks b
+         JOIN moz_bookmarks p ON b.parent = p.id
+         WHERE p.guid = :parent
+         ORDER BY b.position ASC`,
+        { parent: aParent }
+      );
+      for (let row of childRows) {
+        let id = row.getResultByName("id");
+        let position = row.getResultByName("position");
+        if (aResultArray.indexOf(id) != position) {
+          info("Expected order: " + aResultArray);
+          await PlacesTestUtils.dumpTable({
+            db,
+            table: "moz_bookmarks",
+            columns: ["id", "parent", "position"],
+          });
+          do_throw(`Unexpected bookmarks order for ${aParent}.`);
+        }
+      }
+
+      let parentRows = await db.executeCached(
+        `SELECT syncChangeCounter FROM moz_bookmarks
+         WHERE guid = :parent`,
+        { parent: aParent }
+      );
+      for (let row of parentRows) {
+        let actualChangeCounter = row.getResultByName("syncChangeCounter");
+        Assert.ok(actualChangeCounter > 0);
+      }
+    }
+
+    await check_order(
+      PlacesUtils.bookmarks.unfiledGuid,
+      this._unfiledBookmarks
+    );
+    await check_order(
+      PlacesUtils.bookmarks.toolbarGuid,
+      this._toolbarBookmarks
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "D.13",
+  desc: "Fix empty-named tags",
+  _taggedItemIds: {},
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    let placeId = await addPlace();
+    // Create a empty-named tag.
+    this._untitledTagId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.tagsGuid,
+      null,
+      ""
+    );
+    // Insert a bookmark in the tag, otherwise it will be removed.
+    let untitledTagGuid = await PlacesUtils.promiseItemGuid(
+      this._untitledTagId
+    );
+    await addBookmark(
+      placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      untitledTagGuid
+    );
+    // Create a empty-named folder.
+    this._untitledFolderId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      ""
+    );
+    // Create a titled tag.
+    this._titledTagId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.tagsGuid,
+      null,
+      "titledTag"
+    );
+    // Insert a bookmark in the tag, otherwise it will be removed.
+    let titledTagGuid = await PlacesUtils.promiseItemGuid(this._titledTagId);
+    await addBookmark(
+      placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      titledTagGuid
+    );
+    // Create a titled folder.
+    this._titledFolderId = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      "titledFolder"
+    );
+
+    // Create two tagged bookmarks in different folders.
+    this._taggedItemIds.inMenu = await addBookmark(
+      placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.menuGuid,
+      null,
+      "Tagged bookmark in menu"
+    );
+    this._taggedItemIds.inToolbar = await addBookmark(
+      placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.toolbarGuid,
+      null,
+      "Tagged bookmark in toolbar"
+    );
+  },
+
+  async check() {
+    // Check that valid bookmark is still there
+    let value = await PlacesTestUtils.getDatabaseValue(
+      "moz_bookmarks",
+      "title",
+      { id: this._untitledTagId }
+    );
+    Assert.equal(value, "(notitle)");
+    value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "title", {
+      id: this._untitledFolderId,
+    });
+    Assert.equal(value, "");
+    value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "title", {
+      id: this._titledTagId,
+    });
+    Assert.equal(value, "titledTag");
+    value = await PlacesTestUtils.getDatabaseValue("moz_bookmarks", "title", {
+      id: this._titledFolderId,
+    });
+    Assert.equal(value, "titledFolder");
+
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      `SELECT syncChangeCounter FROM moz_bookmarks
+       WHERE id IN (:taggedInMenu, :taggedInToolbar)`,
+      {
+        taggedInMenu: this._taggedItemIds.inMenu,
+        taggedInToolbar: this._taggedItemIds.inToolbar,
+      }
+    );
+    for (let row of rows) {
+      Assert.greaterOrEqual(row.getResultByName("syncChangeCounter"), 1);
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "E.1",
+  desc: "Remove orphan icon entries",
+
+  _placeId: null,
+
+  async setup() {
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeTransaction(async () => {
+        // Insert favicon entries
+        await db.executeCached(
+          `INSERT INTO moz_icons (id, icon_url, fixed_icon_url_hash, root) VALUES(:favicon_id, :url, hash(fixup_url(:url)), :root)`,
+          [
+            {
+              favicon_id: 1,
+              url: "http://www1.mozilla.org/favicon.ico",
+              root: 0,
+            },
+            {
+              favicon_id: 2,
+              url: "http://www2.mozilla.org/favicon.ico",
+              root: 0,
+            },
+            {
+              favicon_id: 3,
+              url: "http://www3.mozilla.org/favicon.ico",
+              root: 1,
+            },
+          ]
+        );
+
+        // Insert orphan page.
+        await db.executeCached(
+          `INSERT INTO moz_pages_w_icons (id, page_url, page_url_hash)
+           VALUES(:page_id, :url, hash(:url))`,
+          { page_id: 99, url: "http://w99.mozilla.org/" }
+        );
+      });
+    });
+
+    // Insert a place using the existing favicon entry
+    this._placeId = await addPlace("http://www.mozilla.org", 1);
+  },
+
+  async check() {
+    // Check that used icon is still there
+    let value = await PlacesTestUtils.getDatabaseValue("moz_icons", "id", {
+      id: 1,
+    });
+    Assert.notStrictEqual(value, undefined);
+    // Check that unused icon has been removed
+    value = await PlacesTestUtils.getDatabaseValue("moz_icons", "id", {
+      id: 2,
+    });
+    Assert.strictEqual(value, undefined);
+    // Check that unused icon has been removed
+    value = await PlacesTestUtils.getDatabaseValue("moz_icons", "id", {
+      id: 3,
+    });
+    Assert.strictEqual(value, undefined);
+    // Check that the orphan page is gone.
+    value = await PlacesTestUtils.getDatabaseValue("moz_pages_w_icons", "id", {
+      id: 99,
+    });
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "F.1",
+  desc: "Remove orphan visits",
+
+  _placeId: null,
+  _invalidPlaceId: 1337,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Add a valid visit and an invalid one
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeCached(
+        `INSERT INTO moz_historyvisits(place_id)
+         VALUES (:place_id_1), (:place_id_2)`,
+        { place_id_1: this._placeId, place_id_2: this._invalidPlaceId }
+      );
+    });
+  },
+
+  async check() {
+    // Check that valid visit is still there
+    let value = await PlacesTestUtils.getDatabaseValue(
+      "moz_historyvisits",
+      "id",
+      {
+        place_id: this._placeId,
+      }
+    );
+    Assert.notStrictEqual(value, undefined);
+    // Check that invalid visit has been removed
+    value = await PlacesTestUtils.getDatabaseValue("moz_historyvisits", "id", {
+      place_id: this._invalidPlaceId,
+    });
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "G.1",
+  desc: "Remove orphan input history",
+
+  _placeId: null,
+  _invalidPlaceId: 1337,
+
+  async setup() {
+    // Add a place to ensure place_id = 1 is valid
+    this._placeId = await addPlace();
+    // Add input history entries
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeCached(
+        `INSERT INTO moz_inputhistory (place_id, input)
+         VALUES (:place_id_1, :input_1), (:place_id_2, :input_2)`,
+        {
+          place_id_1: this._placeId,
+          input_1: "moz",
+          place_id_2: this._invalidPlaceId,
+          input_2: "moz",
+        }
+      );
+    });
+  },
+
+  async check() {
+    // Check that inputhistory on valid place is still there
+    let value = await PlacesTestUtils.getDatabaseValue(
+      "moz_inputhistory",
+      "place_id",
+      { place_id: this._placeId }
+    );
+    Assert.notStrictEqual(value, undefined);
+    // Check that inputhistory on invalid place has gone
+    value = await PlacesTestUtils.getDatabaseValue(
+      "moz_inputhistory",
+      "place_id",
+      { place_id: this._invalidPlaceId }
+    );
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "I.1",
+  desc: "Remove unused keywords",
+
+  _bookmarkId: null,
+  _placeId: null,
+
+  async setup() {
+    // Insert 2 keywords
+    let bm = await PlacesUtils.bookmarks.insert({
+      url: "http://testkw.moz.org/",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+    await PlacesUtils.keywords.insert({
+      url: bm.url,
+      keyword: "used",
+    });
+
+    await PlacesUtils.withConnectionWrapper("setup", async db => {
+      await db.executeCached(
+        `INSERT INTO moz_keywords (id, keyword, place_id)
+         VALUES(NULL, :keyword, :place_id)`,
+        { keyword: "unused", place_id: 100 }
+      );
+    });
+  },
+
+  async check() {
+    // Check that "used" keyword is still there
+    let value = await PlacesTestUtils.getDatabaseValue("moz_keywords", "id", {
+      keyword: "used",
+    });
+    Assert.notStrictEqual(value, undefined);
+    // Check that "unused" keyword has gone
+    value = await PlacesTestUtils.getDatabaseValue("moz_keywords", "id", {
+      keyword: "unused",
+    });
+    Assert.strictEqual(value, undefined);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.1",
+  desc: "remove duplicate URLs",
+  _placeA: -1,
+  _placeD: -1,
+  _placeE: -1,
+  _bookmarkIds: [],
+
+  async setup() {
+    // Place with visits, an autocomplete history entry, anno, and a bookmark.
+    this._placeA = await addPlace("http://example.com", null, "placeAAAAAAA");
+
+    // Duplicate Place with different visits and a keyword.
+    let placeB = await addPlace("http://example.com", null, "placeBBBBBBB");
+
+    // Another duplicate with conflicting autocomplete history entry and
+    // two more bookmarks.
+    let placeC = await addPlace("http://example.com", null, "placeCCCCCCC");
+
+    // Unrelated, unique Place.
+    this._placeD = await addPlace(
+      "http://example.net",
+      null,
+      "placeDDDDDDD",
+      1234
+    );
+
+    // Another unrelated Place, with the same hash as D, but different URL.
+    this._placeE = await addPlace(
+      "http://example.info",
+      null,
+      "placeEEEEEEE",
+      1234
+    );
+
+    let visits = [
+      {
+        placeId: this._placeA,
+        date: new Date(2017, 1, 2),
+        type: PlacesUtils.history.TRANSITIONS.LINK,
+      },
+      {
+        placeId: this._placeA,
+        date: new Date(2018, 3, 4),
+        type: PlacesUtils.history.TRANSITIONS.TYPED,
+      },
+      {
+        placeId: placeB,
+        date: new Date(2016, 5, 6),
+        type: PlacesUtils.history.TRANSITIONS.TYPED,
+      },
+      {
+        // Duplicate visit; should keep both when we merge.
+        placeId: placeB,
+        date: new Date(2018, 3, 4),
+        type: PlacesUtils.history.TRANSITIONS.TYPED,
+      },
+      {
+        placeId: this._placeD,
+        date: new Date(2018, 7, 8),
+        type: PlacesUtils.history.TRANSITIONS.LINK,
+      },
+      {
+        placeId: this._placeE,
+        date: new Date(2018, 8, 9),
+        type: PlacesUtils.history.TRANSITIONS.TYPED,
+      },
+    ];
+
+    let inputs = [
+      {
+        placeId: this._placeA,
+        input: "exam",
+        count: 4,
+      },
+      {
+        placeId: placeC,
+        input: "exam",
+        count: 3,
+      },
+      {
+        placeId: placeC,
+        input: "ex",
+        count: 5,
+      },
+      {
+        placeId: this._placeD,
+        input: "amp",
+        count: 3,
+      },
+    ];
+
+    let annos = [
+      {
+        name: "anno",
+        placeId: this._placeA,
+        content: "splish",
+      },
+      {
+        // Anno that's already set on A; should be ignored when we merge.
+        name: "anno",
+        placeId: placeB,
+        content: "oops",
+      },
+      {
+        name: "other-anno",
+        placeId: placeB,
+        content: "splash",
+      },
+      {
+        name: "other-anno",
+        placeId: this._placeD,
+        content: "sploosh",
+      },
+    ];
+
+    let bookmarks = [
+      {
+        placeId: this._placeA,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        title: "A",
+        guid: "bookmarkAAAA",
+      },
+      {
+        placeId: placeB,
+        parentGuid: PlacesUtils.bookmarks.mobileGuid,
+        title: "B",
+        guid: "bookmarkBBBB",
+      },
+      {
+        placeId: placeC,
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        title: "C1",
+        guid: "bookmarkCCC1",
+      },
+      {
+        placeId: placeC,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        title: "C2",
+        guid: "bookmarkCCC2",
+      },
+      {
+        placeId: this._placeD,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        title: "D",
+        guid: "bookmarkDDDD",
+      },
+      {
+        placeId: this._placeE,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        title: "E",
+        guid: "bookmarkEEEE",
+      },
+    ];
+
+    let keywords = [
+      {
+        placeId: placeB,
+        keyword: "hi",
+      },
+      {
+        placeId: this._placeD,
+        keyword: "bye",
+      },
+    ];
+
+    for (let { placeId, parentGuid, title, guid } of bookmarks) {
+      let itemId = await addBookmark(
+        placeId,
+        PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        parentGuid,
+        null,
+        title,
+        guid
+      );
+      this._bookmarkIds.push(itemId);
+    }
+
+    await PlacesUtils.withConnectionWrapper(
+      "L.1: Insert foreign key refs",
+      function (db) {
+        return db.executeTransaction(async function () {
+          for (let { placeId, date, type } of visits) {
+            await db.executeCached(
+              `INSERT INTO moz_historyvisits(place_id, visit_date, visit_type)
+               VALUES(:placeId, :date, :type)`,
+              { placeId, date: PlacesUtils.toPRTime(date), type }
+            );
+          }
+
+          for (let params of inputs) {
+            await db.executeCached(
+              `INSERT INTO moz_inputhistory(place_id, input, use_count)
+               VALUES(:placeId, :input, :count)`,
+              params
+            );
+          }
+
+          for (let { name, placeId, content } of annos) {
+            await db.executeCached(
+              `INSERT OR IGNORE INTO moz_anno_attributes(name)
+               VALUES(:name)`,
+              { name }
+            );
+
+            await db.executeCached(
+              `INSERT INTO moz_annos(place_id, anno_attribute_id, content)
+               VALUES(:placeId, (SELECT id FROM moz_anno_attributes
+                                 WHERE name = :name), :content)`,
+              { placeId, name, content }
+            );
+          }
+
+          for (let param of keywords) {
+            await db.executeCached(
+              `INSERT INTO moz_keywords(keyword, place_id)
+               VALUES(:keyword, :placeId)`,
+              param
+            );
+          }
+        });
+      }
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+
+    let placeRows = await db.execute(`
+      SELECT id, guid, foreign_count FROM moz_places
+      ORDER BY guid`);
+    let placeInfos = placeRows.map(row => ({
+      id: row.getResultByName("id"),
+      guid: row.getResultByName("guid"),
+      foreignCount: row.getResultByName("foreign_count"),
+    }));
+    Assert.deepEqual(
+      placeInfos,
+      [
+        {
+          id: this._placeA,
+          guid: "placeAAAAAAA",
+          foreignCount: 5, // 4 bookmarks + 1 keyword
+        },
+        {
+          id: this._placeD,
+          guid: "placeDDDDDDD",
+          foreignCount: 2, // 1 bookmark + 1 keyword
+        },
+        {
+          id: this._placeE,
+          guid: "placeEEEEEEE",
+          foreignCount: 1, // 1 bookmark
+        },
+      ],
+      "Should remove duplicate Places B and C"
+    );
+
+    let visitRows = await db.execute(`
+      SELECT place_id, visit_date, visit_type FROM moz_historyvisits
+      ORDER BY visit_date`);
+    let visitInfos = visitRows.map(row => ({
+      placeId: row.getResultByName("place_id"),
+      date: PlacesUtils.toDate(row.getResultByName("visit_date")),
+      type: row.getResultByName("visit_type"),
+    }));
+    Assert.deepEqual(
+      visitInfos,
+      [
+        {
+          placeId: this._placeA,
+          date: new Date(2016, 5, 6),
+          type: PlacesUtils.history.TRANSITIONS.TYPED,
+        },
+        {
+          placeId: this._placeA,
+          date: new Date(2017, 1, 2),
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        },
+        {
+          placeId: this._placeA,
+          date: new Date(2018, 3, 4),
+          type: PlacesUtils.history.TRANSITIONS.TYPED,
+        },
+        {
+          placeId: this._placeA,
+          date: new Date(2018, 3, 4),
+          type: PlacesUtils.history.TRANSITIONS.TYPED,
+        },
+        {
+          placeId: this._placeD,
+          date: new Date(2018, 7, 8),
+          type: PlacesUtils.history.TRANSITIONS.LINK,
+        },
+        {
+          placeId: this._placeE,
+          date: new Date(2018, 8, 9),
+          type: PlacesUtils.history.TRANSITIONS.TYPED,
+        },
+      ],
+      "Should merge history visits"
+    );
+
+    let inputRows = await db.execute(`
+      SELECT place_id, input, use_count FROM moz_inputhistory
+      ORDER BY use_count ASC`);
+    let inputInfos = inputRows.map(row => ({
+      placeId: row.getResultByName("place_id"),
+      input: row.getResultByName("input"),
+      count: row.getResultByName("use_count"),
+    }));
+    Assert.deepEqual(
+      inputInfos,
+      [
+        {
+          placeId: this._placeD,
+          input: "amp",
+          count: 3,
+        },
+        {
+          placeId: this._placeA,
+          input: "ex",
+          count: 5,
+        },
+        {
+          placeId: this._placeA,
+          input: "exam",
+          count: 7,
+        },
+      ],
+      "Should merge autocomplete history"
+    );
+
+    let annoRows = await db.execute(`
+      SELECT a.place_id, n.name, a.content FROM moz_annos a
+      JOIN moz_anno_attributes n ON n.id = a.anno_attribute_id
+      ORDER BY n.name, a.content ASC`);
+    let annoInfos = annoRows.map(row => ({
+      placeId: row.getResultByName("place_id"),
+      name: row.getResultByName("name"),
+      content: row.getResultByName("content"),
+    }));
+    Assert.deepEqual(
+      annoInfos,
+      [
+        {
+          placeId: this._placeA,
+          name: "anno",
+          content: "splish",
+        },
+        {
+          placeId: this._placeA,
+          name: "other-anno",
+          content: "splash",
+        },
+        {
+          placeId: this._placeD,
+          name: "other-anno",
+          content: "sploosh",
+        },
+      ],
+      "Should merge page annos"
+    );
+
+    let itemRows = await db.execute(
+      `
+      SELECT guid, fk, syncChangeCounter FROM moz_bookmarks
+      WHERE id IN (${new Array(this._bookmarkIds.length).fill("?").join(",")})
+      ORDER BY guid ASC`,
+      this._bookmarkIds
+    );
+    let itemInfos = itemRows.map(row => ({
+      guid: row.getResultByName("guid"),
+      placeId: row.getResultByName("fk"),
+      syncChangeCounter: row.getResultByName("syncChangeCounter"),
+    }));
+    Assert.deepEqual(
+      itemInfos,
+      [
+        {
+          guid: "bookmarkAAAA",
+          placeId: this._placeA,
+          syncChangeCounter: 1,
+        },
+        {
+          guid: "bookmarkBBBB",
+          placeId: this._placeA,
+          syncChangeCounter: 1,
+        },
+        {
+          guid: "bookmarkCCC1",
+          placeId: this._placeA,
+          syncChangeCounter: 1,
+        },
+        {
+          guid: "bookmarkCCC2",
+          placeId: this._placeA,
+          syncChangeCounter: 1,
+        },
+        {
+          guid: "bookmarkDDDD",
+          placeId: this._placeD,
+          syncChangeCounter: 0,
+        },
+        {
+          guid: "bookmarkEEEE",
+          placeId: this._placeE,
+          syncChangeCounter: 0,
+        },
+      ],
+      "Should merge bookmarks and bump change counter"
+    );
+
+    let keywordRows = await db.execute(`
+      SELECT keyword, place_id FROM moz_keywords
+      ORDER BY keyword ASC`);
+    let keywordInfos = keywordRows.map(row => ({
+      keyword: row.getResultByName("keyword"),
+      placeId: row.getResultByName("place_id"),
+    }));
+    Assert.deepEqual(
+      keywordInfos,
+      [
+        {
+          keyword: "bye",
+          placeId: this._placeD,
+        },
+        {
+          keyword: "hi",
+          placeId: this._placeA,
+        },
+      ],
+      "Should merge all keywords"
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.2",
+  desc: "Recalculate visit_count and last_visit_date",
+
+  async setup() {
+    async function setVisitCount(aURL, aValue) {
+      await PlacesUtils.withConnectionWrapper("setVisitCount", async db => {
+        await db.executeCached(
+          `UPDATE moz_places SET visit_count = :count
+           WHERE url_hash = hash(:url) AND url = :url`,
+          { count: aValue, url: aURL }
+        );
+      });
+    }
+    async function setLastVisitDate(aURL, aValue) {
+      await PlacesUtils.withConnectionWrapper("setVisitCount", async db => {
+        await db.executeCached(
+          `UPDATE moz_places SET last_visit_date = :date
+           WHERE url_hash = hash(:url) AND url = :url`,
+          { date: aValue, url: aURL }
+        );
+      });
+    }
+
+    let now = Date.now() * 1000;
+    // Add a page with 1 visit.
+    let url = "http://1.moz.org/";
+    await PlacesTestUtils.addVisits({ uri: uri(url), visitDate: now++ });
+    // Add a page with 1 visit and set wrong visit_count.
+    url = "http://2.moz.org/";
+    await PlacesTestUtils.addVisits({ uri: uri(url), visitDate: now++ });
+    await setVisitCount(url, 10);
+    // Add a page with 1 visit and set wrong last_visit_date.
+    url = "http://3.moz.org/";
+    await PlacesTestUtils.addVisits({ uri: uri(url), visitDate: now++ });
+    await setLastVisitDate(url, now++);
+    // Add a page with 1 visit and set wrong stats.
+    url = "http://4.moz.org/";
+    await PlacesTestUtils.addVisits({ uri: uri(url), visitDate: now++ });
+    await setVisitCount(url, 10);
+    await setLastVisitDate(url, now++);
+
+    // Add a page without visits.
+    url = "http://5.moz.org/";
+    await addPlace(url);
+    // Add a page without visits and set wrong visit_count.
+    url = "http://6.moz.org/";
+    await addPlace(url);
+    await setVisitCount(url, 10);
+    // Add a page without visits and set wrong last_visit_date.
+    url = "http://7.moz.org/";
+    await addPlace(url);
+    await setLastVisitDate(url, now++);
+    // Add a page without visits and set wrong stats.
+    url = "http://8.moz.org/";
+    await addPlace(url);
+    await setVisitCount(url, 10);
+    await setLastVisitDate(url, now++);
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      `SELECT h.id, h.last_visit_date as v_date
+       FROM moz_places h
+       LEFT JOIN moz_historyvisits v ON v.place_id = h.id AND visit_type NOT IN (0,4,7,8,9)
+       GROUP BY h.id HAVING h.visit_count <> count(v.id)
+       UNION ALL
+       SELECT h.id, MAX(v.visit_date) as v_date
+       FROM moz_places h
+       LEFT JOIN moz_historyvisits v ON v.place_id = h.id
+       GROUP BY h.id HAVING h.last_visit_date IS NOT v_date`
+    );
+    Assert.equal(rows.length, 0);
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.3",
+  desc: "recalculate hidden for redirects.",
+
+  async setup() {
+    await PlacesTestUtils.addVisits([
+      {
+        uri: NetUtil.newURI("http://l3.moz.org/"),
+        transition: TRANSITION_TYPED,
+      },
+      {
+        uri: NetUtil.newURI("http://l3.moz.org/redirecting/"),
+        transition: TRANSITION_TYPED,
+      },
+      {
+        uri: NetUtil.newURI("http://l3.moz.org/redirecting2/"),
+        transition: TRANSITION_REDIRECT_TEMPORARY,
+        referrer: NetUtil.newURI("http://l3.moz.org/redirecting/"),
+      },
+      {
+        uri: NetUtil.newURI("http://l3.moz.org/target/"),
+        transition: TRANSITION_REDIRECT_PERMANENT,
+        referrer: NetUtil.newURI("http://l3.moz.org/redirecting2/"),
+      },
+    ]);
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      "SELECT h.url FROM moz_places h WHERE h.hidden = 1"
+    );
+    Assert.equal(rows.length, 2);
+    for (let row of rows) {
+      let url = row.getResultByIndex(0);
+      Assert.ok(/redirecting/.test(url));
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.4",
+  desc: "recalculate foreign_count.",
+
+  async setup() {
+    this._pageGuid = (
+      await PlacesUtils.history.insert({
+        url: "http://l4.moz.org/",
+        visits: [{ date: new Date() }],
+      })
+    ).guid;
+    await PlacesUtils.bookmarks.insert({
+      url: "http://l4.moz.org/",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+    await PlacesUtils.keywords.insert({
+      url: "http://l4.moz.org/",
+      keyword: "kw",
+    });
+    Assert.equal(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "foreign_count", {
+        guid: this._pageGuid,
+      }),
+      2
+    );
+  },
+
+  async check() {
+    Assert.equal(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "foreign_count", {
+        guid: this._pageGuid,
+      }),
+      2
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.5",
+  desc: "recalculate hashes when missing.",
+
+  async setup() {
+    this._pageGuid = (
+      await PlacesUtils.history.insert({
+        url: "http://l5.moz.org/",
+        visits: [{ date: new Date() }],
+      })
+    ).guid;
+    Assert.greater(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "url_hash", {
+        guid: this._pageGuid,
+      }),
+      0
+    );
+    await PlacesUtils.withConnectionWrapper(
+      "change url hash",
+      async function (db) {
+        await db.execute(`UPDATE moz_places SET url_hash = 0`);
+      }
+    );
+    Assert.equal(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "url_hash", {
+        guid: this._pageGuid,
+      }),
+      0
+    );
+  },
+
+  async check() {
+    Assert.greater(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "url_hash", {
+        guid: this._pageGuid,
+      }),
+      0
+    );
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "L.6",
+  desc: "fix invalid Place GUIDs",
+  _placeIds: [],
+
+  async setup() {
+    let placeWithValidGuid = await addPlace(
+      "http://example.com/a",
+      null,
+      "placeAAAAAAA"
+    );
+    this._placeIds.push(placeWithValidGuid);
+
+    let placeWithEmptyGuid = await addPlace("http://example.com/b", null, "");
+    this._placeIds.push(placeWithEmptyGuid);
+
+    let placeWithoutGuid = await addPlace("http://example.com/c", null, null);
+    this._placeIds.push(placeWithoutGuid);
+
+    let placeWithInvalidGuid = await addPlace(
+      "http://example.com/c",
+      null,
+      "{123456}"
+    );
+    this._placeIds.push(placeWithInvalidGuid);
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let updatedRows = await db.execute(
+      `
+      SELECT id, guid
+      FROM moz_places
+      WHERE id IN (?, ?, ?, ?)`,
+      this._placeIds
+    );
+
+    for (let row of updatedRows) {
+      let id = row.getResultByName("id");
+      let guid = row.getResultByName("guid");
+      if (id == this._placeIds[0]) {
+        Assert.equal(guid, "placeAAAAAAA");
+      } else {
+        Assert.ok(PlacesUtils.isValidGuid(guid));
+      }
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "S.1",
+  desc: "fix invalid GUIDs for synced bookmarks",
+  _bookmarkInfos: [],
+
+  async setup() {
+    let folderWithInvalidGuid = await addBookmark(
+      null,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.menuGuid,
+      /* aKeywordId */ null,
+      "NORMAL folder with invalid GUID",
+      "{123456}",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+
+    let placeIdForBookmarkWithoutGuid = await addPlace();
+    let bookmarkWithoutGuid = await addBookmark(
+      placeIdForBookmarkWithoutGuid,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      "{123456}",
+      /* aKeywordId */ null,
+      "NEW bookmark without GUID",
+      /* aGuid */ null
+    );
+
+    let placeIdForBookmarkWithInvalidGuid = await addPlace();
+    let bookmarkWithInvalidGuid = await addBookmark(
+      placeIdForBookmarkWithInvalidGuid,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      "{123456}",
+      /* aKeywordId */ null,
+      "NORMAL bookmark with invalid GUID",
+      "bookmarkAAAA\n",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+
+    let placeIdForBookmarkWithValidGuid = await addPlace();
+    let bookmarkWithValidGuid = await addBookmark(
+      placeIdForBookmarkWithValidGuid,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      "{123456}",
+      /* aKeywordId */ null,
+      "NORMAL bookmark with valid GUID",
+      "bookmarkBBBB",
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    );
+
+    this._bookmarkInfos.push(
+      {
+        id: await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.menuGuid),
+        syncChangeCounter: 1,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      },
+      {
+        id: folderWithInvalidGuid,
+        syncChangeCounter: 3,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      },
+      {
+        id: bookmarkWithoutGuid,
+        syncChangeCounter: 1,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      },
+      {
+        id: bookmarkWithInvalidGuid,
+        syncChangeCounter: 1,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      },
+      {
+        id: bookmarkWithValidGuid,
+        syncChangeCounter: 0,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      }
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let updatedRows = await db.execute(
+      `SELECT id, guid, syncChangeCounter, syncStatus
+       FROM moz_bookmarks
+       WHERE id IN (?, ?, ?, ?, ?)`,
+      this._bookmarkInfos.map(info => info.id)
+    );
+
+    for (let row of updatedRows) {
+      let id = row.getResultByName("id");
+      let guid = row.getResultByName("guid");
+      Assert.ok(PlacesUtils.isValidGuid(guid));
+
+      let cachedGuid = await PlacesUtils.promiseItemGuid(id);
+      Assert.equal(cachedGuid, guid);
+
+      let expectedInfo = this._bookmarkInfos.find(info => info.id == id);
+
+      let syncChangeCounter = row.getResultByName("syncChangeCounter");
+      Assert.equal(syncChangeCounter, expectedInfo.syncChangeCounter);
+
+      let syncStatus = row.getResultByName("syncStatus");
+      Assert.equal(syncStatus, expectedInfo.syncStatus);
+    }
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["bookmarkAAAA\n", "{123456}"]
+    );
+  },
+});
+
+tests.push({
+  name: "S.2",
+  desc: "drop tombstones for bookmarks that aren't deleted",
+
+  async setup() {
+    let placeId = await addPlace();
+    await addBookmark(
+      placeId,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.menuGuid,
+      null,
+      "",
+      "bookmarkAAAA"
+    );
+
+    await PlacesUtils.withConnectionWrapper("Insert tombstones", db =>
+      db.executeTransaction(async function () {
+        for (let guid of ["bookmarkAAAA", "bookmarkBBBB"]) {
+          await db.executeCached(
+            `INSERT INTO moz_bookmarks_deleted(guid)
+             VALUES(:guid)`,
+            { guid }
+          );
+        }
+      })
+    );
+  },
+
+  async check() {
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    Assert.deepEqual(
+      tombstones.map(info => info.guid),
+      ["bookmarkBBBB"]
+    );
+  },
+});
+
+tests.push({
+  name: "S.3",
+  desc: "set missing added and last modified dates",
+  _placeVisits: [],
+  _bookmarksWithDates: [],
+
+  async setup() {
+    let placeIdWithVisits = await addPlace();
+    let placeIdWithZeroVisit = await addPlace();
+    this._placeVisits.push(
+      {
+        placeId: placeIdWithVisits,
+        visitDate: PlacesUtils.toPRTime(new Date(2017, 9, 4)),
+      },
+      {
+        placeId: placeIdWithVisits,
+        visitDate: PlacesUtils.toPRTime(new Date(2017, 9, 8)),
+      },
+      {
+        placeId: placeIdWithZeroVisit,
+        visitDate: 0,
+      }
+    );
+
+    this._bookmarksWithDates.push(
+      {
+        guid: "bookmarkAAAA",
+        placeId: null,
+        parent: PlacesUtils.bookmarks.menuGuid,
+        dateAdded: null,
+        lastModified: PlacesUtils.toPRTime(new Date(2017, 9, 1)),
+      },
+      {
+        guid: "bookmarkBBBB",
+        placeId: null,
+        parent: PlacesUtils.bookmarks.menuGuid,
+        dateAdded: PlacesUtils.toPRTime(new Date(2017, 9, 2)),
+        lastModified: null,
+      },
+      {
+        guid: "bookmarkCCCC",
+        placeId: null,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        dateAdded: null,
+        lastModified: null,
+      },
+      {
+        guid: "bookmarkDDDD",
+        placeId: placeIdWithVisits,
+        parent: PlacesUtils.bookmarks.mobileGuid,
+        dateAdded: null,
+        lastModified: null,
+      },
+      {
+        guid: "bookmarkEEEE",
+        placeId: placeIdWithVisits,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        dateAdded: PlacesUtils.toPRTime(new Date(2017, 9, 3)),
+        lastModified: PlacesUtils.toPRTime(new Date(2017, 9, 6)),
+      },
+      {
+        guid: "bookmarkFFFF",
+        placeId: placeIdWithZeroVisit,
+        parent: PlacesUtils.bookmarks.unfiledGuid,
+        dateAdded: 0,
+        lastModified: 0,
+      }
+    );
+
+    await PlacesUtils.withConnectionWrapper(
+      "S.3: Insert bookmarks and visits",
+      db =>
+        db.executeTransaction(async () => {
+          await db.execute(
+            `INSERT INTO moz_historyvisits(place_id, visit_date)
+             VALUES(:placeId, :visitDate)`,
+            this._placeVisits
+          );
+
+          await db.execute(
+            `INSERT INTO moz_bookmarks(fk, type, parent, guid, dateAdded,
+                                       lastModified)
+             VALUES(:placeId, 1, (SELECT id FROM moz_bookmarks WHERE guid = :parent),
+                    :guid, :dateAdded, :lastModified)`,
+            this._bookmarksWithDates
+          );
+
+          await db.execute(
+            `UPDATE moz_bookmarks SET dateAdded = 0, lastModified = NULL
+             WHERE guid = :toolbarFolder`,
+            { toolbarFolder: PlacesUtils.bookmarks.toolbarGuid }
+          );
+        })
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let updatedRows = await db.execute(
+      `SELECT guid, dateAdded, lastModified
+       FROM moz_bookmarks
+       WHERE guid = :guid`,
+      [
+        { guid: PlacesUtils.bookmarks.toolbarGuid },
+        ...this._bookmarksWithDates.map(({ guid }) => ({ guid })),
+      ]
+    );
+
+    for (let row of updatedRows) {
+      let guid = row.getResultByName("guid");
+
+      let dateAdded = row.getResultByName("dateAdded");
+      Assert.ok(Number.isInteger(dateAdded));
+
+      let lastModified = row.getResultByName("lastModified");
+      Assert.ok(Number.isInteger(lastModified));
+
+      switch (guid) {
+        // Last modified date exists, so we should use it for date added.
+        case "bookmarkAAAA": {
+          let expectedInfo = this._bookmarksWithDates[0];
+          Assert.equal(dateAdded, expectedInfo.lastModified);
+          Assert.equal(lastModified, expectedInfo.lastModified);
+          break;
+        }
+
+        // Date added exists, so we should use it for last modified date.
+        case "bookmarkBBBB": {
+          let expectedInfo = this._bookmarksWithDates[1];
+          Assert.equal(dateAdded, expectedInfo.dateAdded);
+          Assert.equal(lastModified, expectedInfo.dateAdded);
+          break;
+        }
+
+        // C has no visits, date added, or last modified time, F has zeros for
+        // all, and the toolbar has a zero date added and no last modified time.
+        // In all cases, we should fall back to the current time.
+        case "bookmarkCCCC":
+        case "bookmarkFFFF":
+        case PlacesUtils.bookmarks.toolbarGuid: {
+          let nowAsPRTime = PlacesUtils.toPRTime(new Date());
+          Assert.greater(dateAdded, 0);
+          Assert.equal(dateAdded, lastModified);
+          Assert.ok(dateAdded <= nowAsPRTime);
+          break;
+        }
+
+        // Neither date added nor last modified exists, but we have two
+        // visits, so we should fall back to the earliest and latest visit
+        // dates.
+        case "bookmarkDDDD": {
+          let oldestVisit = this._placeVisits[0];
+          Assert.equal(dateAdded, oldestVisit.visitDate);
+          let newestVisit = this._placeVisits[1];
+          Assert.equal(lastModified, newestVisit.visitDate);
+          break;
+        }
+
+        // We have two visits, but both date added and last modified exist,
+        // so we shouldn't update them.
+        case "bookmarkEEEE": {
+          let expectedInfo = this._bookmarksWithDates[4];
+          Assert.equal(dateAdded, expectedInfo.dateAdded);
+          Assert.equal(lastModified, expectedInfo.lastModified);
+          break;
+        }
+
+        default:
+          throw new Error(`Unexpected row for bookmark ${guid}`);
+      }
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "S.4",
+  desc: "reset added dates that are ahead of last modified dates",
+  _bookmarksWithDates: [],
+
+  async setup() {
+    this._bookmarksWithDates.push({
+      guid: "bookmarkGGGG",
+      parent: PlacesUtils.bookmarks.unfiledGuid,
+      dateAdded: PlacesUtils.toPRTime(new Date(2017, 9, 6)),
+      lastModified: PlacesUtils.toPRTime(new Date(2017, 9, 3)),
+    });
+
+    await PlacesUtils.withConnectionWrapper(
+      "S.4: Insert bookmarks and visits",
+      db =>
+        db.executeTransaction(async () => {
+          await db.execute(
+            `INSERT INTO moz_bookmarks(type, parent, guid, dateAdded,
+                                       lastModified)
+             VALUES(1, (SELECT id FROM moz_bookmarks WHERE guid = :parent),
+                    :guid, :dateAdded, :lastModified)`,
+            this._bookmarksWithDates
+          );
+        })
+    );
+  },
+
+  async check() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let updatedRows = await db.execute(
+      `SELECT guid, dateAdded, lastModified
+       FROM moz_bookmarks
+       WHERE guid = :guid`,
+      this._bookmarksWithDates.map(({ guid }) => ({ guid }))
+    );
+
+    for (let row of updatedRows) {
+      let guid = row.getResultByName("guid");
+      let dateAdded = row.getResultByName("dateAdded");
+      let lastModified = row.getResultByName("lastModified");
+      switch (guid) {
+        case "bookmarkGGGG": {
+          let expectedInfo = this._bookmarksWithDates[0];
+          Assert.equal(dateAdded, expectedInfo.lastModified);
+          Assert.equal(lastModified, expectedInfo.lastModified);
+          break;
+        }
+
+        default:
+          throw new Error(`Unexpected row for bookmark ${guid}`);
+      }
+    }
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "T.1",
+  desc: "history.recalculateOriginFrecencyStats() is called",
+
+  async setup() {
+    let urls = [
+      "http://example1.com/",
+      "http://example2.com/",
+      "http://example3.com/",
+    ];
+    await PlacesTestUtils.addVisits(urls.map(u => ({ uri: u })));
+
+    this._frecencies = [];
+    for (let url of urls) {
+      this._frecencies.push(
+        await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+          url,
+        })
+      );
+    }
+
+    let stats = await this._promiseStats();
+    Assert.equal(stats.count, this._frecencies.length, "Sanity check");
+    Assert.equal(stats.sum, this._sum(this._frecencies), "Sanity check");
+    Assert.equal(
+      stats.squares,
+      this._squares(this._frecencies),
+      "Sanity check"
+    );
+
+    await PlacesUtils.withConnectionWrapper("T.1", db =>
+      db.execute(`
+        INSERT OR REPLACE INTO moz_meta VALUES
+        ('origin_frecency_count', 99),
+        ('origin_frecency_sum', 99999),
+        ('origin_frecency_sum_of_squares', 99999 * 99999);
+      `)
+    );
+
+    stats = await this._promiseStats();
+    Assert.equal(stats.count, 99);
+    Assert.equal(stats.sum, 99999);
+    Assert.equal(stats.squares, 99999 * 99999);
+  },
+
+  async check() {
+    let stats = await this._promiseStats();
+    Assert.equal(stats.count, this._frecencies.length);
+    Assert.equal(stats.sum, this._sum(this._frecencies));
+    Assert.equal(stats.squares, this._squares(this._frecencies));
+  },
+
+  _sum(frecs) {
+    return frecs.reduce((memo, f) => memo + f, 0);
+  },
+
+  _squares(frecs) {
+    return frecs.reduce((memo, f) => memo + f * f, 0);
+  },
+
+  async _promiseStats() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(`
+      SELECT
+        IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_count'), 0),
+        IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_sum'), 0),
+        IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_sum_of_squares'), 0)
+    `);
+    return {
+      count: rows[0].getResultByIndex(0),
+      sum: rows[0].getResultByIndex(1),
+      squares: rows[0].getResultByIndex(2),
+    };
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+tests.push({
+  name: "Z",
+  desc: "Sanity: Preventive maintenance does not touch valid items",
+
+  _uri1: uri("http://www1.mozilla.org"),
+  _uri2: uri("http://www2.mozilla.org"),
+  _folder: null,
+  _bookmark: null,
+  _bookmarkId: null,
+  _separator: null,
+
+  async setup() {
+    // use valid api calls to create a bunch of items
+    await PlacesTestUtils.addVisits([{ uri: this._uri1 }, { uri: this._uri2 }]);
+
+    let bookmarks = await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      children: [
+        {
+          title: "testfolder",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          children: [
+            {
+              title: "testbookmark",
+              url: this._uri1,
+            },
+          ],
+        },
+      ],
+    });
+
+    this._folder = bookmarks[0];
+    this._bookmark = bookmarks[1];
+    this._bookmarkId = await PlacesUtils.promiseItemId(bookmarks[1].guid);
+
+    this._separator = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+    });
+
+    PlacesUtils.tagging.tagURI(this._uri1, ["testtag"]);
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      this._uri2,
+      SMALLPNG_DATA_URI,
+      false,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      null,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+    await PlacesUtils.keywords.insert({
+      url: this._uri1.spec,
+      keyword: "testkeyword",
+    });
+    await PlacesUtils.history.update({
+      url: this._uri2,
+      annotations: new Map([["anno", "anno"]]),
+    });
+  },
+
+  async check() {
+    // Check that all items are correct
+    let isVisited = await PlacesUtils.history.hasVisits(this._uri1);
+    Assert.ok(isVisited);
+    isVisited = await PlacesUtils.history.hasVisits(this._uri2);
+    Assert.ok(isVisited);
+
+    Assert.equal(
+      (await PlacesUtils.bookmarks.fetch(this._bookmark.guid)).url,
+      this._uri1.spec
+    );
+    let folder = await PlacesUtils.bookmarks.fetch(this._folder.guid);
+    Assert.equal(folder.index, 0);
+    Assert.equal(folder.type, PlacesUtils.bookmarks.TYPE_FOLDER);
+    Assert.equal(
+      (await PlacesUtils.bookmarks.fetch(this._separator.guid)).type,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR
+    );
+
+    Assert.equal(PlacesUtils.tagging.getTagsForURI(this._uri1).length, 1);
+    Assert.equal(
+      (await PlacesUtils.keywords.fetch({ url: this._uri1.spec })).keyword,
+      "testkeyword"
+    );
+    let pageInfo = await PlacesUtils.history.fetch(this._uri2, {
+      includeAnnotations: true,
+    });
+    Assert.equal(pageInfo.annotations.get("anno"), "anno");
+
+    await new Promise(resolve => {
+      PlacesUtils.favicons.getFaviconURLForPage(this._uri2, aFaviconURI => {
+        Assert.ok(aFaviconURI.equals(SMALLPNG_DATA_URI));
+        resolve();
+      });
+    });
+  },
+});
+
+// ------------------------------------------------------------------------------
+
+add_task(async function test_preventive_maintenance() {
+  let db = await PlacesUtils.promiseDBConnection();
+  // Get current bookmarks max ID for cleanup
+  defaultBookmarksMaxId = (
+    await db.executeCached("SELECT MAX(id) FROM moz_bookmarks")
+  )[0].getResultByIndex(0);
+  Assert.ok(defaultBookmarksMaxId > 0);
+
+  for (let test of tests) {
+    await PlacesTestUtils.markBookmarksAsSynced();
+
+    info("\nExecuting test: " + test.name + "\n*** " + test.desc + "\n");
+    await test.setup();
+
+    Services.prefs.clearUserPref("places.database.lastMaintenance");
+    await PlacesDBUtils.maintenanceOnIdle();
+
+    // Check the lastMaintenance time has been saved.
+    Assert.notEqual(
+      Services.prefs.getIntPref("places.database.lastMaintenance"),
+      null
+    );
+
+    await test.check();
+
+    await cleanDatabase();
+  }
+
+  // Sanity check: all roots should be intact
+  Assert.strictEqual(
+    (await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.rootGuid))
+      .parentGuid,
+    undefined
+  );
+  Assert.deepEqual(
+    (await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.menuGuid))
+      .parentGuid,
+    PlacesUtils.bookmarks.rootGuid
+  );
+  Assert.deepEqual(
+    (await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.tagsGuid))
+      .parentGuid,
+    PlacesUtils.bookmarks.rootGuid
+  );
+  Assert.deepEqual(
+    (await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.unfiledGuid))
+      .parentGuid,
+    PlacesUtils.bookmarks.rootGuid
+  );
+  Assert.deepEqual(
+    (await PlacesUtils.bookmarks.fetch(PlacesUtils.bookmarks.toolbarGuid))
+      .parentGuid,
+    PlacesUtils.bookmarks.rootGuid
+  );
+});
+
+// ------------------------------------------------------------------------------
+
+add_task(async function test_idle_daily() {
+  const { sinon } = ChromeUtils.importESModule(
+    "resource://testing-common/Sinon.sys.mjs"
+  );
+  const sandbox = sinon.createSandbox();
+  sandbox.stub(PlacesDBUtils, "maintenanceOnIdle");
+  Services.prefs.clearUserPref("places.database.lastMaintenance");
+  Cc["@mozilla.org/places/databaseUtilsIdleMaintenance;1"]
+    .getService(Ci.nsIObserver)
+    .observe(null, "idle-daily", "");
+  Assert.ok(
+    PlacesDBUtils.maintenanceOnIdle.calledOnce,
+    "maintenanceOnIdle was invoked"
+  );
+  sandbox.restore();
+});
diff --git a/toolkit/components/places/tests/maintenance/test_preventive_maintenance_checkAndFixDatabase.js b/toolkit/components/places/tests/maintenance/test_preventive_maintenance_checkAndFixDatabase.js
new file mode 100644
index 0000000000..db9466b784
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_preventive_maintenance_checkAndFixDatabase.js
@@ -0,0 +1,36 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Test preventive maintenance checkAndFixDatabase.
+ */
+
+add_task(async function () {
+  // We must initialize places first, or we won't have a db to check.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CREATE
+  );
+
+  let tasksStatusMap = await PlacesDBUtils.checkAndFixDatabase();
+  let numberOfTasksRun = tasksStatusMap.size;
+  let successfulTasks = [];
+  let failedTasks = [];
+  tasksStatusMap.forEach(val => {
+    if (val.succeeded && val.logs) {
+      successfulTasks.push(val);
+    } else {
+      failedTasks.push(val);
+    }
+  });
+  Assert.equal(numberOfTasksRun, 8, "Check that we have run all tasks.");
+  Assert.equal(
+    successfulTasks.length,
+    8,
+    "Check that we have run all tasks successfully"
+  );
+  Assert.equal(failedTasks.length, 0, "Check that no task is failing");
+});
diff --git a/toolkit/components/places/tests/maintenance/test_preventive_maintenance_runTasks.js b/toolkit/components/places/tests/maintenance/test_preventive_maintenance_runTasks.js
new file mode 100644
index 0000000000..6740acae57
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/test_preventive_maintenance_runTasks.js
@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Test preventive maintenance runTasks.
+ */
+
+add_task(async function () {
+  let tasksStatusMap = await PlacesDBUtils.runTasks([
+    PlacesDBUtils.invalidateCaches,
+  ]);
+  let numberOfTasksRun = tasksStatusMap.size;
+  let successfulTasks = [];
+  let failedTasks = [];
+  tasksStatusMap.forEach(val => {
+    if (val.succeeded) {
+      successfulTasks.push(val);
+    } else {
+      failedTasks.push(val);
+    }
+  });
+
+  Assert.equal(numberOfTasksRun, 1, "Check that we have run all tasks.");
+  Assert.equal(
+    successfulTasks.length,
+    1,
+    "Check that we have run all tasks successfully"
+  );
+  Assert.equal(failedTasks.length, 0, "Check that no task is failing");
+});
diff --git a/toolkit/components/places/tests/maintenance/xpcshell.ini b/toolkit/components/places/tests/maintenance/xpcshell.ini
new file mode 100644
index 0000000000..f6e2148024
--- /dev/null
+++ b/toolkit/components/places/tests/maintenance/xpcshell.ini
@@ -0,0 +1,20 @@
+[DEFAULT]
+head = head.js
+firefox-appdir = browser
+support-files =
+  corruptDB.sqlite
+  corruptPayload.sqlite
+
+[test_corrupt_favicons.js]
+[test_corrupt_favicons_schema.js]
+[test_corrupt_places_schema.js]
+[test_corrupt_telemetry.js]
+[test_favicons_replaceOnStartup.js]
+[test_favicons_replaceOnStartup_clone.js]
+[test_integrity_replacement.js]
+[test_places_purge_caches.js]
+[test_places_replaceOnStartup.js]
+[test_places_replaceOnStartup_clone.js]
+[test_preventive_maintenance.js]
+[test_preventive_maintenance_checkAndFixDatabase.js]
+[test_preventive_maintenance_runTasks.js]
diff --git a/toolkit/components/places/tests/migration/favicons_v41.sqlite b/toolkit/components/places/tests/migration/favicons_v41.sqlite
new file mode 100644
index 0000000000..a59d9d286f
Binary files /dev/null and b/toolkit/components/places/tests/migration/favicons_v41.sqlite differ
diff --git a/toolkit/components/places/tests/migration/head_migration.js b/toolkit/components/places/tests/migration/head_migration.js
new file mode 100644
index 0000000000..27fb06a3ef
--- /dev/null
+++ b/toolkit/components/places/tests/migration/head_migration.js
@@ -0,0 +1,47 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+const CURRENT_SCHEMA_VERSION = 74;
+const FIRST_UPGRADABLE_SCHEMA_VERSION = 43;
+
+async function assertAnnotationsRemoved(db, expectedAnnos) {
+  for (let anno of expectedAnnos) {
+    let rows = await db.execute(
+      `
+      SELECT id FROM moz_anno_attributes
+      WHERE name = :anno
+    `,
+      { anno }
+    );
+
+    Assert.equal(rows.length, 0, `${anno} should not exist in the database`);
+  }
+}
+
+async function assertNoOrphanAnnotations(db) {
+  let rows = await db.execute(`
+    SELECT item_id FROM moz_items_annos
+    WHERE item_id NOT IN (SELECT id from moz_bookmarks)
+  `);
+
+  Assert.equal(rows.length, 0, `Should have no orphan annotations.`);
+
+  rows = await db.execute(`
+    SELECT id FROM moz_anno_attributes
+    WHERE id NOT IN (SELECT id from moz_items_annos)
+  `);
+
+  Assert.equal(rows.length, 0, `Should have no orphan annotation attributes.`);
+}
diff --git a/toolkit/components/places/tests/migration/places_outdated.sqlite b/toolkit/components/places/tests/migration/places_outdated.sqlite
new file mode 100644
index 0000000000..2852a4cf97
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_outdated.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v43.sqlite b/toolkit/components/places/tests/migration/places_v43.sqlite
new file mode 100644
index 0000000000..9210f215fa
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v43.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v54.sqlite b/toolkit/components/places/tests/migration/places_v54.sqlite
new file mode 100644
index 0000000000..a203b28c10
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v54.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v66.sqlite b/toolkit/components/places/tests/migration/places_v66.sqlite
new file mode 100644
index 0000000000..9578ee11e6
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v66.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v68.sqlite b/toolkit/components/places/tests/migration/places_v68.sqlite
new file mode 100644
index 0000000000..414fa170ec
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v68.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v69.sqlite b/toolkit/components/places/tests/migration/places_v69.sqlite
new file mode 100644
index 0000000000..bc3053c18e
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v69.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v70.sqlite b/toolkit/components/places/tests/migration/places_v70.sqlite
new file mode 100644
index 0000000000..907e7f5046
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v70.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v72.sqlite b/toolkit/components/places/tests/migration/places_v72.sqlite
new file mode 100644
index 0000000000..59d0d8fdab
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v72.sqlite differ
diff --git a/toolkit/components/places/tests/migration/places_v74.sqlite b/toolkit/components/places/tests/migration/places_v74.sqlite
new file mode 100644
index 0000000000..e7078a054f
Binary files /dev/null and b/toolkit/components/places/tests/migration/places_v74.sqlite differ
diff --git a/toolkit/components/places/tests/migration/test_current_from_downgraded.js b/toolkit/components/places/tests/migration/test_current_from_downgraded.js
new file mode 100644
index 0000000000..5daec14e2f
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_downgraded.js
@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures we can pass twice through migration methods without
+// failing, that is what happens in case of a downgrade followed by an upgrade.
+
+add_task(async function setup() {
+  let dbFile = PathUtils.join(
+    do_get_cwd().path,
+    `places_v${CURRENT_SCHEMA_VERSION}.sqlite`
+  );
+  Assert.ok(await IOUtils.exists(dbFile));
+  await setupPlacesDatabase(`places_v${CURRENT_SCHEMA_VERSION}.sqlite`);
+  // Downgrade the schema version to the first supported one.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+  await db.setSchemaVersion(FIRST_UPGRADABLE_SCHEMA_VERSION);
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_outdated.js b/toolkit/components/places/tests/migration/test_current_from_outdated.js
new file mode 100644
index 0000000000..e7fad5b3a4
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_outdated.js
@@ -0,0 +1,47 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests migration from a preliminary schema version 6 that
+ * lacks frecency column and moz_inputhistory table.
+ */
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_outdated.sqlite");
+});
+
+add_task(async function corrupt_database_not_exists() {
+  let corruptPath = PathUtils.join(
+    PathUtils.profileDir,
+    "places.sqlite.corrupt"
+  );
+  Assert.ok(
+    !(await IOUtils.exists(corruptPath)),
+    "Corrupt file should not exist"
+  );
+});
+
+add_task(async function database_is_valid() {
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_CORRUPT
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function check_columns() {
+  // Check the database has been replaced, these would throw otherwise.
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute("SELECT frecency from moz_places");
+  await db.execute("SELECT 1 from moz_inputhistory");
+});
+
+add_task(async function corrupt_database_exists() {
+  let corruptPath = PathUtils.join(
+    PathUtils.profileDir,
+    "places.sqlite.corrupt"
+  );
+  Assert.ok(await IOUtils.exists(corruptPath), "Corrupt file should exist");
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v43.js b/toolkit/components/places/tests/migration/test_current_from_v43.js
new file mode 100644
index 0000000000..70a383bb2e
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v43.js
@@ -0,0 +1,253 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const EXPECTED_REMAINING_ROOTS = [
+  ...PlacesUtils.bookmarks.userContentRoots,
+  PlacesUtils.bookmarks.tagsGuid,
+];
+
+const EXPECTED_REMOVED_BOOKMARK_GUIDS = [
+  // These first ones are the old left-pane folder queries
+  "SNLmwJH6GtW9", // Root Query
+  "r0dY_2_y4mlx", // History
+  "xGGhZK3b6GnW", // Downloads
+  "EJG6I1nKkQFQ", // Tags
+  "gSyHo5oNSUJV", // All Bookmarks
+  // These are simulated add-on injections that we expect to be removed.
+  "exaddon_____",
+  "exaddon1____",
+  "exaddon2____",
+  "exaddon3____",
+  "test________",
+];
+
+const EXPECTED_REMOVED_ANNOTATIONS = [
+  "PlacesOrganizer/OrganizerFolder",
+  "PlacesOrganizer/OrganizerQuery",
+];
+
+const EXPECTED_REMOVED_PLACES_ENTRIES = ["exaddonh____", "exaddonh3___"];
+const EXPECTED_KEPT_PLACES_ENTRY = "exaddonh2___";
+const EXPECTED_REMOVED_KEYWORDS = ["exaddon", "exaddon2"];
+
+async function assertItemIn(db, table, field, expectedItems) {
+  let rows = await db.execute(`SELECT ${field} from ${table}`);
+
+  Assert.ok(
+    rows.length >= expectedItems.length,
+    "Should be at least the number of annotations we expect to be removed."
+  );
+
+  let fieldValues = rows.map(row => row.getResultByName(field));
+
+  for (let item of expectedItems) {
+    Assert.ok(
+      fieldValues.includes(item),
+      `${table} should have ${expectedItems}`
+    );
+  }
+}
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+
+  // Setup database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+
+  let rows = await db.execute(`SELECT * FROM moz_bookmarks_deleted`);
+  Assert.equal(rows.length, 0, "Should be nothing in moz_bookmarks_deleted");
+
+  // Break roots parenting, to test for Bug 1472127.
+  await db.execute(`INSERT INTO moz_bookmarks (title, parent, position, guid)
+                    VALUES ("test", 1, 0, "test________")`);
+  await db.execute(`UPDATE moz_bookmarks
+                    SET parent = (SELECT id FROM moz_bookmarks WHERE guid = "test________")
+                    WHERE guid = "menu________"`);
+
+  await assertItemIn(
+    db,
+    "moz_anno_attributes",
+    "name",
+    EXPECTED_REMOVED_ANNOTATIONS
+  );
+  await assertItemIn(
+    db,
+    "moz_bookmarks",
+    "guid",
+    EXPECTED_REMOVED_BOOKMARK_GUIDS
+  );
+  await assertItemIn(db, "moz_keywords", "keyword", EXPECTED_REMOVED_KEYWORDS);
+  await assertItemIn(db, "moz_places", "guid", EXPECTED_REMOVED_PLACES_ENTRIES);
+
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function test_roots_removed() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(
+    `
+    SELECT id FROM moz_bookmarks
+    WHERE guid = :guid
+  `,
+    { guid: PlacesUtils.bookmarks.rootGuid }
+  );
+  Assert.equal(rows.length, 1, "Should have exactly one root row.");
+  let rootId = rows[0].getResultByName("id");
+
+  rows = await db.execute(
+    `
+    SELECT guid FROM moz_bookmarks
+    WHERE parent = :rootId`,
+    { rootId }
+  );
+
+  Assert.equal(
+    rows.length,
+    EXPECTED_REMAINING_ROOTS.length,
+    "Should only have the built-in folder roots."
+  );
+
+  for (let row of rows) {
+    let guid = row.getResultByName("guid");
+    Assert.ok(
+      EXPECTED_REMAINING_ROOTS.includes(guid),
+      `Should have only the expected guids remaining, unexpected guid: ${guid}`
+    );
+  }
+
+  // Check the reparented menu now.
+  rows = await db.execute(
+    `
+    SELECT id, parent FROM moz_bookmarks
+    WHERE guid = :guid
+  `,
+    { guid: PlacesUtils.bookmarks.menuGuid }
+  );
+  Assert.equal(rows.length, 1, "Should have found the menu root.");
+  Assert.equal(
+    rows[0].getResultByName("parent"),
+    await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.rootGuid),
+    "Should have moved the menu back to the Places root."
+  );
+});
+
+add_task(async function test_tombstones_added() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let rows = await db.execute(`
+    SELECT guid FROM moz_bookmarks_deleted
+  `);
+
+  for (let row of rows) {
+    let guid = row.getResultByName("guid");
+    Assert.ok(
+      EXPECTED_REMOVED_BOOKMARK_GUIDS.includes(guid),
+      `Should have tombstoned the expected guids, unexpected guid: ${guid}`
+    );
+  }
+
+  Assert.equal(
+    rows.length,
+    EXPECTED_REMOVED_BOOKMARK_GUIDS.length,
+    "Should have removed all the expected bookmarks."
+  );
+});
+
+add_task(async function test_annotations_removed() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  await assertAnnotationsRemoved(db, EXPECTED_REMOVED_ANNOTATIONS);
+});
+
+add_task(async function test_check_history_entries() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  for (let entry of EXPECTED_REMOVED_PLACES_ENTRIES) {
+    let rows = await db.execute(`
+      SELECT id FROM moz_places
+      WHERE guid = '${entry}'`);
+
+    Assert.equal(
+      rows.length,
+      0,
+      `Should have removed an orphaned history entry ${EXPECTED_REMOVED_PLACES_ENTRIES}.`
+    );
+  }
+
+  let rows = await db.execute(
+    `
+    SELECT foreign_count FROM moz_places
+    WHERE guid = :guid
+  `,
+    { guid: EXPECTED_KEPT_PLACES_ENTRY }
+  );
+
+  Assert.equal(
+    rows.length,
+    1,
+    `Should have kept visited history entry ${EXPECTED_KEPT_PLACES_ENTRY}`
+  );
+
+  let foreignCount = rows[0].getResultByName("foreign_count");
+  Assert.equal(
+    foreignCount,
+    0,
+    `Should have updated the foreign_count for ${EXPECTED_KEPT_PLACES_ENTRY}`
+  );
+});
+
+add_task(async function test_check_keyword_removed() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  for (let keyword of EXPECTED_REMOVED_KEYWORDS) {
+    let rows = await db.execute(
+      `
+      SELECT keyword FROM moz_keywords
+      WHERE keyword = :keyword
+    `,
+      { keyword }
+    );
+
+    Assert.equal(
+      rows.length,
+      0,
+      `Should have removed the expected keyword: ${keyword}.`
+    );
+  }
+});
+
+add_task(async function test_no_orphan_annotations() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  await assertNoOrphanAnnotations(db);
+});
+
+add_task(async function test_no_orphan_keywords() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let rows = await db.execute(`
+    SELECT place_id FROM moz_keywords
+    WHERE place_id NOT IN (SELECT id from moz_places)
+  `);
+
+  Assert.equal(rows.length, 0, `Should have no orphan keywords.`);
+});
+
+add_task(async function test_meta_exists() {
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute(`SELECT 1 FROM moz_meta`);
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v45.js b/toolkit/components/places/tests/migration/test_current_from_v45.js
new file mode 100644
index 0000000000..af940d75d4
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v45.js
@@ -0,0 +1,100 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+let gTags = [
+  {
+    folder: 123456,
+    url: "place:folder=123456&type=7&queryType=1",
+    title: "tag1",
+    hash: "268505532566465",
+  },
+  {
+    folder: 234567,
+    url: "place:folder=234567&type=7&queryType=1&somethingelse",
+    title: "tag2",
+    hash: "268506675127932",
+  },
+  {
+    folder: 345678,
+    url: "place:type=7&folder=345678&queryType=1",
+    title: "tag3",
+    hash: "268506471927988",
+  },
+  // This will point to an invalid folder id.
+  {
+    folder: 456789,
+    url: "place:type=7&folder=456789&queryType=1",
+    expectedUrl:
+      "place:type=7&invalidOldParentId=456789&queryType=1&excludeItems=1",
+    title: "invalid",
+    hash: "268505972797836",
+  },
+];
+gTags.forEach(t => (t.guid = t.title.padEnd(12, "_")));
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+
+  // Setup database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+
+  for (let tag of gTags) {
+    // We can reuse the same guid, it doesn't matter for this test.
+    await db.execute(
+      `INSERT INTO moz_places (url, guid, url_hash)
+                      VALUES (:url, :guid, :hash)
+                     `,
+      { url: tag.url, guid: tag.guid, hash: tag.hash }
+    );
+    if (tag.title != "invalid") {
+      await db.execute(
+        `INSERT INTO moz_bookmarks (id, fk, guid, title)
+                      VALUES (:id, (SELECT id FROM moz_places WHERE guid = :guid), :guid, :title)
+                      `,
+        { id: tag.folder, guid: tag.guid, title: tag.title }
+      );
+    }
+  }
+
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function test_queries_converted() {
+  for (let tag of gTags) {
+    let url =
+      tag.title == "invalid" ? tag.expectedUrl : "place:tag=" + tag.title;
+    let page = await PlacesUtils.history.fetch(tag.guid);
+    Assert.equal(page.url.href, url);
+  }
+});
+
+add_task(async function test_sync_fields() {
+  let db = await PlacesUtils.promiseDBConnection();
+  for (let tag of gTags) {
+    if (tag.title != "invalid") {
+      let rows = await db.execute(
+        `
+        SELECT syncChangeCounter
+        FROM moz_bookmarks
+        WHERE guid = :guid
+      `,
+        { guid: tag.guid }
+      );
+      Assert.equal(rows[0].getResultByIndex(0), 2);
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v46.js b/toolkit/components/places/tests/migration/test_current_from_v46.js
new file mode 100644
index 0000000000..a613a3027e
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v46.js
@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+let guid = "null".padEnd(12, "_");
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+
+  // Setup database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+  // We can reuse the same guid, it doesn't matter for this test.
+
+  await db.execute(
+    `INSERT INTO moz_places (url, guid, url_hash)
+                    VALUES (NULL, :guid, "123456")`,
+    { guid }
+  );
+  await db.execute(
+    `INSERT INTO moz_bookmarks (fk, guid)
+                    VALUES ((SELECT id FROM moz_places WHERE guid = :guid), :guid)
+                    `,
+    { guid }
+  );
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  let page = await PlacesUtils.history.fetch(guid);
+  Assert.equal(page.url.href, "place:excludeItems=1");
+
+  let rows = await db.execute(
+    `
+    SELECT syncChangeCounter
+    FROM moz_bookmarks
+    WHERE guid = :guid
+  `,
+    { guid }
+  );
+  Assert.equal(rows[0].getResultByIndex(0), 2);
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v47.js b/toolkit/components/places/tests/migration/test_current_from_v47.js
new file mode 100644
index 0000000000..b3d5f47211
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v47.js
@@ -0,0 +1,128 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+});
+
+// Accessing the database for the first time should trigger migration, and the
+// schema version should be updated.
+add_task(async function database_is_valid() {
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  // Now wait for moz_origins.frecency to be populated before continuing with
+  // other test tasks.
+  await TestUtils.waitForCondition(
+    () => {
+      return !Services.prefs.getBoolPref(
+        "places.database.migrateV52OriginFrecencies",
+        false
+      );
+    },
+    "Waiting for v52 origin frecencies to be migrated",
+    100,
+    3000
+  );
+});
+
+// moz_origins should be populated.
+add_task(async function test_origins() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  // Collect origins.
+  let rows = await db.execute(`
+    SELECT id, prefix, host, frecency
+    FROM moz_origins
+    ORDER BY id ASC;
+  `);
+  Assert.notEqual(rows.length, 0);
+  let origins = rows.map(r => ({
+    id: r.getResultByName("id"),
+    prefix: r.getResultByName("prefix"),
+    host: r.getResultByName("host"),
+    frecency: r.getResultByName("frecency"),
+  }));
+
+  // Get moz_places.
+  rows = await db.execute(`
+    SELECT get_prefix(url) AS prefix, get_host_and_port(url) AS host,
+           origin_id, frecency
+    FROM moz_places;
+  `);
+  Assert.notEqual(rows.length, 0);
+
+  let seenOriginIDs = [];
+  let frecenciesByOriginID = {};
+
+  // Make sure moz_places.origin_id refers to the right origins.
+  for (let row of rows) {
+    let originID = row.getResultByName("origin_id");
+    let origin = origins.find(o => o.id == originID);
+    Assert.ok(origin);
+    Assert.equal(origin.prefix, row.getResultByName("prefix"));
+    Assert.equal(origin.host, row.getResultByName("host"));
+
+    seenOriginIDs.push(originID);
+
+    let frecency = row.getResultByName("frecency");
+    frecenciesByOriginID[originID] = frecenciesByOriginID[originID] || 0;
+    frecenciesByOriginID[originID] += frecency;
+  }
+
+  for (let origin of origins) {
+    // Make sure each origin corresponds to at least one moz_place.
+    Assert.ok(seenOriginIDs.includes(origin.id));
+
+    // moz_origins.frecency should be the sum of frecencies of all moz_places
+    // with the origin.
+    Assert.equal(origin.frecency, frecenciesByOriginID[origin.id]);
+  }
+
+  // Make sure moz_hosts was emptied.
+  rows = await db.execute(`
+    SELECT *
+    FROM moz_hosts;
+  `);
+  Assert.equal(rows.length, 0);
+});
+
+// Frecency stats should have been collected.
+add_task(async function test_frecency_stats() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  // Collect positive frecencies from moz_origins.
+  let rows = await db.execute(`
+    SELECT frecency FROM moz_origins WHERE frecency > 0
+  `);
+  Assert.notEqual(rows.length, 0);
+  let frecencies = rows.map(r => r.getResultByName("frecency"));
+
+  // Collect stats.
+  rows = await db.execute(`
+    SELECT
+      (SELECT value FROM moz_meta WHERE key = "origin_frecency_count"),
+      (SELECT value FROM moz_meta WHERE key = "origin_frecency_sum"),
+      (SELECT value FROM moz_meta WHERE key = "origin_frecency_sum_of_squares")
+  `);
+  let count = rows[0].getResultByIndex(0);
+  let sum = rows[0].getResultByIndex(1);
+  let squares = rows[0].getResultByIndex(2);
+
+  Assert.equal(count, frecencies.length);
+  Assert.equal(
+    sum,
+    frecencies.reduce((memo, f) => memo + f, 0)
+  );
+  Assert.equal(
+    squares,
+    frecencies.reduce((memo, f) => memo + f * f, 0)
+  );
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v48.js b/toolkit/components/places/tests/migration/test_current_from_v48.js
new file mode 100644
index 0000000000..f2c7c683ed
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v48.js
@@ -0,0 +1,190 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const gCreatedParentGuid = "m47___FOLDER";
+
+const gTestItems = [
+  {
+    // Folder shortcuts to built-in folders.
+    guid: "m47_____ROOT",
+    url: "place:folder=PLACES_ROOT",
+    targetParentGuid: "rootGuid",
+  },
+  {
+    guid: "m47_____MENU",
+    url: "place:folder=BOOKMARKS_MENU",
+    targetParentGuid: "menuGuid",
+  },
+  {
+    guid: "m47_____TAGS",
+    url: "place:folder=TAGS",
+    targetParentGuid: "tagsGuid",
+  },
+  {
+    guid: "m47____OTHER",
+    url: "place:folder=UNFILED_BOOKMARKS",
+    targetParentGuid: "unfiledGuid",
+  },
+  {
+    guid: "m47__TOOLBAR",
+    url: "place:folder=TOOLBAR",
+    targetParentGuid: "toolbarGuid",
+  },
+  {
+    guid: "m47___MOBILE",
+    url: "place:folder=MOBILE_BOOKMARKS",
+    targetParentGuid: "mobileGuid",
+  },
+  {
+    // Folder shortcut to using id.
+    guid: "m47_______ID",
+    url: "place:folder=%id%",
+    expectedUrl: "place:parent=%guid%",
+  },
+  {
+    // Folder shortcut to multiple folders.
+    guid: "m47____MULTI",
+    url: "place:folder=TOOLBAR&folder=%id%&sort=1",
+    expectedUrl: "place:parent=%toolbarGuid%&parent=%guid%&sort=1",
+  },
+  {
+    // Folder shortcut to non-existent folder.
+    guid: "m47______NON",
+    url: "place:folder=454554545",
+    expectedUrl: "place:invalidOldParentId=454554545&excludeItems=1",
+  },
+];
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+
+  // Setup database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+
+  let rows = await db.execute(
+    `SELECT id FROM moz_bookmarks
+                               WHERE guid = :guid`,
+    { guid: PlacesUtils.bookmarks.unfiledGuid }
+  );
+
+  let unfiledId = rows[0].getResultByName("id");
+
+  // Insert a test folder.
+  await db.execute(
+    `INSERT INTO moz_bookmarks (guid, title, parent)
+                    VALUES (:guid, "Folder", :parent)`,
+    { guid: gCreatedParentGuid, parent: unfiledId }
+  );
+
+  rows = await db.execute(
+    `SELECT id FROM moz_bookmarks
+                           WHERE guid = :guid`,
+    { guid: gCreatedParentGuid }
+  );
+
+  let createdFolderId = rows[0].getResultByName("id");
+
+  for (let item of gTestItems) {
+    item.url = item.url.replace("%id%", createdFolderId);
+
+    // We can reuse the same guid, it doesn't matter for this test.
+    await db.execute(
+      `INSERT INTO moz_places (url, guid, url_hash)
+                      VALUES (:url, :guid, :hash)
+                     `,
+      {
+        url: item.url,
+        guid: item.guid,
+        hash: PlacesUtils.history.hashURL(item.url),
+      }
+    );
+    await db.execute(
+      `INSERT INTO moz_bookmarks (id, fk, guid, title, parent)
+                      VALUES (:id, (SELECT id FROM moz_places WHERE guid = :guid),
+                              :guid, :title,
+                              (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid))
+                    `,
+      {
+        id: item.folder,
+        guid: item.guid,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        title: item.guid,
+      }
+    );
+  }
+
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function test_correct_folder_queries() {
+  for (let item of gTestItems) {
+    let bm = await PlacesUtils.bookmarks.fetch(item.guid);
+
+    if (item.targetParentGuid) {
+      Assert.equal(
+        bm.url,
+        `place:parent=${PlacesUtils.bookmarks[item.targetParentGuid]}`,
+        `Should have updated the URL for ${item.guid}`
+      );
+    } else {
+      let expected = item.expectedUrl
+        .replace("%guid%", gCreatedParentGuid)
+        .replace("%toolbarGuid%", PlacesUtils.bookmarks.toolbarGuid);
+
+      Assert.equal(
+        bm.url,
+        expected,
+        `Should have updated the URL for ${item.guid}`
+      );
+    }
+  }
+});
+
+add_task(async function test_hashes_valid() {
+  let db = await PlacesUtils.promiseDBConnection();
+  // Ensure all the hashes in moz_places are valid.
+  let rows = await db.execute(`SELECT url, url_hash FROM moz_places`);
+
+  for (let row of rows) {
+    let url = row.getResultByName("url");
+    let url_hash = row.getResultByName("url_hash");
+    Assert.equal(
+      url_hash,
+      PlacesUtils.history.hashURL(url),
+      `url hash should be correct for ${url}`
+    );
+  }
+});
+
+add_task(async function test_sync_counters_updated() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  for (let test of gTestItems) {
+    let rows = await db.execute(
+      `SELECT syncChangeCounter FROM moz_bookmarks
+                                 WHERE guid = :guid`,
+      { guid: test.guid }
+    );
+
+    Assert.equal(rows.length, 1, `Should only be one record for ${test.guid}`);
+    Assert.equal(
+      rows[0].getResultByName("syncChangeCounter"),
+      2,
+      `Should have bumped the syncChangeCounter for ${test.guid}`
+    );
+  }
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v50.js b/toolkit/components/places/tests/migration/test_current_from_v50.js
new file mode 100644
index 0000000000..af181091c0
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v50.js
@@ -0,0 +1,209 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const BASE_GUID = "null".padEnd(11, "_");
+const LAST_USED_ANNO = "bookmarkPropertiesDialog/folderLastUsed";
+const LAST_USED_META_DATA = "places/bookmarks/edit/lastusedfolder";
+
+let expectedGuids = [];
+
+async function adjustIndices(db, itemGuid) {
+  await db.execute(
+    `
+    UPDATE moz_bookmarks SET
+      position = position - 1
+    WHERE parent = (SELECT parent FROM moz_bookmarks
+                    WHERE guid = :itemGuid) AND
+          position >= (SELECT position FROM moz_bookmarks
+                       WHERE guid = :itemGuid)`,
+    { itemGuid }
+  );
+}
+
+async function fetchChildInfos(db, parentGuid) {
+  let rows = await db.execute(
+    `
+    SELECT b.guid, b.position, b.syncChangeCounter
+    FROM moz_bookmarks b
+    JOIN moz_bookmarks p ON p.id = b.parent
+    WHERE p.guid = :parentGuid
+    ORDER BY b.position`,
+    { parentGuid }
+  );
+  return rows.map(row => ({
+    guid: row.getResultByName("guid"),
+    position: row.getResultByName("position"),
+    syncChangeCounter: row.getResultByName("syncChangeCounter"),
+  }));
+}
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v43.sqlite");
+
+  // Setup database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+  // We can reuse the same guid, it doesn't matter for this test.
+  await db.execute(
+    `INSERT INTO moz_anno_attributes (name)
+                    VALUES (:last_used_anno)`,
+    { last_used_anno: LAST_USED_ANNO }
+  );
+
+  for (let i = 0; i < 3; i++) {
+    let guid = `${BASE_GUID}${i}`;
+    await db.execute(
+      `INSERT INTO moz_bookmarks (guid, type)
+                      VALUES (:guid, :type)
+                      `,
+      { guid, type: PlacesUtils.bookmarks.TYPE_FOLDER }
+    );
+    await db.execute(
+      `INSERT INTO moz_items_annos (item_id, anno_attribute_id, content)
+                      VALUES ((SELECT id FROM moz_bookmarks WHERE guid = :guid),
+                              (SELECT id FROM moz_anno_attributes WHERE name = :last_used_anno),
+                              :content)`,
+      {
+        guid,
+        content: new Date(1517318477569) - (3 - i) * 60 * 60 * 1000,
+        last_used_anno: LAST_USED_ANNO,
+      }
+    );
+    expectedGuids.unshift(guid);
+  }
+
+  info("Move menu into unfiled");
+  await adjustIndices(db, "menu________");
+  await db.execute(
+    `
+    UPDATE moz_bookmarks SET
+      parent = (SELECT id FROM moz_bookmarks WHERE guid = :newParentGuid),
+      position = IFNULL((SELECT MAX(position) + 1 FROM moz_bookmarks
+                         WHERE guid = :newParentGuid), 0)
+    WHERE guid = :itemGuid`,
+    { newParentGuid: "unfiled_____", itemGuid: "menu________" }
+  );
+
+  info("Move toolbar into mobile");
+  let mobileChildren = [
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+    "toolbar_____",
+    "bookmarkCCCC",
+    "bookmarkDDDD",
+  ];
+  await adjustIndices(db, "toolbar_____");
+  for (let position = 0; position < mobileChildren.length; position++) {
+    await db.execute(
+      `
+      INSERT INTO moz_bookmarks(guid, parent, position)
+      VALUES(:guid, (SELECT id FROM moz_bookmarks WHERE guid = 'mobile______'),
+             :position)
+      ON CONFLICT(guid) DO UPDATE SET
+        parent = excluded.parent,
+        position = excluded.position`,
+      { guid: mobileChildren[position], position }
+    );
+  }
+
+  info("Reset Sync change counters");
+  await db.execute(`UPDATE moz_bookmarks SET syncChangeCounter = 0`);
+
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function test_folders_migrated() {
+  let metaData = await PlacesUtils.metadata.get(LAST_USED_META_DATA);
+
+  Assert.deepEqual(JSON.parse(metaData), expectedGuids);
+});
+
+add_task(async function test_annotations_removed() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  await assertAnnotationsRemoved(db, [LAST_USED_ANNO]);
+});
+
+add_task(async function test_no_orphan_annotations() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  await assertNoOrphanAnnotations(db);
+});
+
+add_task(async function test_roots_fixed() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let expectedRootInfos = [
+    {
+      guid: PlacesUtils.bookmarks.tagsGuid,
+      position: 0,
+      syncChangeCounter: 0,
+    },
+    {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      position: 1,
+      syncChangeCounter: 1,
+    },
+    {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      position: 2,
+      syncChangeCounter: 1,
+    },
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      position: 3,
+      syncChangeCounter: 1,
+    },
+    {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      position: 4,
+      syncChangeCounter: 1,
+    },
+  ];
+  Assert.deepEqual(
+    expectedRootInfos,
+    await fetchChildInfos(db, PlacesUtils.bookmarks.rootGuid),
+    "All roots should be reparented to the Places root"
+  );
+
+  let expectedMobileInfos = [
+    {
+      guid: "bookmarkAAAA",
+      position: 0,
+      syncChangeCounter: 0,
+    },
+    {
+      guid: "bookmarkBBBB",
+      position: 1,
+      syncChangeCounter: 0,
+    },
+    {
+      guid: "bookmarkCCCC",
+      position: 2,
+      syncChangeCounter: 0,
+    },
+    {
+      guid: "bookmarkDDDD",
+      position: 3,
+      syncChangeCounter: 0,
+    },
+  ];
+  Assert.deepEqual(
+    expectedMobileInfos,
+    await fetchChildInfos(db, PlacesUtils.bookmarks.mobileGuid),
+    "Should fix misparented root sibling positions"
+  );
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v53.js b/toolkit/components/places/tests/migration/test_current_from_v53.js
new file mode 100644
index 0000000000..f872dea5d5
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v53.js
@@ -0,0 +1,23 @@
+add_task(async function setup() {
+  // Since this migration doesn't affect places.sqlite, we can reuse v43.
+  await setupPlacesDatabase("places_v43.sqlite");
+  await setupPlacesDatabase("favicons_v41.sqlite", "favicons.sqlite");
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  let count = (
+    await db.execute(
+      `SELECT count(*) FROM moz_icons_to_pages WHERE expire_ms = 0`
+    )
+  )[0].getResultByIndex(0);
+  Assert.equal(count, 0, "All the expirations should be set");
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v54.js b/toolkit/components/places/tests/migration/test_current_from_v54.js
new file mode 100644
index 0000000000..94c8a26474
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v54.js
@@ -0,0 +1,58 @@
+add_task(async function setup() {
+  // Since this migration doesn't affect places.sqlite, we can reuse v43.
+  await setupPlacesDatabase("places_v54.sqlite");
+  await setupPlacesDatabase("favicons_v41.sqlite", "favicons.sqlite");
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  for (let table of [
+    "moz_places_metadata",
+    "moz_places_metadata_search_queries",
+  ]) {
+    let count = (
+      await db.execute(`SELECT count(*) FROM ${table}`)
+    )[0].getResultByIndex(0);
+    Assert.equal(count, 0, `Empty table ${table}`);
+  }
+
+  for (let table of [
+    "moz_places_metadata_snapshots",
+    "moz_places_metadata_snapshots_extra",
+    "moz_places_metadata_snapshots_groups",
+    "moz_places_metadata_groups_to_snapshots",
+    "moz_session_metadata",
+    "moz_session_to_places",
+  ]) {
+    await Assert.rejects(
+      db.execute(`SELECT count(*) FROM ${table}`),
+      /no such table/,
+      `Table ${table} should not exist`
+    );
+  }
+});
+
+add_task(async function scrolling_fields_in_database() {
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute(
+    `SELECT scrolling_time,scrolling_distance FROM moz_places_metadata`
+  );
+});
+
+add_task(async function site_name_field_in_database() {
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute(`SELECT site_name FROM moz_places`);
+});
+
+add_task(async function previews_tombstones_in_database() {
+  let db = await PlacesUtils.promiseDBConnection();
+  await db.execute(`SELECT hash FROM moz_previews_tombstones`);
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v66.js b/toolkit/components/places/tests/migration/test_current_from_v66.js
new file mode 100644
index 0000000000..5ea14f3b9d
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v66.js
@@ -0,0 +1,53 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  const path = await setupPlacesDatabase("places_v66.sqlite");
+
+  const db = await Sqlite.openConnection({ path });
+  await db.execute(`
+    INSERT INTO moz_inputhistory (input, use_count, place_id)
+    VALUES
+      ('abc', 1, 1),
+      ('aBc', 0.9, 1),
+      ('ABC', 5, 1),
+      ('ABC', 1, 2),
+      ('DEF', 1, 3)
+  `);
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function moz_inputhistory() {
+  await PlacesUtils.withConnectionWrapper("test_sqlite_migration", async db => {
+    const rows = await db.execute(
+      "SELECT * FROM moz_inputhistory ORDER BY place_id"
+    );
+
+    Assert.equal(rows.length, 3);
+
+    Assert.equal(rows[0].getResultByName("place_id"), 1);
+    Assert.equal(rows[0].getResultByName("input"), "abc");
+    Assert.equal(rows[0].getResultByName("use_count"), 5);
+
+    Assert.equal(rows[1].getResultByName("place_id"), 2);
+    Assert.equal(rows[1].getResultByName("input"), "abc");
+    Assert.equal(rows[1].getResultByName("use_count"), 1);
+
+    Assert.equal(rows[2].getResultByName("place_id"), 3);
+    Assert.equal(rows[2].getResultByName("input"), "def");
+    Assert.equal(rows[2].getResultByName("use_count"), 1);
+  });
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v68.js b/toolkit/components/places/tests/migration/test_current_from_v68.js
new file mode 100644
index 0000000000..689fcbfd40
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v68.js
@@ -0,0 +1,35 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  const path = await setupPlacesDatabase("places_v68.sqlite");
+
+  const db = await Sqlite.openConnection({ path });
+  await db.execute("INSERT INTO moz_historyvisits (from_visit) VALUES (-1)");
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  const db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function moz_historyvisits() {
+  await PlacesUtils.withConnectionWrapper("test_sqlite_migration", async db => {
+    const rows = await db.execute(
+      "SELECT * FROM moz_historyvisits WHERE from_visit=-1"
+    );
+
+    Assert.equal(rows.length, 1);
+    Assert.equal(rows[0].getResultByName("source"), 0);
+    Assert.equal(rows[0].getResultByName("triggeringPlaceId"), null);
+  });
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v69.js b/toolkit/components/places/tests/migration/test_current_from_v69.js
new file mode 100644
index 0000000000..09c66fb66e
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v69.js
@@ -0,0 +1,84 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  const path = await setupPlacesDatabase("places_v69.sqlite");
+
+  const db = await Sqlite.openConnection({ path });
+  await db.execute(`
+    INSERT INTO moz_places (url, guid, url_hash, origin_id, frecency)
+    VALUES
+      ('https://test1.com', '___________1', '123456', 100, 0),
+      ('https://test2.com', '___________2', '123456', 101, -1),
+      ('https://test3.com', '___________3', '123456', 102, -1234)
+    `);
+  await db.execute(`
+    INSERT INTO moz_origins (id, prefix, host, frecency)
+    VALUES
+      (100, 'https://', 'test1.com', 0),
+      (101, 'https://', 'test2.com', 0),
+      (102, 'https://', 'test3.com', 0)
+    `);
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  const db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+});
+
+add_task(async function moz_historyvisits() {
+  await PlacesUtils.withConnectionWrapper("test_sqlite_migration", async db => {
+    function expectedFrecency(guid) {
+      switch (guid) {
+        case "___________1":
+          return 0;
+        case "___________2":
+          return -1;
+        case "___________3":
+          return 1234;
+        default:
+          throw new Error("Unknown guid");
+      }
+    }
+    const rows = await db.execute(
+      "SELECT guid, frecency FROM moz_places WHERE url_hash = '123456'"
+    );
+    for (let row of rows) {
+      Assert.equal(
+        row.getResultByName("frecency"),
+        expectedFrecency(row.getResultByName("guid")),
+        "Check expected frecency"
+      );
+    }
+    const origins = new Map(
+      (await db.execute("SELECT host, frecency FROM moz_origins")).map(r => [
+        r.getResultByName("host"),
+        r.getResultByName("frecency"),
+      ])
+    );
+    Assert.equal(origins.get("test1.com"), 0);
+    Assert.equal(origins.get("test2.com"), 0);
+    Assert.equal(origins.get("test3.com"), 1234);
+
+    const statSum = (
+      await db.execute(
+        "SELECT value FROM moz_meta WHERE key = 'origin_frecency_sum'"
+      )
+    )[0].getResultByName("value");
+    const sum = (
+      await db.execute(
+        "SELECT SUM(frecency) AS sum from moz_origins WHERE frecency > 0"
+      )
+    )[0].getResultByName("sum");
+    Assert.equal(sum, statSum, "Check stats were updated");
+  });
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v70.js b/toolkit/components/places/tests/migration/test_current_from_v70.js
new file mode 100644
index 0000000000..e5e41852e3
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v70.js
@@ -0,0 +1,96 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  let path = await setupPlacesDatabase("places_v70.sqlite");
+
+  let db = await Sqlite.openConnection({ path });
+  await db.execute(`
+    INSERT INTO moz_places (url, guid, url_hash, origin_id, frecency, foreign_count)
+    VALUES
+      ('https://test1.com', '___________1', '123456', 100, 0, 2),
+      ('https://test2.com', '___________2', '123456', 101, -1, 2),
+      ('https://test3.com', '___________3', '123456', 102, -1234, 1)
+    `);
+  await db.execute(`
+    INSERT INTO moz_origins (id, prefix, host, frecency)
+    VALUES
+      (100, 'https://', 'test1.com', 0),
+      (101, 'https://', 'test2.com', 0),
+      (102, 'https://', 'test3.com', 0)
+    `);
+  await db.execute(
+    `INSERT INTO moz_session_metadata
+      (id, guid)
+      VALUES (0, "0")
+    `
+  );
+
+  await db.execute(
+    `INSERT INTO moz_places_metadata_snapshots
+      (place_id, created_at, first_interaction_at, last_interaction_at)
+      VALUES ((SELECT id FROM moz_places WHERE guid = :guid), 0, 0, 0)
+    `,
+    { guid: "___________1" }
+  );
+  await db.execute(
+    `INSERT INTO moz_bookmarks
+      (fk, guid)
+      VALUES ((SELECT id FROM moz_places WHERE guid = :guid), :guid)
+    `,
+    { guid: "___________1" }
+  );
+
+  await db.execute(
+    `INSERT INTO moz_places_metadata_snapshots
+      (place_id, created_at, first_interaction_at, last_interaction_at)
+      VALUES ((SELECT id FROM moz_places WHERE guid = :guid), 0, 0, 0)
+    `,
+    { guid: "___________2" }
+  );
+  await db.execute(
+    `INSERT INTO moz_session_to_places
+      (session_id, place_id)
+      VALUES (0, (SELECT id FROM moz_places WHERE guid = :guid))
+    `,
+    { guid: "___________2" }
+  );
+
+  await db.execute(
+    `INSERT INTO moz_session_to_places
+      (session_id, place_id)
+      VALUES (0, (SELECT id FROM moz_places WHERE guid = :guid))
+    `,
+    { guid: "___________3" }
+  );
+
+  await db.close();
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  const db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  let rows = await db.execute("SELECT guid, foreign_count FROM moz_places");
+  for (let row of rows) {
+    let guid = row.getResultByName("guid");
+    let count = row.getResultByName("foreign_count");
+    if (guid == "___________1") {
+      Assert.equal(count, 1, "test1 should have the correct foreign_count");
+    }
+    if (guid == "___________2") {
+      Assert.equal(count, 0, "test2 should have the correct foreign_count");
+    }
+    if (guid == "___________3") {
+      Assert.equal(count, 0, "test3 should have the correct foreign_count");
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/migration/test_current_from_v72.js b/toolkit/components/places/tests/migration/test_current_from_v72.js
new file mode 100644
index 0000000000..626279fce4
--- /dev/null
+++ b/toolkit/components/places/tests/migration/test_current_from_v72.js
@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function setup() {
+  await setupPlacesDatabase("places_v72.sqlite");
+});
+
+add_task(async function database_is_valid() {
+  // Accessing the database for the first time triggers migration.
+  Assert.equal(
+    PlacesUtils.history.databaseStatus,
+    PlacesUtils.history.DATABASE_STATUS_UPGRADED
+  );
+
+  const db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(await db.getSchemaVersion(), CURRENT_SCHEMA_VERSION);
+
+  await db.execute(
+    "SELECT recalc_frecency, alt_frecency, recalc_alt_frecency FROM moz_origins"
+  );
+
+  await db.execute("SELECT alt_frecency, recalc_alt_frecency FROM moz_places");
+  Assert.ok(
+    await db.indexExists("moz_places_altfrecencyindex"),
+    "Should have created an index"
+  );
+});
diff --git a/toolkit/components/places/tests/migration/xpcshell.ini b/toolkit/components/places/tests/migration/xpcshell.ini
new file mode 100644
index 0000000000..6f864171fc
--- /dev/null
+++ b/toolkit/components/places/tests/migration/xpcshell.ini
@@ -0,0 +1,33 @@
+[DEFAULT]
+head = head_migration.js
+tags = condprof
+
+support-files =
+  favicons_v41.sqlite
+  places_outdated.sqlite
+  places_v43.sqlite
+  places_v54.sqlite
+  places_v66.sqlite
+  places_v68.sqlite
+  places_v69.sqlite
+  places_v70.sqlite
+  places_v72.sqlite
+  places_v74.sqlite
+
+[test_current_from_downgraded.js]
+[test_current_from_outdated.js]
+[test_current_from_v43.js]
+[test_current_from_v45.js]
+[test_current_from_v46.js]
+[test_current_from_v47.js]
+[test_current_from_v48.js]
+[test_current_from_v50.js]
+[test_current_from_v53.js]
+skip-if = condprof  # Bug 1769154 - not supported
+[test_current_from_v54.js]
+skip-if = condprof  # Bug 1769154 - not supported
+[test_current_from_v66.js]
+[test_current_from_v68.js]
+[test_current_from_v69.js]
+[test_current_from_v70.js]
+[test_current_from_v72.js]
diff --git a/toolkit/components/places/tests/moz.build b/toolkit/components/places/tests/moz.build
new file mode 100644
index 0000000000..97662e2a4b
--- /dev/null
+++ b/toolkit/components/places/tests/moz.build
@@ -0,0 +1,77 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+TEST_DIRS += ["gtest"]
+
+TESTING_JS_MODULES += [
+    "PlacesTestUtils.sys.mjs",
+]
+
+XPCSHELL_TESTS_MANIFESTS += [
+    "bookmarks/xpcshell.ini",
+    "expiration/xpcshell.ini",
+    "favicons/xpcshell.ini",
+    "history/xpcshell.ini",
+    "legacy/xpcshell.ini",
+    "maintenance/xpcshell.ini",
+    "migration/xpcshell.ini",
+    "queries/xpcshell.ini",
+    "sync/xpcshell.ini",
+    "unit/xpcshell.ini",
+]
+
+BROWSER_CHROME_MANIFESTS += [
+    "browser/browser.ini",
+    "browser/previews/browser.ini",
+]
+
+MOCHITEST_CHROME_MANIFESTS += [
+    "chrome/chrome.ini",
+]
+
+TEST_HARNESS_FILES.xpcshell.toolkit.components.places.tests += [
+    "head_common.js",
+]
+
+TEST_HARNESS_FILES.testing.mochitest.tests.toolkit.components.places.tests.browser += [
+    "browser/1601563-1.html",
+    "browser/1601563-2.html",
+    "browser/399606-history.go-0.html",
+    "browser/399606-httprefresh.html",
+    "browser/399606-location.reload.html",
+    "browser/399606-location.replace.html",
+    "browser/399606-window.location.href.html",
+    "browser/399606-window.location.html",
+    "browser/461710_link_page-2.html",
+    "browser/461710_link_page-3.html",
+    "browser/461710_link_page.html",
+    "browser/461710_visited_page.html",
+    "browser/begin.html",
+    "browser/favicon-normal16.png",
+    "browser/favicon-normal32.png",
+    "browser/favicon.html",
+    "browser/final.html",
+    "browser/history_post.html",
+    "browser/history_post.sjs",
+    "browser/redirect-target.html",
+    "browser/redirect.sjs",
+    "browser/redirect_once.sjs",
+    "browser/redirect_self.sjs",
+    "browser/redirect_thrice.sjs",
+    "browser/redirect_twice.sjs",
+    "browser/redirect_twice_perma.sjs",
+    "browser/title1.html",
+    "browser/title2.html",
+]
+
+TEST_HARNESS_FILES.testing.mochitest.tests.toolkit.components.places.tests.chrome += [
+    "chrome/bad_links.atom",
+    "chrome/link-less-items-no-site-uri.rss",
+    "chrome/link-less-items.rss",
+    "chrome/rss_as_html.rss",
+    "chrome/rss_as_html.rss^headers^",
+    "chrome/sample_feed.atom",
+]
diff --git a/toolkit/components/places/tests/queries/head_queries.js b/toolkit/components/places/tests/queries/head_queries.js
new file mode 100644
index 0000000000..d31d50d252
--- /dev/null
+++ b/toolkit/components/places/tests/queries/head_queries.js
@@ -0,0 +1,354 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+// Some Useful Date constants - PRTime uses microseconds, so convert
+const DAY_MICROSEC = 86400000000;
+const today = PlacesUtils.toPRTime(Date.now());
+const yesterday = today - DAY_MICROSEC;
+const lastweek = today - DAY_MICROSEC * 7;
+const daybefore = today - DAY_MICROSEC * 2;
+const old = today - DAY_MICROSEC * 3;
+const futureday = today + DAY_MICROSEC * 3;
+const olderthansixmonths = today - DAY_MICROSEC * 31 * 7;
+
+/**
+ * Generalized function to pull in an array of objects of data and push it into
+ * the database.  It does NOT do any checking to see that the input is
+ * appropriate.  This function is an asynchronous task, it can be called using
+ * "Task.spawn" or using the "yield" function inside another task.
+ */
+async function task_populateDB(aArray) {
+  // Iterate over aArray and execute all instructions.
+  for (let arrayItem of aArray) {
+    try {
+      // make the data object into a query data object in order to create proper
+      // default values for anything left unspecified
+      var qdata = new queryData(arrayItem);
+      if (qdata.isVisit) {
+        // Then we should add a visit for this node
+        await PlacesTestUtils.addVisits({
+          uri: uri(qdata.uri),
+          transition: qdata.transType,
+          visitDate: qdata.lastVisit,
+          referrer: qdata.referrer ? uri(qdata.referrer) : null,
+          title: qdata.title,
+        });
+        if (qdata.visitCount && !qdata.isDetails) {
+          // Set a fake visit_count, this is not a real count but can be used
+          // to test sorting by visit_count.
+          let stmt = DBConn().createAsyncStatement(
+            "UPDATE moz_places SET visit_count = :vc WHERE url_hash = hash(:url) AND url = :url"
+          );
+          stmt.params.vc = qdata.visitCount;
+          stmt.params.url = qdata.uri;
+          try {
+            stmt.executeAsync();
+          } catch (ex) {
+            print("Error while setting visit_count.");
+          } finally {
+            stmt.finalize();
+          }
+        }
+      }
+
+      if (qdata.isRedirect) {
+        // This must be async to properly enqueue after the updateFrecency call
+        // done by the visit addition.
+        let stmt = DBConn().createAsyncStatement(
+          "UPDATE moz_places SET hidden = 1 WHERE url_hash = hash(:url) AND url = :url"
+        );
+        stmt.params.url = qdata.uri;
+        try {
+          stmt.executeAsync();
+        } catch (ex) {
+          print("Error while setting hidden.");
+        } finally {
+          stmt.finalize();
+        }
+      }
+
+      if (qdata.isDetails) {
+        // Then we add extraneous page details for testing
+        await PlacesTestUtils.addVisits({
+          uri: uri(qdata.uri),
+          visitDate: qdata.lastVisit,
+          title: qdata.title,
+        });
+      }
+
+      if (qdata.markPageAsTyped) {
+        PlacesUtils.history.markPageAsTyped(uri(qdata.uri));
+      }
+
+      if (qdata.isPageAnnotation) {
+        await PlacesUtils.history.update({
+          url: qdata.uri,
+          annotations: new Map([
+            [qdata.annoName, qdata.removeAnnotation ? null : qdata.annoVal],
+          ]),
+        });
+      }
+
+      if (qdata.isFolder) {
+        await PlacesUtils.bookmarks.insert({
+          parentGuid: qdata.parentGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          title: qdata.title,
+          index: qdata.index,
+        });
+      }
+
+      if (qdata.isBookmark) {
+        let data = {
+          parentGuid: qdata.parentGuid,
+          index: qdata.index,
+          title: qdata.title,
+          url: qdata.uri,
+        };
+
+        if (qdata.dateAdded) {
+          data.dateAdded = new Date(qdata.dateAdded / 1000);
+        }
+
+        if (qdata.lastModified) {
+          data.lastModified = new Date(qdata.lastModified / 1000);
+        }
+
+        await PlacesUtils.bookmarks.insert(data);
+
+        if (qdata.keyword) {
+          await PlacesUtils.keywords.insert({
+            url: qdata.uri,
+            keyword: qdata.keyword,
+          });
+        }
+      }
+
+      if (qdata.isTag) {
+        PlacesUtils.tagging.tagURI(uri(qdata.uri), qdata.tagArray);
+      }
+
+      if (qdata.isSeparator) {
+        await PlacesUtils.bookmarks.insert({
+          parentGuid: qdata.parentGuid,
+          type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+          index: qdata.index,
+        });
+      }
+    } catch (ex) {
+      // use the arrayItem object here in case instantiation of qdata failed
+      info("Problem with this URI: " + arrayItem.uri);
+      do_throw("Error creating database: " + ex + "\n");
+    }
+  }
+}
+
+/**
+ * The Query Data Object - this object encapsulates data for our queries and is
+ * used to parameterize our calls to the Places APIs to put data into the
+ * database. It also has some interesting meta functions to determine which APIs
+ * should be called, and to determine if this object should show up in the
+ * resulting query.
+ * Its parameter is an object specifying which attributes you want to set.
+ * For ex:
+ * var myobj = new queryData({isVisit: true, uri:"http://mozilla.com", title="foo"});
+ * Note that it doesn't do any input checking on that object.
+ */
+function queryData(obj) {
+  this.isVisit = obj.isVisit ? obj.isVisit : false;
+  this.isBookmark = obj.isBookmark ? obj.isBookmark : false;
+  this.uri = obj.uri ? obj.uri : "";
+  this.lastVisit = obj.lastVisit ? obj.lastVisit : today;
+  this.referrer = obj.referrer ? obj.referrer : null;
+  this.transType = obj.transType
+    ? obj.transType
+    : Ci.nsINavHistoryService.TRANSITION_TYPED;
+  this.isRedirect = obj.isRedirect ? obj.isRedirect : false;
+  this.isDetails = obj.isDetails ? obj.isDetails : false;
+  this.title = obj.title ? obj.title : "";
+  this.markPageAsTyped = obj.markPageAsTyped ? obj.markPageAsTyped : false;
+  this.isPageAnnotation = obj.isPageAnnotation ? obj.isPageAnnotation : false;
+  this.removeAnnotation = !!obj.removeAnnotation;
+  this.annoName = obj.annoName ? obj.annoName : "";
+  this.annoVal = obj.annoVal ? obj.annoVal : "";
+  this.itemId = obj.itemId ? obj.itemId : 0;
+  this.annoMimeType = obj.annoMimeType ? obj.annoMimeType : "";
+  this.isTag = obj.isTag ? obj.isTag : false;
+  this.tagArray = obj.tagArray ? obj.tagArray : null;
+  this.parentGuid = obj.parentGuid || PlacesUtils.bookmarks.unfiledGuid;
+  this.feedURI = obj.feedURI ? obj.feedURI : "";
+  this.index = obj.index ? obj.index : PlacesUtils.bookmarks.DEFAULT_INDEX;
+  this.isFolder = obj.isFolder ? obj.isFolder : false;
+  this.contractId = obj.contractId ? obj.contractId : "";
+  this.lastModified = obj.lastModified ? obj.lastModified : null;
+  this.dateAdded = obj.dateAdded ? obj.dateAdded : null;
+  this.keyword = obj.keyword ? obj.keyword : "";
+  this.visitCount = obj.visitCount ? obj.visitCount : 0;
+  this.isSeparator = obj.hasOwnProperty("isSeparator") && obj.isSeparator;
+
+  // And now, the attribute for whether or not this object should appear in the
+  // resulting query
+  this.isInQuery = obj.isInQuery ? obj.isInQuery : false;
+}
+
+// All attributes are set in the constructor above
+queryData.prototype = {};
+
+/**
+ * Helper function to compare an array of query objects with a result set.
+ * It assumes the array of query objects contains the SAME SORT as the result
+ * set.  It checks the the uri, title, time, and bookmarkIndex properties of
+ * the results, where appropriate.
+ */
+function compareArrayToResult(aArray, aRoot) {
+  info("Comparing Array to Results");
+
+  var wasOpen = aRoot.containerOpen;
+  if (!wasOpen) {
+    aRoot.containerOpen = true;
+  }
+
+  // check expected number of results against actual
+  var expectedResultCount = aArray.filter(function (aEl) {
+    return aEl.isInQuery;
+  }).length;
+  if (expectedResultCount != aRoot.childCount) {
+    // Debugging code for failures.
+    dump_table("moz_places");
+    dump_table("moz_historyvisits");
+    info("Found children:");
+    for (let i = 0; i < aRoot.childCount; i++) {
+      info(aRoot.getChild(i).uri);
+    }
+    info("Expected:");
+    for (let i = 0; i < aArray.length; i++) {
+      if (aArray[i].isInQuery) {
+        info(aArray[i].uri);
+      }
+    }
+  }
+  Assert.equal(expectedResultCount, aRoot.childCount);
+
+  var inQueryIndex = 0;
+  for (var i = 0; i < aArray.length; i++) {
+    if (aArray[i].isInQuery) {
+      var child = aRoot.getChild(inQueryIndex);
+      // do_print("testing testData[" + i + "] vs result[" + inQueryIndex + "]");
+      if (!aArray[i].isFolder && !aArray[i].isSeparator) {
+        info(
+          "testing testData[" + aArray[i].uri + "] vs result[" + child.uri + "]"
+        );
+        if (aArray[i].uri != child.uri) {
+          dump_table("moz_places");
+          do_throw("Expected " + aArray[i].uri + " found " + child.uri);
+        }
+      }
+      if (!aArray[i].isSeparator && aArray[i].title != child.title) {
+        do_throw("Expected " + aArray[i].title + " found " + child.title);
+      }
+      if (
+        aArray[i].hasOwnProperty("lastVisit") &&
+        aArray[i].lastVisit != child.time
+      ) {
+        do_throw("Expected " + aArray[i].lastVisit + " found " + child.time);
+      }
+      if (
+        aArray[i].hasOwnProperty("index") &&
+        aArray[i].index != PlacesUtils.bookmarks.DEFAULT_INDEX &&
+        aArray[i].index != child.bookmarkIndex
+      ) {
+        do_throw(
+          "Expected " + aArray[i].index + " found " + child.bookmarkIndex
+        );
+      }
+
+      inQueryIndex++;
+    }
+  }
+
+  if (!wasOpen) {
+    aRoot.containerOpen = false;
+  }
+  info("Comparing Array to Results passes");
+}
+
+/**
+ * Helper function to check to see if one object either is or is not in the
+ * result set.  It can accept either a queryData object or an array of queryData
+ * objects.  If it gets an array, it only compares the first object in the array
+ * to see if it is in the result set.
+ * Returns: True if item is in query set, and false if item is not in query set
+ *          If input is an array, returns True if FIRST object in array is in
+ *          query set.  To compare entire array, use the function above.
+ */
+function isInResult(aQueryData, aRoot) {
+  var rv = false;
+  var uri;
+  var wasOpen = aRoot.containerOpen;
+  if (!wasOpen) {
+    aRoot.containerOpen = true;
+  }
+
+  // If we have an array, pluck out the first item. If an object, pluc out the
+  // URI, we just compare URI's here.
+  if ("uri" in aQueryData) {
+    uri = aQueryData.uri;
+  } else {
+    uri = aQueryData[0].uri;
+  }
+
+  for (var i = 0; i < aRoot.childCount; i++) {
+    if (uri == aRoot.getChild(i).uri) {
+      rv = true;
+      break;
+    }
+  }
+  if (!wasOpen) {
+    aRoot.containerOpen = false;
+  }
+  return rv;
+}
+
+/**
+ * A nice helper function for debugging things. It prints the contents of a
+ * result set.
+ */
+function displayResultSet(aRoot) {
+  var wasOpen = aRoot.containerOpen;
+  if (!wasOpen) {
+    aRoot.containerOpen = true;
+  }
+
+  if (!aRoot.hasChildren) {
+    // Something wrong? Empty result set?
+    info("Result Set Empty");
+    return;
+  }
+
+  for (var i = 0; i < aRoot.childCount; ++i) {
+    info(
+      "Result Set URI: " +
+        aRoot.getChild(i).uri +
+        "   Title: " +
+        aRoot.getChild(i).title +
+        "   Visit Time: " +
+        aRoot.getChild(i).time
+    );
+  }
+  if (!wasOpen) {
+    aRoot.containerOpen = false;
+  }
+}
diff --git a/toolkit/components/places/tests/queries/readme.txt b/toolkit/components/places/tests/queries/readme.txt
new file mode 100644
index 0000000000..19414f96ed
--- /dev/null
+++ b/toolkit/components/places/tests/queries/readme.txt
@@ -0,0 +1,16 @@
+These are tests specific to the Places Query API.
+
+We are tracking the coverage of these tests here:
+http://wiki.mozilla.org/QA/TDAI/Projects/Places_Tests
+
+When creating one of these tests, you need to update those tables so that we
+know how well our test coverage is of this area.  Furthermore, when adding tests
+ensure to cover live update (changing the query set) by performing the following
+operations on the query set you get after running the query:
+* Adding a new item to the query set
+* Updating an existing item so that it matches the query set
+* Change an existing item so that it does not match the query set
+* Do multiple of the above inside an Update Batch transaction.
+* Try these transactions in different orders.
+
+Use the stub test to help you create a test with the proper structure.
diff --git a/toolkit/components/places/tests/queries/test_415716.js b/toolkit/components/places/tests/queries/test_415716.js
new file mode 100644
index 0000000000..a39c278024
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_415716.js
@@ -0,0 +1,109 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function modHistoryTypes(val) {
+  switch (val % 8) {
+    case 0:
+    case 1:
+      return TRANSITION_LINK;
+    case 2:
+      return TRANSITION_TYPED;
+    case 3:
+      return TRANSITION_BOOKMARK;
+    case 4:
+      return TRANSITION_EMBED;
+    case 5:
+      return TRANSITION_REDIRECT_PERMANENT;
+    case 6:
+      return TRANSITION_REDIRECT_TEMPORARY;
+    case 7:
+      return TRANSITION_DOWNLOAD;
+    case 8:
+      return TRANSITION_FRAMED_LINK;
+  }
+  return TRANSITION_TYPED;
+}
+
+/**
+ * Builds a test database by hand using various times, annotations and
+ * visit numbers for this test
+ */
+add_task(async function test_buildTestDatabase() {
+  // This is the set of visits that we will match - our min visit is 2 so that's
+  // why we add more visits to the same URIs.
+  let testURI = "http://www.foo.com";
+  let places = [];
+
+  for (let i = 0; i < 12; ++i) {
+    places.push({
+      uri: testURI,
+      transition: modHistoryTypes(i),
+      visitDate: today,
+    });
+  }
+
+  testURI = "http://foo.com/youdontseeme.html";
+  let testAnnoName = "moz-test-places/testing123";
+  let testAnnoVal = "test";
+  for (let i = 0; i < 12; ++i) {
+    places.push({
+      uri: testURI,
+      transition: modHistoryTypes(i),
+      visitDate: today,
+    });
+  }
+
+  await PlacesTestUtils.addVisits(places);
+
+  await PlacesUtils.history.update({
+    url: testURI,
+    annotations: new Map([[testAnnoName, testAnnoVal]]),
+  });
+});
+
+/**
+ * This test will test Queries that use relative Time Range, minVists, maxVisits,
+ * annotation.
+ * The Query:
+ * Annotation == "moz-test-places/testing123" &&
+ * TimeRange == "now() - 2d" &&
+ * minVisits == 2 &&
+ * maxVisits == 10
+ */
+add_task(function test_execute() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.annotation = "moz-test-places/testing123";
+  query.beginTime = daybefore * 1000;
+  query.beginTimeReference = PlacesUtils.history.TIME_RELATIVE_NOW;
+  query.endTime = today * 1000;
+  query.endTimeReference = PlacesUtils.history.TIME_RELATIVE_NOW;
+  query.minVisits = 2;
+  query.maxVisits = 10;
+
+  // Options
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+
+  // Results
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  let cc = root.childCount;
+  dump("----> cc is: " + cc + "\n");
+  for (let i = 0; i < root.childCount; ++i) {
+    let resultNode = root.getChild(i);
+    let accesstime = Date(resultNode.time / 1000);
+    dump(
+      "----> result: " +
+        resultNode.uri +
+        "   Date: " +
+        accesstime.toLocaleString() +
+        "\n"
+    );
+  }
+  Assert.equal(cc, 0);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_abstime-annotation-domain.js b/toolkit/components/places/tests/queries/test_abstime-annotation-domain.js
new file mode 100644
index 0000000000..3f563ea7d8
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_abstime-annotation-domain.js
@@ -0,0 +1,321 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const DAY_MSEC = 86400000;
+const MIN_MSEC = 60000;
+const HOUR_MSEC = 3600000;
+// Jan 6 2008 at 8am is our begin edge of the query
+var beginTimeDate = new Date(2008, 0, 6, 8, 0, 0, 0);
+// Jan 15 2008 at 9:30pm is our ending edge of the query
+var endTimeDate = new Date(2008, 0, 15, 21, 30, 0, 0);
+
+// These as millisecond values
+var beginTime = beginTimeDate.getTime();
+var endTime = endTimeDate.getTime();
+
+// Some range dates inside our query - mult by 1000 to convert to PRTIME
+var jan7_800 = (beginTime + DAY_MSEC) * 1000;
+var jan6_815 = (beginTime + MIN_MSEC * 15) * 1000;
+var jan11_800 = (beginTime + DAY_MSEC * 5) * 1000;
+var jan14_2130 = (endTime - DAY_MSEC) * 1000;
+var jan15_2045 = (endTime - MIN_MSEC * 45) * 1000;
+var jan12_1730 = (endTime - DAY_MSEC * 3 - HOUR_MSEC * 4) * 1000;
+
+// Dates outside our query - mult by 1000 to convert to PRTIME
+var jan6_700 = (beginTime - HOUR_MSEC) * 1000;
+var dec27_800 = (beginTime - DAY_MSEC * 10) * 1000;
+
+// So that we can easily use these too, convert them to PRTIME
+beginTime *= 1000;
+endTime *= 1000;
+
+/**
+ * Array of objects to build our test database
+ */
+var goodAnnoName = "moz-test-places/testing123";
+var val = "test";
+var badAnnoName = "text/foo";
+
+// The test data for our database, note that the ordering of the results that
+// will be returned by the query (the isInQuery: true objects) is IMPORTANT.
+// see compareArrayToResult in head_queries.js for more info.
+var testData = [
+  // Test ftp protocol - vary the title length
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    uri: "ftp://foo.com/ftp",
+    lastVisit: jan12_1730,
+    title: "hugelongconfmozlagurationofwordswithasearchtermsinit whoo-hoo",
+  },
+
+  // Test flat domain with annotation
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://foo.com/",
+    annoName: goodAnnoName,
+    annoVal: val,
+    lastVisit: jan14_2130,
+    title: "moz",
+  },
+
+  // Test subdomain included with isRedirect=true, different transtype
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    isRedirect: true,
+    uri: "http://mail.foo.com/redirect",
+    lastVisit: jan11_800,
+    transType: PlacesUtils.history.TRANSITION_LINK,
+  },
+
+  // Test subdomain inclued at the leading time edge
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    uri: "http://mail.foo.com/yiihah",
+    title: "moz",
+    lastVisit: jan6_815,
+  },
+
+  // Test www. style URI is included, with an annotation
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://www.foo.com/yiihah",
+    annoName: goodAnnoName,
+    annoVal: val,
+    lastVisit: jan7_800,
+    title: "moz",
+  },
+
+  // Test https protocol
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "https://foo.com/",
+    lastVisit: jan15_2045,
+  },
+
+  // Test begin edge of time
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "moz mozilla",
+    uri: "https://foo.com/begin.html",
+    lastVisit: beginTime,
+  },
+
+  // Test end edge of time
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "moz mozilla",
+    uri: "https://foo.com/end.html",
+    lastVisit: endTime,
+  },
+
+  // Test an image link, with annotations
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    title: "mozzie the dino",
+    uri: "https://foo.com/mozzie.png",
+    annoName: goodAnnoName,
+    annoVal: val,
+    lastVisit: jan14_2130,
+  },
+
+  // Begin the invalid queries: Test too early
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://foo.com/tooearly.php",
+    lastVisit: jan6_700,
+  },
+
+  // Test Bad Annotation
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    title: "moz",
+    uri: "http://foo.com/badanno.htm",
+    lastVisit: jan12_1730,
+    annoName: badAnnoName,
+    annoVal: val,
+  },
+
+  // Test bad URI
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://somefoo.com/justwrong.htm",
+    lastVisit: jan11_800,
+  },
+
+  // Test afterward, one to update
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "changeme",
+    uri: "http://foo.com/changeme1.htm",
+    lastVisit: jan12_1730,
+  },
+
+  // Test invalid title
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "changeme2",
+    uri: "http://foo.com/changeme2.htm",
+    lastVisit: jan7_800,
+  },
+
+  // Test changing the lastVisit
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://foo.com/changeme3.htm",
+    lastVisit: dec27_800,
+  },
+];
+
+/**
+ * This test will test a Query using several terms and do a bit of negative
+ * testing for items that should be ignored while querying over history.
+ * The Query:WHERE absoluteTime(matches) AND searchTerms AND URI
+ *                 AND annotationIsNot(match) GROUP BY Domain, Day SORT BY uri,ascending
+ *                 excludeITems(should be ignored)
+ */
+add_task(async function test_abstime_annotation_domain() {
+  // Initialize database
+  await task_populateDB(testData);
+
+  // Query
+  var query = PlacesUtils.history.getNewQuery();
+  query.beginTime = beginTime;
+  query.endTime = endTime;
+  query.beginTimeReference = PlacesUtils.history.TIME_RELATIVE_EPOCH;
+  query.endTimeReference = PlacesUtils.history.TIME_RELATIVE_EPOCH;
+  query.searchTerms = "moz";
+  query.domain = "foo.com";
+  query.domainIsHost = false;
+  query.annotation = "text/foo";
+  query.annotationIsNot = true;
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_URI_ASCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+  // The next two options should be ignored
+  // can't use this one, breaks test - bug 419779
+  // options.excludeItems = true;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  // Ensure the result set is correct
+  compareArrayToResult(testData, root);
+
+  // Make some changes to the result set
+  // Let's add something first
+  var addItem = [
+    {
+      isInQuery: true,
+      isVisit: true,
+      isDetails: true,
+      title: "moz",
+      uri: "http://www.foo.com/i-am-added.html",
+      lastVisit: jan11_800,
+    },
+  ];
+  await task_populateDB(addItem);
+  info("Adding item foo.com/i-am-added.html");
+  Assert.equal(isInResult(addItem, root), true);
+
+  // Let's update something by title
+  var change1 = [
+    {
+      isDetails: true,
+      uri: "http://foo.com/changeme1",
+      lastVisit: jan12_1730,
+      title: "moz moz mozzie",
+    },
+  ];
+  await task_populateDB(change1);
+  info("LiveUpdate by changing title");
+  Assert.equal(isInResult(change1, root), true);
+
+  // Let's update something by annotation
+  // Updating a page by removing an annotation does not cause it to join this
+  // query set.  I tend to think that it should cause that page to join this
+  // query set, because this visit fits all theother specified criteria once the
+  // annotation is removed. Uncommenting this will fail the test.
+  // Bug 424050
+  /* var change2 = [{isPageAnnotation: true, uri: "http://foo.com/badannotaion.html",
+                  annoName: "text/mozilla", annoVal: "test"}];
+  yield task_populateDB(change2);
+  do_print("LiveUpdate by removing annotation");
+  do_check_eq(isInResult(change2, root), true);*/
+
+  // Let's update by adding a visit in the time range for an existing URI
+  var change3 = [
+    {
+      isDetails: true,
+      uri: "http://foo.com/changeme3.htm",
+      title: "moz",
+      lastVisit: jan15_2045,
+    },
+  ];
+  await task_populateDB(change3);
+  info("LiveUpdate by adding visit within timerange");
+  Assert.equal(isInResult(change3, root), true);
+
+  // And delete something from the result set - using annotation
+  // Once again, bug 424050 prevents this from passing
+  /* var change4 = [{isPageAnnotation: true, uri: "ftp://foo.com/ftp",
+                  annoVal: "test", annoName: badAnnoName}];
+  yield task_populateDB(change4);
+  do_print("LiveUpdate by deleting item from set by adding annotation");
+  do_check_eq(isInResult(change4, root), false);*/
+
+  // Delete something by changing the title
+  var change5 = [
+    { isDetails: true, uri: "http://foo.com/end.html", title: "deleted" },
+  ];
+  await task_populateDB(change5);
+  info("LiveUpdate by deleting item by changing title");
+  Assert.equal(isInResult(change5, root), false);
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_abstime-annotation-uri.js b/toolkit/components/places/tests/queries/test_abstime-annotation-uri.js
new file mode 100644
index 0000000000..9ad1478726
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_abstime-annotation-uri.js
@@ -0,0 +1,226 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const DAY_MSEC = 86400000;
+const MIN_MSEC = 60000;
+const HOUR_MSEC = 3600000;
+// Jan 6 2008 at 8am is our begin edge of the query
+var beginTimeDate = new Date(2008, 0, 6, 8, 0, 0, 0);
+// Jan 15 2008 at 9:30pm is our ending edge of the query
+var endTimeDate = new Date(2008, 0, 15, 21, 30, 0, 0);
+
+// These as millisecond values
+var beginTime = beginTimeDate.getTime();
+var endTime = endTimeDate.getTime();
+
+// Some range dates inside our query - mult by 1000 to convert to PRTIME
+var jan7_800 = (beginTime + DAY_MSEC) * 1000;
+var jan6_815 = (beginTime + MIN_MSEC * 15) * 1000;
+var jan14_2130 = (endTime - DAY_MSEC) * 1000;
+var jan15_2045 = (endTime - MIN_MSEC * 45) * 1000;
+var jan12_1730 = (endTime - DAY_MSEC * 3 - HOUR_MSEC * 4) * 1000;
+
+// Dates outside our query - mult by 1000 to convert to PRTIME
+var jan6_700 = (beginTime - HOUR_MSEC) * 1000;
+var dec27_800 = (beginTime - DAY_MSEC * 10) * 1000;
+
+// So that we can easily use these too, convert them to PRTIME
+beginTime *= 1000;
+endTime *= 1000;
+
+/**
+ * Array of objects to build our test database
+ */
+var goodAnnoName = "moz-test-places/testing123";
+var val = "test";
+var badAnnoName = "text/foo";
+
+// The test data for our database, note that the ordering of the results that
+// will be returned by the query (the isInQuery: true objects) is IMPORTANT.
+// see compareArrayToResult in head_queries.js for more info.
+var testData = [
+  // Test flat domain with annotation
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://foo.com/",
+    annoName: goodAnnoName,
+    annoVal: val,
+    lastVisit: jan14_2130,
+    title: "moz",
+  },
+
+  // Begin the invalid queries:
+  // Test www. style URI is not included, with an annotation
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://www.foo.com/yiihah",
+    annoName: goodAnnoName,
+    annoVal: val,
+    lastVisit: jan7_800,
+    title: "moz",
+  },
+
+  // Test subdomain not inclued at the leading time edge
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    uri: "http://mail.foo.com/yiihah",
+    title: "moz",
+    lastVisit: jan6_815,
+  },
+
+  // Test https protocol
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "https://foo.com/",
+    lastVisit: jan15_2045,
+  },
+
+  // Test ftp protocol
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    uri: "ftp://foo.com/ftp",
+    lastVisit: jan12_1730,
+    title: "hugelongconfmozlagurationofwordswithasearchtermsinit whoo-hoo",
+  },
+
+  // Test too early
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://foo.com/tooearly.php",
+    lastVisit: jan6_700,
+  },
+
+  // Test Bad Annotation
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    title: "moz",
+    uri: "http://foo.com/badanno.htm",
+    lastVisit: jan12_1730,
+    annoName: badAnnoName,
+    annoVal: val,
+  },
+
+  // Test afterward, one to update
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "changeme",
+    uri: "http://foo.com/changeme1.htm",
+    lastVisit: jan12_1730,
+  },
+
+  // Test invalid title
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "changeme2",
+    uri: "http://foo.com/changeme2.htm",
+    lastVisit: jan7_800,
+  },
+
+  // Test changing the lastVisit
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://foo.com/changeme3.htm",
+    lastVisit: dec27_800,
+  },
+];
+
+/**
+ * This test will test a Query using several terms and do a bit of negative
+ * testing for items that should be ignored while querying over history.
+ * The Query:WHERE absoluteTime(matches) AND searchTerms AND URI
+ *                 AND annotationIsNot(match) GROUP BY Domain, Day SORT BY uri,ascending
+ *                 excludeITems(should be ignored)
+ */
+add_task(async function test_abstime_annotation_uri() {
+  // Initialize database
+  await task_populateDB(testData);
+
+  // Query
+  var query = PlacesUtils.history.getNewQuery();
+  query.beginTime = beginTime;
+  query.endTime = endTime;
+  query.beginTimeReference = PlacesUtils.history.TIME_RELATIVE_EPOCH;
+  query.endTimeReference = PlacesUtils.history.TIME_RELATIVE_EPOCH;
+  query.searchTerms = "moz";
+  query.uri = uri("http://foo.com");
+  query.annotation = "text/foo";
+  query.annotationIsNot = true;
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_URI_ASCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+  // The next two options should be ignored
+  // can't use this one, breaks test - bug 419779
+  // options.excludeItems = true;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  // Ensure the result set is correct
+  compareArrayToResult(testData, root);
+
+  // live update.
+  info("change title");
+  var change1 = [{ isDetails: true, uri: "http://foo.com/", title: "mo" }];
+  await task_populateDB(change1);
+  Assert.ok(!isInResult({ uri: "http://foo.com/" }, root));
+
+  var change2 = [
+    {
+      isDetails: true,
+      uri: "http://foo.com/",
+      title: "moz",
+      lastvisit: endTime,
+    },
+  ];
+  await task_populateDB(change2);
+  dump_table("moz_places");
+  Assert.ok(!isInResult({ uri: "http://foo.com/" }, root));
+
+  // Let's delete something from the result set - using annotation
+  var change3 = [
+    {
+      isPageAnnotation: true,
+      uri: "http://foo.com/",
+      annoName: badAnnoName,
+      annoVal: "test",
+    },
+  ];
+  await task_populateDB(change3);
+  info("LiveUpdate by removing annotation");
+  Assert.ok(!isInResult({ uri: "http://foo.com/" }, root));
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_async.js b/toolkit/components/places/tests/queries/test_async.js
new file mode 100644
index 0000000000..acdb0eef48
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_async.js
@@ -0,0 +1,379 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var tests = [
+  {
+    desc:
+      "nsNavHistoryFolderResultNode: Basic test, asynchronously open and " +
+      "close container with a single child",
+
+    loading(node, newState, oldState) {
+      this.checkStateChanged("loading", 1);
+      this.checkArgs("loading", node, oldState, node.STATE_CLOSED);
+    },
+
+    opened(node, newState, oldState) {
+      this.checkStateChanged("opened", 1);
+      this.checkState("loading", 1);
+      this.checkArgs("opened", node, oldState, node.STATE_LOADING);
+
+      print("Checking node children");
+      compareArrayToResult(this.data, node);
+
+      print("Closing container");
+      node.containerOpen = false;
+    },
+
+    closed(node, newState, oldState) {
+      this.checkStateChanged("closed", 1);
+      this.checkState("opened", 1);
+      this.checkArgs("closed", node, oldState, node.STATE_OPENED);
+      this.success();
+    },
+  },
+
+  {
+    desc:
+      "nsNavHistoryFolderResultNode: After async open and no changes, " +
+      "second open should be synchronous",
+
+    loading(node, newState, oldState) {
+      this.checkStateChanged("loading", 1);
+      this.checkState("closed", 0);
+      this.checkArgs("loading", node, oldState, node.STATE_CLOSED);
+    },
+
+    opened(node, newState, oldState) {
+      let cnt = this.checkStateChanged("opened", 1, 2);
+      let expectOldState = cnt === 1 ? node.STATE_LOADING : node.STATE_CLOSED;
+      this.checkArgs("opened", node, oldState, expectOldState);
+
+      print("Checking node children");
+      compareArrayToResult(this.data, node);
+
+      print("Closing container");
+      node.containerOpen = false;
+    },
+
+    closed(node, newState, oldState) {
+      let cnt = this.checkStateChanged("closed", 1, 2);
+      this.checkArgs("closed", node, oldState, node.STATE_OPENED);
+
+      switch (cnt) {
+        case 1:
+          node.containerOpen = true;
+          break;
+        case 2:
+          this.success();
+          break;
+      }
+    },
+  },
+
+  {
+    desc:
+      "nsNavHistoryFolderResultNode: After closing container in " +
+      "loading(), opened() should not be called",
+
+    loading(node, newState, oldState) {
+      this.checkStateChanged("loading", 1);
+      this.checkArgs("loading", node, oldState, node.STATE_CLOSED);
+      print("Closing container");
+      node.containerOpen = false;
+    },
+
+    opened(node, newState, oldState) {
+      do_throw("opened should not be called");
+    },
+
+    closed(node, newState, oldState) {
+      this.checkStateChanged("closed", 1);
+      this.checkState("loading", 1);
+      this.checkArgs("closed", node, oldState, node.STATE_LOADING);
+      this.success();
+    },
+  },
+];
+
+/**
+ * Instances of this class become the prototypes of the test objects above.
+ * Each test can therefore use the methods of this class, or they can override
+ * them if they want.  To run a test, call setup() and then run().
+ */
+function Test() {
+  // This maps a state name to the number of times it's been observed.
+  this.stateCounts = {};
+  // Promise object resolved when the next test can be run.
+  this.deferNextTest = PromiseUtils.defer();
+}
+
+Test.prototype = {
+  /**
+   * Call this when an observer observes a container state change to sanity
+   * check the arguments.
+   *
+   * @param aNewState
+   *        The name of the new state.  Used only for printing out helpful info.
+   * @param aNode
+   *        The node argument passed to containerStateChanged.
+   * @param aOldState
+   *        The old state argument passed to containerStateChanged.
+   * @param aExpectOldState
+   *        The expected old state.
+   */
+  checkArgs(aNewState, aNode, aOldState, aExpectOldState) {
+    print("Node passed on " + aNewState + " should be result.root");
+    Assert.equal(this.result.root, aNode);
+    print("Old state passed on " + aNewState + " should be " + aExpectOldState);
+
+    // aOldState comes from xpconnect and will therefore be defined.  It may be
+    // zero, though, so use strict equality just to make sure aExpectOldState is
+    // also defined.
+    Assert.ok(aOldState === aExpectOldState);
+  },
+
+  /**
+   * Call this when an observer observes a container state change.  It registers
+   * the state change and ensures that it has been observed the given number
+   * of times.  See checkState for parameter explanations.
+   *
+   * @return The number of times aState has been observed, including the new
+   *         observation.
+   */
+  checkStateChanged(aState, aExpectedMin, aExpectedMax) {
+    print(aState + " state change observed");
+    if (!this.stateCounts.hasOwnProperty(aState)) {
+      this.stateCounts[aState] = 0;
+    }
+    this.stateCounts[aState]++;
+    return this.checkState(aState, aExpectedMin, aExpectedMax);
+  },
+
+  /**
+   * Ensures that the state has been observed the given number of times.
+   *
+   * @param  aState
+   *         The name of the state.
+   * @param  aExpectedMin
+   *         The state must have been observed at least this number of times.
+   * @param  aExpectedMax
+   *         The state must have been observed at most this number of times.
+   *         This parameter is optional.  If undefined, it's set to
+   *         aExpectedMin.
+   * @return The number of times aState has been observed, including the new
+   *         observation.
+   */
+  checkState(aState, aExpectedMin, aExpectedMax) {
+    let cnt = this.stateCounts[aState] || 0;
+    if (aExpectedMax === undefined) {
+      aExpectedMax = aExpectedMin;
+    }
+    if (aExpectedMin === aExpectedMax) {
+      print(
+        aState +
+          " should be observed only " +
+          aExpectedMin +
+          " times (actual = " +
+          cnt +
+          ")"
+      );
+    } else {
+      print(
+        aState +
+          " should be observed at least " +
+          aExpectedMin +
+          " times and at most " +
+          aExpectedMax +
+          " times (actual = " +
+          cnt +
+          ")"
+      );
+    }
+    Assert.ok(cnt >= aExpectedMin && cnt <= aExpectedMax);
+    return cnt;
+  },
+
+  /**
+   * Asynchronously opens the root of the test's result.
+   */
+  openContainer() {
+    // Set up the result observer.  It delegates to this object's callbacks and
+    // wraps them in a try-catch so that errors don't get eaten.
+    let self = this;
+    this.observer = {
+      containerStateChanged(container, oldState, newState) {
+        print(
+          "New state passed to containerStateChanged() should equal the " +
+            "container's current state"
+        );
+        Assert.equal(newState, container.state);
+
+        try {
+          switch (newState) {
+            case Ci.nsINavHistoryContainerResultNode.STATE_LOADING:
+              self.loading(container, newState, oldState);
+              break;
+            case Ci.nsINavHistoryContainerResultNode.STATE_OPENED:
+              self.opened(container, newState, oldState);
+              break;
+            case Ci.nsINavHistoryContainerResultNode.STATE_CLOSED:
+              self.closed(container, newState, oldState);
+              break;
+            default:
+              do_throw("Unexpected new state! " + newState);
+          }
+        } catch (err) {
+          do_throw(err);
+        }
+      },
+    };
+    this.result.addObserver(this.observer);
+
+    print("Opening container");
+    this.result.root.containerOpen = true;
+  },
+
+  /**
+   * Starts the test and returns a promise resolved when the test completes.
+   */
+  run() {
+    this.openContainer();
+    return this.deferNextTest.promise;
+  },
+
+  /**
+   * This must be called before run().  It adds a bookmark and sets up the
+   * test's result.  Override if need be.
+   */
+  async setup() {
+    // Populate the database with different types of bookmark items.
+    this.data = DataHelper.makeDataArray([
+      { type: "bookmark" },
+      { type: "separator" },
+      { type: "folder" },
+      { type: "bookmark", uri: "place:terms=foo" },
+    ]);
+    await task_populateDB(this.data);
+
+    // Make a query.
+    this.query = PlacesUtils.history.getNewQuery();
+    this.query.setParents([DataHelper.defaults.bookmark.parentGuid]);
+    this.opts = PlacesUtils.history.getNewQueryOptions();
+    this.opts.asyncEnabled = true;
+    this.result = PlacesUtils.history.executeQuery(this.query, this.opts);
+  },
+
+  /**
+   * Call this when the test has succeeded.  It cleans up resources and starts
+   * the next test.
+   */
+  success() {
+    this.result.removeObserver(this.observer);
+
+    // Resolve the promise object that indicates that the next test can be run.
+    this.deferNextTest.resolve();
+  },
+};
+
+/**
+ * This makes it a little bit easier to use the functions of head_queries.js.
+ */
+var DataHelper = {
+  defaults: {
+    bookmark: {
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      uri: "http://example.com/",
+      title: "test bookmark",
+    },
+
+    folder: {
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      title: "test folder",
+    },
+
+    separator: {
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    },
+  },
+
+  /**
+   * Converts an array of simple bookmark item descriptions to the more verbose
+   * format required by task_populateDB() in head_queries.js.
+   *
+   * @param  aData
+   *         An array of objects, each of which describes a bookmark item.
+   * @return An array of objects suitable for passing to populateDB().
+   */
+  makeDataArray: function DH_makeDataArray(aData) {
+    let self = this;
+    return aData.map(function (dat) {
+      let type = dat.type;
+      dat = self._makeDataWithDefaults(dat, self.defaults[type]);
+      switch (type) {
+        case "bookmark":
+          return {
+            isBookmark: true,
+            uri: dat.uri,
+            parentGuid: dat.parentGuid,
+            index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+            title: dat.title,
+            isInQuery: true,
+          };
+        case "separator":
+          return {
+            isSeparator: true,
+            parentGuid: dat.parentGuid,
+            index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+            isInQuery: true,
+          };
+        case "folder":
+          return {
+            isFolder: true,
+            parentGuid: dat.parentGuid,
+            index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+            title: dat.title,
+            isInQuery: true,
+          };
+        default:
+          do_throw("Unknown data type when populating DB: " + type);
+          return undefined;
+      }
+    });
+  },
+
+  /**
+   * Returns a copy of aData, except that any properties that are undefined but
+   * defined in aDefaults are set to the corresponding values in aDefaults.
+   *
+   * @param  aData
+   *         An object describing a bookmark item.
+   * @param  aDefaults
+   *         An object describing the default bookmark item.
+   * @return A copy of aData with defaults values set.
+   */
+  _makeDataWithDefaults: function DH__makeDataWithDefaults(aData, aDefaults) {
+    let dat = {};
+    for (let [prop, val] of Object.entries(aDefaults)) {
+      dat[prop] = aData.hasOwnProperty(prop) ? aData[prop] : val;
+    }
+    return dat;
+  },
+};
+
+add_task(async function test_async() {
+  for (let test of tests) {
+    await PlacesUtils.bookmarks.eraseEverything();
+
+    Object.setPrototypeOf(test, new Test());
+    await test.setup();
+
+    print("------ Running test: " + test.desc);
+    await test.run();
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  print("All tests done, exiting");
+});
diff --git a/toolkit/components/places/tests/queries/test_bookmarks.js b/toolkit/components/places/tests/queries/test_bookmarks.js
new file mode 100644
index 0000000000..b5f2ef754f
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_bookmarks.js
@@ -0,0 +1,105 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_eraseEverything() {
+  info("Test folder with eraseEverything");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        title: "remove-folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          { url: "http://mozilla.org/", title: "title 1" },
+          { url: "http://mozilla.org/", title: "title 2" },
+          { title: "sub-folder", type: PlacesUtils.bookmarks.TYPE_FOLDER },
+          { type: PlacesUtils.bookmarks.TYPE_SEPARATOR },
+        ],
+      },
+    ],
+  });
+
+  let unfiled = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  Assert.equal(unfiled.childCount, 1, "There should be 1 folder");
+  let folder = unfiled.getChild(0);
+  // Test dateAdded and lastModified properties.
+  Assert.equal(typeof folder.dateAdded, "number");
+  Assert.ok(folder.dateAdded > 0);
+  Assert.equal(typeof folder.lastModified, "number");
+  Assert.ok(folder.lastModified > 0);
+
+  let root = PlacesUtils.getFolderContents(folder.bookmarkGuid).root;
+  Assert.equal(root.childCount, 4, "The folder should have 4 children");
+  for (let i = 0; i < root.childCount; ++i) {
+    let node = root.getChild(i);
+    Assert.greater(node.itemId, 0, "The node should have an itemId");
+  }
+  Assert.equal(root.getChild(0).title, "title 1");
+  Assert.equal(root.getChild(1).title, "title 2");
+
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  // Refetch the guid to refresh the data.
+  unfiled = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  Assert.equal(unfiled.childCount, 0);
+  unfiled.containerOpen = false;
+});
+
+add_task(async function test_search_title() {
+  let title = "ZZZXXXYYY";
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://mozilla.org/",
+    title,
+  });
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "ZZZXXXYYY";
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  let node = root.getChild(0);
+  Assert.equal(node.title, title);
+
+  // Test dateAdded and lastModified properties.
+  Assert.equal(typeof node.dateAdded, "number");
+  Assert.ok(node.dateAdded > 0);
+  Assert.equal(typeof node.lastModified, "number");
+  Assert.ok(node.lastModified > 0);
+  Assert.equal(node.bookmarkGuid, bm.guid);
+
+  await PlacesUtils.bookmarks.remove(bm);
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+});
+
+add_task(async function test_long_title() {
+  let title = Array(TITLE_LENGTH_MAX + 5).join("A");
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://mozilla.org/",
+    title,
+  });
+  let root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  let node = root.getChild(0);
+  Assert.equal(node.title, title.substr(0, TITLE_LENGTH_MAX));
+
+  // Update with another long title.
+  let newTitle = Array(TITLE_LENGTH_MAX + 5).join("B");
+  bm.title = newTitle;
+  await PlacesUtils.bookmarks.update(bm);
+  Assert.equal(node.title, newTitle.substr(0, TITLE_LENGTH_MAX));
+
+  await PlacesUtils.bookmarks.remove(bm);
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_containersQueries_sorting.js b/toolkit/components/places/tests/queries/test_containersQueries_sorting.js
new file mode 100644
index 0000000000..9cdc0f2a52
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_containersQueries_sorting.js
@@ -0,0 +1,492 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Testing behavior of bug 473157
+ * "Want to sort history in container view without sorting the containers"
+ * and regression bug 488783
+ * Tags list no longer sorted (alphabetized).
+ * This test is for global testing sorting containers queries.
+ */
+
+// Globals and Constants
+
+var resultTypes = [
+  {
+    value: Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY,
+    name: "RESULTS_AS_DATE_QUERY",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY,
+    name: "RESULTS_AS_SITE_QUERY",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY,
+    name: "RESULTS_AS_DATE_SITE_QUERY",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAGS_ROOT,
+    name: "RESULTS_AS_TAGS_ROOT",
+  },
+];
+
+var sortingModes = [
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING,
+    name: "SORT_BY_TITLE_ASCENDING",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_DESCENDING,
+    name: "SORT_BY_TITLE_DESCENDING",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_ASCENDING,
+    name: "SORT_BY_DATE_ASCENDING",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING,
+    name: "SORT_BY_DATE_DESCENDING",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_ASCENDING,
+    name: "SORT_BY_DATEADDED_ASCENDING",
+  },
+  {
+    value: Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_DESCENDING,
+    name: "SORT_BY_DATEADDED_DESCENDING",
+  },
+];
+
+// These pages will be added from newest to oldest and from less visited to most
+// visited.
+var pages = [
+  "http://www.mozilla.org/c/",
+  "http://www.mozilla.org/a/",
+  "http://www.mozilla.org/b/",
+  "http://www.mozilla.com/c/",
+  "http://www.mozilla.com/a/",
+  "http://www.mozilla.com/b/",
+];
+
+var tags = ["mozilla", "Development", "test"];
+
+// Test Runner
+
+/**
+ * Enumerates all the sequences of the cartesian product of the arrays contained
+ * in aSequences.  Examples:
+ *
+ *   cartProd([[1, 2, 3], ["a", "b"]], callback);
+ *   // callback is called 3 * 2 = 6 times with the following arrays:
+ *   // [1, "a"], [1, "b"], [2, "a"], [2, "b"], [3, "a"], [3, "b"]
+ *
+ *   cartProd([["a"], [1, 2, 3], ["X", "Y"]], callback);
+ *   // callback is called 1 * 3 * 2 = 6 times with the following arrays:
+ *   // ["a", 1, "X"], ["a", 1, "Y"], ["a", 2, "X"], ["a", 2, "Y"],
+ *   // ["a", 3, "X"], ["a", 3, "Y"]
+ *
+ *   cartProd([[1], [2], [3], [4]], callback);
+ *   // callback is called 1 * 1 * 1 * 1 = 1 time with the following array:
+ *   // [1, 2, 3, 4]
+ *
+ *   cartProd([], callback);
+ *   // callback is 0 times
+ *
+ *   cartProd([[1, 2, 3, 4]], callback);
+ *   // callback is called 4 times with the following arrays:
+ *   // [1], [2], [3], [4]
+ *
+ * @param  aSequences
+ *         an array that contains an arbitrary number of arrays
+ * @param  aCallback
+ *         a function that is passed each sequence of the product as it's
+ *         computed
+ * @return the total number of sequences in the product
+ */
+function cartProd(aSequences, aCallback) {
+  if (aSequences.length === 0) {
+    return 0;
+  }
+
+  // For each sequence in aSequences, we maintain a pointer (an array index,
+  // really) to the element we're currently enumerating in that sequence
+  var seqEltPtrs = aSequences.map(i => 0);
+
+  var numProds = 0;
+  var done = false;
+  while (!done) {
+    numProds++;
+
+    // prod = sequence in product we're currently enumerating
+    var prod = [];
+    for (var i = 0; i < aSequences.length; i++) {
+      prod.push(aSequences[i][seqEltPtrs[i]]);
+    }
+    aCallback(prod);
+
+    // The next sequence in the product differs from the current one by just a
+    // single element.  Determine which element that is.  We advance the
+    // "rightmost" element pointer to the "right" by one.  If we move past the
+    // end of that pointer's sequence, reset the pointer to the first element
+    // in its sequence and then try the sequence to the "left", and so on.
+
+    // seqPtr = index of rightmost input sequence whose element pointer is not
+    // past the end of the sequence
+    var seqPtr = aSequences.length - 1;
+    while (!done) {
+      // Advance the rightmost element pointer.
+      seqEltPtrs[seqPtr]++;
+
+      // The rightmost element pointer is past the end of its sequence.
+      if (seqEltPtrs[seqPtr] >= aSequences[seqPtr].length) {
+        seqEltPtrs[seqPtr] = 0;
+        seqPtr--;
+
+        // All element pointers are past the ends of their sequences.
+        if (seqPtr < 0) {
+          done = true;
+        }
+      } else {
+        break;
+      }
+    }
+  }
+  return numProds;
+}
+
+/**
+ * Test a query based on passed-in options.
+ *
+ * @param aSequence
+ *        array of options we will use to query.
+ */
+function test_query_callback(aSequence) {
+  Assert.equal(aSequence.length, 2);
+  var resultType = aSequence[0];
+  var sortingMode = aSequence[1];
+  print(
+    "\n\n*** Testing default sorting for resultType (" +
+      resultType.name +
+      ") and sortingMode (" +
+      sortingMode.name +
+      ")"
+  );
+
+  // Skip invalid combinations sorting queries by none.
+  if (
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAGS_ROOT &&
+    (sortingMode.value == Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_ASCENDING ||
+      sortingMode.value == Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING)
+  ) {
+    // This is a bookmark query, we can't sort by visit date.
+    sortingMode.value = Ci.nsINavHistoryQueryOptions.SORT_BY_NONE;
+  }
+  if (
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY ||
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY ||
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  ) {
+    // This is an history query, we can't sort by date added.
+    if (
+      sortingMode.value ==
+        Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_ASCENDING ||
+      sortingMode.value ==
+        Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_DESCENDING
+    ) {
+      sortingMode.value = Ci.nsINavHistoryQueryOptions.SORT_BY_NONE;
+    }
+  }
+
+  // Create a new query with required options.
+  var query = PlacesUtils.history.getNewQuery();
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = resultType.value;
+  options.sortingMode = sortingMode.value;
+
+  // Compare resultset with expectedData.
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  if (
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY ||
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  ) {
+    // Date containers are always sorted by date descending.
+    check_children_sorting(
+      root,
+      Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING
+    );
+  } else {
+    check_children_sorting(root, sortingMode.value);
+  }
+
+  // Now Check sorting of the first child container.
+  var container = root
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  container.containerOpen = true;
+
+  if (
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  ) {
+    // Has more than one level of containers, first we check the sorting of
+    // the first level (site containers), those won't inherit sorting...
+    check_children_sorting(
+      container,
+      Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING
+    );
+    // ...then we check sorting of the contained urls, we can't inherit sorting
+    // since the above level does not inherit it, so they will be sorted by
+    // title ascending.
+    let innerContainer = container
+      .getChild(0)
+      .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+    innerContainer.containerOpen = true;
+    check_children_sorting(
+      innerContainer,
+      Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING
+    );
+    innerContainer.containerOpen = false;
+  } else if (
+    resultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAGS_ROOT
+  ) {
+    // Sorting mode for tag contents is hardcoded for now, to allow for faster
+    // duplicates filtering.
+    check_children_sorting(
+      container,
+      Ci.nsINavHistoryQueryOptions.SORT_BY_NONE
+    );
+  } else {
+    check_children_sorting(container, sortingMode.value);
+  }
+
+  container.containerOpen = false;
+  root.containerOpen = false;
+
+  test_result_sortingMode_change(result, resultType, sortingMode);
+}
+
+/**
+ * Sets sortingMode on aResult and checks for correct sorting of children.
+ * Containers should not change their sorting, while contained uri nodes should.
+ *
+ * @param aResult
+ *        nsINavHistoryResult generated by our query.
+ * @param aResultType
+ *        required result type.
+ * @param aOriginalSortingMode
+ *        the sorting mode from query's options.
+ */
+function test_result_sortingMode_change(
+  aResult,
+  aResultType,
+  aOriginalSortingMode
+) {
+  var root = aResult.root;
+  // Now we set sortingMode on the result and check that containers are not
+  // sorted while children are.
+  sortingModes.forEach(function sortingModeChecker(aForcedSortingMode) {
+    print(
+      "\n* Test setting sortingMode (" +
+        aForcedSortingMode.name +
+        ") " +
+        "on result with resultType (" +
+        aResultType.name +
+        ") " +
+        "currently sorted as (" +
+        aOriginalSortingMode.name +
+        ")"
+    );
+
+    aResult.sortingMode = aForcedSortingMode.value;
+    root.containerOpen = true;
+
+    if (
+      aResultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY ||
+      aResultType.value ==
+        Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+    ) {
+      // Date containers are always sorted by date descending.
+      check_children_sorting(
+        root,
+        Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING
+      );
+    } else if (
+      aResultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY &&
+      (aOriginalSortingMode.value ==
+        Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_ASCENDING ||
+        aOriginalSortingMode.value ==
+          Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING)
+    ) {
+      // Site containers don't have a good time property to sort by.
+      check_children_sorting(root, Ci.nsINavHistoryQueryOptions.SORT_BY_NONE);
+    } else {
+      check_children_sorting(root, aOriginalSortingMode.value);
+    }
+
+    // Now Check sorting of the first child container.
+    var container = root
+      .getChild(0)
+      .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+    container.containerOpen = true;
+
+    if (
+      aResultType.value ==
+      Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+    ) {
+      // Has more than one level of containers, first we check the sorting of
+      // the first level (site containers), those won't be sorted...
+      check_children_sorting(
+        container,
+        Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING
+      );
+      // ...then we check sorting of the second level of containers, result
+      // will sort them through recursiveSort.
+      let innerContainer = container
+        .getChild(0)
+        .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+      innerContainer.containerOpen = true;
+      check_children_sorting(innerContainer, aForcedSortingMode.value);
+      innerContainer.containerOpen = false;
+    } else {
+      if (
+        aResultType.value ==
+          Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY ||
+        aResultType.value ==
+          Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY ||
+        aResultType.value == Ci.nsINavHistoryQueryOptions.RESULTS_AS_SITE_QUERY
+      ) {
+        // Date containers are always sorted by date descending.
+        check_children_sorting(root, Ci.nsINavHistoryQueryOptions.SORT_BY_NONE);
+      } else {
+        check_children_sorting(root, aOriginalSortingMode.value);
+      }
+
+      // Children should always be sorted.
+      check_children_sorting(container, aForcedSortingMode.value);
+    }
+
+    container.containerOpen = false;
+    root.containerOpen = false;
+  });
+}
+
+/**
+ * Test if children of aRootNode are correctly sorted.
+ * @param aRootNode
+ *        already opened root node from our query's result.
+ * @param aExpectedSortingMode
+ *        The sortingMode we expect results to be.
+ */
+function check_children_sorting(aRootNode, aExpectedSortingMode) {
+  var results = [];
+  print("Found children:");
+  for (let i = 0; i < aRootNode.childCount; i++) {
+    results[i] = aRootNode.getChild(i);
+    print(i + " " + results[i].title);
+  }
+
+  // Helper for case insensitive string comparison.
+  function caseInsensitiveStringComparator(a, b) {
+    var aLC = a.toLowerCase();
+    var bLC = b.toLowerCase();
+    if (aLC < bLC) {
+      return -1;
+    }
+    if (aLC > bLC) {
+      return 1;
+    }
+    return 0;
+  }
+
+  // Get a comparator based on expected sortingMode.
+  var comparator;
+  switch (aExpectedSortingMode) {
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_NONE:
+      comparator = function (a, b) {
+        return 0;
+      };
+      break;
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING:
+      comparator = function (a, b) {
+        return caseInsensitiveStringComparator(a.title, b.title);
+      };
+      break;
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_DESCENDING:
+      comparator = function (a, b) {
+        return -caseInsensitiveStringComparator(a.title, b.title);
+      };
+      break;
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_ASCENDING:
+      comparator = function (a, b) {
+        return a.time - b.time;
+      };
+      break;
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING:
+      comparator = function (a, b) {
+        return b.time - a.time;
+      };
+    // fall through - we shouldn't do this, see bug 1572437.
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_ASCENDING:
+      comparator = function (a, b) {
+        return a.dateAdded - b.dateAdded;
+      };
+      break;
+    case Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_DESCENDING:
+      comparator = function (a, b) {
+        return b.dateAdded - a.dateAdded;
+      };
+      break;
+    default:
+      do_throw("Unknown sorting type: " + aExpectedSortingMode);
+  }
+
+  // Make an independent copy of the results array and sort it.
+  var sortedResults = results.slice();
+  sortedResults.sort(comparator);
+  // Actually compare returned children with our sorted array.
+  for (let i = 0; i < sortedResults.length; i++) {
+    if (sortedResults[i].title != results[i].title) {
+      print(
+        i +
+          " index wrong, expected " +
+          sortedResults[i].title +
+          " found " +
+          results[i].title
+      );
+    }
+    Assert.equal(sortedResults[i].title, results[i].title);
+  }
+}
+
+// Main
+
+add_task(async function test_containersQueries_sorting() {
+  // Add visits, bookmarks and tags to our database.
+  var timeInMilliseconds = Date.now();
+  var visitCount = 0;
+  var dayOffset = 0;
+  var visits = [];
+  pages.forEach(aPageUrl =>
+    visits.push({
+      isVisit: true,
+      isBookmark: true,
+      transType: Ci.nsINavHistoryService.TRANSITION_TYPED,
+      uri: aPageUrl,
+      title: aPageUrl,
+      // subtract 5 hours per iteration, to expose more than one day container.
+      lastVisit: (timeInMilliseconds - 18000 * 1000 * dayOffset++) * 1000,
+      visitCount: visitCount++,
+      isTag: true,
+      tagArray: tags,
+      isInQuery: true,
+    })
+  );
+  await task_populateDB(visits);
+
+  cartProd([resultTypes, sortingModes], test_query_callback);
+});
diff --git a/toolkit/components/places/tests/queries/test_downloadHistory_liveUpdate.js b/toolkit/components/places/tests/queries/test_downloadHistory_liveUpdate.js
new file mode 100644
index 0000000000..ba0f528b62
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_downloadHistory_liveUpdate.js
@@ -0,0 +1,121 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures that download history (filtered by transition) queries
+// don't invalidate (and requery) too often.
+
+function accumulateNotifications(result) {
+  let notifications = [];
+  let resultObserver = new Proxy(NavHistoryResultObserver, {
+    get(target, name) {
+      if (name == "check") {
+        result.removeObserver(resultObserver, false);
+        return expectedNotifications =>
+          Assert.deepEqual(notifications, expectedNotifications);
+      }
+      // ignore a few uninteresting notifications.
+      if (["QueryInterface", "containerStateChanged"].includes(name)) {
+        return () => {};
+      }
+      return () => {
+        notifications.push(name);
+      };
+    },
+  });
+  result.addObserver(resultObserver, false);
+  return resultObserver;
+}
+
+add_task(async function test_downloadhistory_query_notifications() {
+  const MAX_RESULTS = 5;
+  let query = PlacesUtils.history.getNewQuery();
+  query.setTransitions([PlacesUtils.history.TRANSITIONS.DOWNLOAD]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING;
+  options.maxResults = MAX_RESULTS;
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let notifications = accumulateNotifications(result);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+  // Add more maxResults downloads in order.
+  let transitions = Object.values(PlacesUtils.history.TRANSITIONS);
+  for (let transition of transitions) {
+    let uri = "http://fx-search.com/" + transition;
+    await PlacesTestUtils.addVisits({
+      uri,
+      transition,
+      title: "test " + transition,
+    });
+    // For each visit also set apart:
+    //  - a bookmark
+    //  - an annotation
+    //  - an icon
+    await PlacesUtils.bookmarks.insert({
+      url: uri,
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+    await PlacesUtils.history.update({
+      url: uri,
+      annotations: new Map([["test/anno", "testValue"]]),
+    });
+    await PlacesTestUtils.addFavicons(new Map([[uri, SMALLPNG_DATA_URI.spec]]));
+  }
+  // Remove all the visits one by one.
+  for (let transition of transitions) {
+    let uri = Services.io.newURI("http://fx-search.com/" + transition);
+    await PlacesUtils.history.remove(uri);
+  }
+  root.containerOpen = false;
+  // We pretty much don't want to see invalidateContainer here, because that
+  // means we requeried.
+  // We also don't want to see changes caused by filtered-out transition types.
+  notifications.check([
+    "nodeHistoryDetailsChanged",
+    "nodeInserted",
+    "nodeTitleChanged",
+    "nodeIconChanged",
+    "nodeRemoved",
+  ]);
+});
+
+add_task(async function test_downloadhistory_query_filtering() {
+  const MAX_RESULTS = 3;
+  let query = PlacesUtils.history.getNewQuery();
+  query.setTransitions([PlacesUtils.history.TRANSITIONS.DOWNLOAD]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING;
+  options.maxResults = MAX_RESULTS;
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 0, "No visits found");
+  // Add more than maxResults downloads.
+  let uris = [];
+  // Define a monotonic visit date to ensure results order stability.
+  let visitDate = Date.now() * 1000;
+  for (let i = 0; i < MAX_RESULTS + 1; ++i, visitDate += 1000) {
+    let uri = `http://fx-search.com/download/${i}`;
+    await PlacesTestUtils.addVisits({
+      uri,
+      transition: PlacesUtils.history.TRANSITIONS.DOWNLOAD,
+      visitDate,
+    });
+    uris.push(uri);
+  }
+  // Add an older download visit out of the maxResults timeframe.
+  await PlacesTestUtils.addVisits({
+    uri: `http://fx-search.com/download/unordered`,
+    transition: PlacesUtils.history.TRANSITIONS.DOWNLOAD,
+    visitDate: new Date(Date.now() - 7200000),
+  });
+
+  Assert.equal(root.childCount, MAX_RESULTS, "Result should be limited");
+  // Invert the uris array because we are sorted by date descending.
+  uris.reverse();
+  for (let i = 0; i < root.childCount; ++i) {
+    let node = root.getChild(i);
+    Assert.equal(node.uri, uris[i], "Found the expected uri");
+  }
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_excludeQueries.js b/toolkit/components/places/tests/queries/test_excludeQueries.js
new file mode 100644
index 0000000000..c48f84c7f4
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_excludeQueries.js
@@ -0,0 +1,118 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+var bm;
+var fakeQuery;
+var folderShortcut;
+
+add_task(async function setup() {
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://example.com/",
+    title: "a bookmark",
+  });
+  fakeQuery = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "place:terms=foo",
+    title: "a bookmark",
+  });
+  folderShortcut = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: `place:parent=${PlacesUtils.bookmarks.toolbarGuid}`,
+    title: "a bookmark",
+  });
+
+  checkBookmarkObject(bm);
+  checkBookmarkObject(fakeQuery);
+  checkBookmarkObject(folderShortcut);
+});
+
+add_task(async function test_bookmarks_url_query_implicit_exclusions() {
+  // When we run bookmarks url queries, we implicity filter out queries and
+  // folder shortcuts regardless of excludeQueries. They don't make sense to
+  // include in the results.
+  let expectedGuids = [bm.guid];
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.excludeQueries = true;
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  Assert.equal(
+    root.childCount,
+    expectedGuids.length,
+    "Checking root child count"
+  );
+  for (let i = 0; i < expectedGuids.length; i++) {
+    Assert.equal(
+      root.getChild(i).bookmarkGuid,
+      expectedGuids[i],
+      "should have got the expected item"
+    );
+  }
+
+  root.containerOpen = false;
+});
+
+add_task(async function test_bookmarks_excludeQueries() {
+  // When excluding queries, we exclude actual queries, but not folder shortcuts.
+  let expectedGuids = [bm.guid, folderShortcut.guid];
+  let query = {},
+    options = {};
+  let queryString = `place:parent=${PlacesUtils.bookmarks.unfiledGuid}&excludeQueries=1`;
+  PlacesUtils.history.queryStringToQuery(queryString, query, options);
+
+  let root = PlacesUtils.history.executeQuery(query.value, options.value).root;
+  root.containerOpen = true;
+
+  Assert.equal(
+    root.childCount,
+    expectedGuids.length,
+    "Checking root child count"
+  );
+  for (let i = 0; i < expectedGuids.length; i++) {
+    Assert.equal(
+      root.getChild(i).bookmarkGuid,
+      expectedGuids[i],
+      "should have got the expected item"
+    );
+  }
+
+  root.containerOpen = false;
+});
+
+add_task(async function test_search_excludesQueries() {
+  // Searching implicity removes queries and folder shortcuts even if excludeQueries
+  // is not specified.
+  let expectedGuids = [bm.guid];
+
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "bookmark";
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  Assert.equal(
+    root.childCount,
+    expectedGuids.length,
+    "Checking root child count"
+  );
+  for (let i = 0; i < expectedGuids.length; i++) {
+    Assert.equal(
+      root.getChild(i).bookmarkGuid,
+      expectedGuids[i],
+      "should have got the expected item"
+    );
+  }
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_history_queries_tags_liveUpdate.js b/toolkit/components/places/tests/queries/test_history_queries_tags_liveUpdate.js
new file mode 100644
index 0000000000..53f680bf53
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_history_queries_tags_liveUpdate.js
@@ -0,0 +1,189 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures that tags changes are correctly live-updated in a history
+// query.
+
+let timeInMicroseconds = PlacesUtils.toPRTime(Date.now() - 10000);
+
+function newTimeInMicroseconds() {
+  timeInMicroseconds = timeInMicroseconds + 1000;
+  return timeInMicroseconds;
+}
+
+var gTestData = [
+  {
+    isVisit: true,
+    uri: "http://example.com/1/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    isBookmark: true,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    title: "example1",
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/2/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    isBookmark: true,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    title: "example2",
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/3/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    isBookmark: true,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    title: "example3",
+  },
+];
+
+function newQueryWithOptions() {
+  return [
+    PlacesUtils.history.getNewQuery(),
+    PlacesUtils.history.getNewQueryOptions(),
+  ];
+}
+
+function testQueryContents(aQuery, aOptions, aCallback) {
+  let root = PlacesUtils.history.executeQuery(aQuery, aOptions).root;
+  root.containerOpen = true;
+  aCallback(root);
+  root.containerOpen = false;
+}
+
+add_task(async function test_initialize() {
+  await task_populateDB(gTestData);
+});
+
+add_task(function pages_query() {
+  let [query, options] = newQueryWithOptions();
+  testQueryContents(query, options, function (root) {
+    compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+    for (let i = 0; i < root.childCount; i++) {
+      let node = root.getChild(i);
+      let uri = NetUtil.newURI(node.uri);
+      Assert.equal(node.tags, null);
+      PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, "test-tag");
+      PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, null);
+    }
+  });
+});
+
+add_task(function visits_query() {
+  let [query, options] = newQueryWithOptions();
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  testQueryContents(query, options, function (root) {
+    compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+    for (let i = 0; i < root.childCount; i++) {
+      let node = root.getChild(i);
+      let uri = NetUtil.newURI(node.uri);
+      Assert.equal(node.tags, null);
+      PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, "test-tag");
+      PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, null);
+    }
+  });
+});
+
+add_task(function bookmarks_query() {
+  let [query, options] = newQueryWithOptions();
+  query.setParents([PlacesUtils.bookmarks.unfiledGuid]);
+  testQueryContents(query, options, function (root) {
+    compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+    for (let i = 0; i < root.childCount; i++) {
+      let node = root.getChild(i);
+      let uri = NetUtil.newURI(node.uri);
+      Assert.equal(node.tags, null);
+      PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, "test-tag");
+      PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, null);
+    }
+  });
+});
+
+add_task(function pages_searchterm_query() {
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "example";
+  testQueryContents(query, options, function (root) {
+    compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+    for (let i = 0; i < root.childCount; i++) {
+      let node = root.getChild(i);
+      let uri = NetUtil.newURI(node.uri);
+      Assert.equal(node.tags, null);
+      PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, "test-tag");
+      PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, null);
+    }
+  });
+});
+
+add_task(function visits_searchterm_query() {
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "example";
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  testQueryContents(query, options, function (root) {
+    compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+    for (let i = 0; i < root.childCount; i++) {
+      let node = root.getChild(i);
+      let uri = NetUtil.newURI(node.uri);
+      Assert.equal(node.tags, null);
+      PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, "test-tag");
+      PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+      Assert.equal(node.tags, null);
+    }
+  });
+});
+
+add_task(async function pages_searchterm_is_tag_query() {
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "test-tag";
+  let root;
+  testQueryContents(query, options, rv => (root = rv));
+  compareArrayToResult([], root);
+  for (let data of gTestData) {
+    let uri = NetUtil.newURI(data.uri);
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: uri,
+      title: data.title,
+    });
+    PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+    compareArrayToResult([data], root);
+    PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+    compareArrayToResult([], root);
+  }
+});
+
+add_task(async function visits_searchterm_is_tag_query() {
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "test-tag";
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  let root;
+  testQueryContents(query, options, rv => (root = rv));
+  compareArrayToResult([], root);
+  for (let data of gTestData) {
+    let uri = NetUtil.newURI(data.uri);
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: uri,
+      title: data.title,
+    });
+    PlacesUtils.tagging.tagURI(uri, ["test-tag"]);
+    compareArrayToResult([data], root);
+    PlacesUtils.tagging.untagURI(uri, ["test-tag"]);
+    compareArrayToResult([], root);
+  }
+});
diff --git a/toolkit/components/places/tests/queries/test_history_queries_titles_liveUpdate.js b/toolkit/components/places/tests/queries/test_history_queries_titles_liveUpdate.js
new file mode 100644
index 0000000000..ac3931892f
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_history_queries_titles_liveUpdate.js
@@ -0,0 +1,217 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures that tags changes are correctly live-updated in a history
+// query.
+
+let timeInMicroseconds = PlacesUtils.toPRTime(Date.now() - 10000);
+
+function newTimeInMicroseconds() {
+  timeInMicroseconds = timeInMicroseconds + 1000;
+  return timeInMicroseconds;
+}
+
+var gTestData = [
+  {
+    isVisit: true,
+    uri: "http://example.com/1/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    title: "title1",
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/2/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    title: "title2",
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/3/",
+    lastVisit: newTimeInMicroseconds(),
+    isInQuery: true,
+    title: "title3",
+  },
+];
+
+function searchNodeHavingUrl(aRoot, aUrl) {
+  for (let i = 0; i < aRoot.childCount; i++) {
+    if (aRoot.getChild(i).uri == aUrl) {
+      return aRoot.getChild(i);
+    }
+  }
+  return undefined;
+}
+
+function newQueryWithOptions() {
+  return [
+    PlacesUtils.history.getNewQuery(),
+    PlacesUtils.history.getNewQueryOptions(),
+  ];
+}
+
+add_task(async function pages_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+  for (let i = 0; i < root.childCount; i++) {
+    let node = root.getChild(i);
+    Assert.equal(node.title, gTestData[i].title);
+    let uri = NetUtil.newURI(node.uri);
+    await PlacesTestUtils.addVisits({ uri, title: "changedTitle" });
+    Assert.equal(node.title, "changedTitle");
+    await PlacesTestUtils.addVisits({ uri, title: gTestData[i].title });
+    Assert.equal(node.title, gTestData[i].title);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function visits_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+
+  for (let testData of gTestData) {
+    let uri = NetUtil.newURI(testData.uri);
+    let node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, testData.title);
+    await PlacesTestUtils.addVisits({ uri, title: "changedTitle" });
+    node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, "changedTitle");
+    await PlacesTestUtils.addVisits({ uri, title: testData.title });
+    node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, testData.title);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function pages_searchterm_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "example";
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+  for (let i = 0; i < root.childCount; i++) {
+    let node = root.getChild(i);
+    let uri = NetUtil.newURI(node.uri);
+    Assert.equal(node.title, gTestData[i].title);
+    await PlacesTestUtils.addVisits({ uri, title: "changedTitle" });
+    Assert.equal(node.title, "changedTitle");
+    await PlacesTestUtils.addVisits({ uri, title: gTestData[i].title });
+    Assert.equal(node.title, gTestData[i].title);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function visits_searchterm_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "example";
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  compareArrayToResult([gTestData[0], gTestData[1], gTestData[2]], root);
+  for (let testData of gTestData) {
+    let uri = NetUtil.newURI(testData.uri);
+    let node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, testData.title);
+    await PlacesTestUtils.addVisits({ uri, title: "changedTitle" });
+    node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, "changedTitle");
+    await PlacesTestUtils.addVisits({ uri, title: testData.title });
+    node = searchNodeHavingUrl(root, testData.uri);
+    Assert.equal(node.title, testData.title);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function pages_searchterm_is_title_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "match";
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  compareArrayToResult([], root);
+  for (let data of gTestData) {
+    let uri = NetUtil.newURI(data.uri);
+    let origTitle = data.title;
+    data.title = "match";
+    await PlacesTestUtils.addVisits({
+      uri,
+      title: data.title,
+      visitDate: data.lastVisit,
+    });
+    compareArrayToResult([data], root);
+    data.title = origTitle;
+    await PlacesTestUtils.addVisits({
+      uri,
+      title: data.title,
+      visitDate: data.lastVisit,
+    });
+    compareArrayToResult([], root);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function visits_searchterm_is_title_query() {
+  await task_populateDB(gTestData);
+
+  let [query, options] = newQueryWithOptions();
+  query.searchTerms = "match";
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  compareArrayToResult([], root);
+  for (let data of gTestData) {
+    let uri = NetUtil.newURI(data.uri);
+    let origTitle = data.title;
+    data.title = "match";
+
+    info("Adding " + uri.spec);
+    await PlacesTestUtils.addVisits({
+      uri,
+      title: data.title,
+      visitDate: data.lastVisit,
+    });
+
+    compareArrayToResult([data], root);
+    data.title = origTitle;
+    info("Clobbering " + uri.spec);
+    await PlacesTestUtils.addVisits({
+      uri,
+      title: data.title,
+      visitDate: data.lastVisit,
+    });
+
+    compareArrayToResult([], root);
+  }
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/queries/test_onlyBookmarked.js b/toolkit/components/places/tests/queries/test_onlyBookmarked.js
new file mode 100644
index 0000000000..28c42c190c
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_onlyBookmarked.js
@@ -0,0 +1,110 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The next thing we do is create a test database for us.  Each test runs with
+ * its own database (tail_queries.js will clear it after the run).  Take a look
+ * at the queryData object in head_queries.js, and you'll see how this object
+ * works.  You can call it anything you like, but I usually use "testData".
+ * I'll include a couple of example entries in the database.
+ *
+ * Note that to use the compareArrayToResult API, you need to put all the
+ * results that are in the query set at the top of the testData list, and those
+ * results MUST be in the same sort order as the items in the resulting query.
+ */
+
+var testData = [
+  // Add a bookmark that should be in the results
+  {
+    isBookmark: true,
+    uri: "http://bookmarked.com/",
+    title: "",
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    isInQuery: true,
+  },
+
+  // Add a bookmark that should not be in the results
+  {
+    isBookmark: true,
+    uri: "http://bookmarked-elsewhere.com/",
+    title: "",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    isInQuery: false,
+  },
+
+  // Add an un-bookmarked visit
+  {
+    isVisit: true,
+    uri: "http://notbookmarked.com/",
+    title: "",
+    isInQuery: false,
+  },
+];
+
+add_task(async function test_onlyBookmarked() {
+  // This function in head_queries.js creates our database with the above data
+  await task_populateDB(testData);
+
+  // Query
+  var query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  query.onlyBookmarked = true;
+
+  // query options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_HISTORY;
+
+  // Results - this gets the result set and opens it for reading and modification.
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  // You can use this to compare the data in the array with the result set,
+  // if the array's isInQuery: true items are sorted the same way as the result
+  // set.
+  info("begin first test");
+  compareArrayToResult(testData, root);
+  info("end first test");
+
+  // Test live-update
+  var liveUpdateTestData = [
+    // Add a bookmark that should show up
+    {
+      isBookmark: true,
+      uri: "http://bookmarked2.com/",
+      title: "",
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+      isInQuery: true,
+    },
+
+    // Add a bookmark that should not show up
+    {
+      isBookmark: true,
+      uri: "http://bookmarked-elsewhere2.com/",
+      title: "",
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+      isInQuery: false,
+    },
+  ];
+
+  await task_populateDB(liveUpdateTestData); // add to the db
+
+  // add to the test data
+  testData.push(liveUpdateTestData[0]);
+  testData.push(liveUpdateTestData[1]);
+
+  // re-query and test
+  info("begin live-update test");
+  compareArrayToResult(testData, root);
+  info("end live-update test");
+
+  // Close the container when finished
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_options_inherit.js b/toolkit/components/places/tests/queries/test_options_inherit.js
new file mode 100644
index 0000000000..ae43350eda
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_options_inherit.js
@@ -0,0 +1,118 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Tests inheritance of certain query options like:
+ * excludeItems, excludeQueries, expandQueries.
+ */
+
+"use strict";
+
+add_task(async function () {
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        title: "folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            title: "query",
+            url:
+              "place:queryType=" +
+              Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS,
+          },
+          { title: "bm", url: "http://example.com" },
+          {
+            type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+          },
+        ],
+      },
+      { title: "bm", url: "http://example.com" },
+      {
+        type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      },
+    ],
+  });
+
+  await test_query({}, 3, 3, 2);
+  await test_query({ expandQueries: false }, 3, 3, 0);
+  await test_query({ excludeItems: true }, 1, 1, 0);
+  await test_query({ excludeItems: true, expandQueries: false }, 1, 1, 0);
+  await test_query({ excludeItems: true, excludeQueries: true }, 1, 0, 0);
+});
+
+async function test_query(
+  opts,
+  expectedRootCc,
+  expectedFolderCc,
+  expectedQueryCc
+) {
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.unfiledGuid]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  for (const [o, v] of Object.entries(opts)) {
+    info(`Setting ${o} to ${v}`);
+    options[o] = v;
+  }
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  Assert.equal(root.childCount, expectedRootCc, "Checking root child count");
+  if (root.childCount > 0) {
+    let folder = root.getChild(0);
+    Assert.equal(folder.title, "folder", "Found the expected folder");
+
+    // Check the folder uri doesn't reflect the root options, since those
+    // options are inherited and not part of this node declaration.
+    checkURIOptions(folder.uri);
+
+    PlacesUtils.asContainer(folder).containerOpen = true;
+    Assert.equal(
+      folder.childCount,
+      expectedFolderCc,
+      "Checking folder child count"
+    );
+    if (folder.childCount) {
+      let placeQuery = folder.getChild(0);
+      PlacesUtils.asQuery(placeQuery).containerOpen = true;
+      Assert.equal(
+        placeQuery.childCount,
+        expectedQueryCc,
+        "Checking query child count"
+      );
+      placeQuery.containerOpen = false;
+    }
+    folder.containerOpen = false;
+  }
+  let f = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+  });
+  checkURIOptions(root.getChild(root.childCount - 1).uri);
+  await PlacesUtils.bookmarks.remove(f);
+
+  root.containerOpen = false;
+}
+
+function checkURIOptions(uri) {
+  info("Checking options for uri " + uri);
+  let folderOptions = {};
+  PlacesUtils.history.queryStringToQuery(uri, {}, folderOptions);
+  folderOptions = folderOptions.value;
+  Assert.equal(
+    folderOptions.excludeItems,
+    false,
+    "ExcludeItems should not be changed"
+  );
+  Assert.equal(
+    folderOptions.excludeQueries,
+    false,
+    "ExcludeQueries should not be changed"
+  );
+  Assert.equal(
+    folderOptions.expandQueries,
+    true,
+    "ExpandQueries should not be changed"
+  );
+}
diff --git a/toolkit/components/places/tests/queries/test_queryMultipleFolder.js b/toolkit/components/places/tests/queries/test_queryMultipleFolder.js
new file mode 100644
index 0000000000..7c24bef74e
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_queryMultipleFolder.js
@@ -0,0 +1,106 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+var folderGuids = [];
+var bookmarkGuids = [];
+
+add_task(async function setup() {
+  // adding bookmarks in the folders
+  for (let i = 0; i < 3; ++i) {
+    let folder = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      title: `Folder${i}`,
+    });
+    folderGuids.push(folder.guid);
+
+    for (let j = 0; j < 7; ++j) {
+      let bm = await PlacesUtils.bookmarks.insert({
+        parentGuid: folderGuids[i],
+        url: `http://Bookmark${i}_${j}.com`,
+        title: "",
+      });
+      bookmarkGuids.push(bm.guid);
+    }
+  }
+});
+
+add_task(async function test_queryMultipleFolders_ids() {
+  // using queryStringToQuery
+  let query = {},
+    options = {};
+  let maxResults = 20;
+  let queryString = `place:${folderGuids
+    .map(guid => "parent=" + guid)
+    .join("&")}&sort=5&maxResults=${maxResults}`;
+  PlacesUtils.history.queryStringToQuery(queryString, query, options);
+  let rootNode = PlacesUtils.history.executeQuery(
+    query.value,
+    options.value
+  ).root;
+  rootNode.containerOpen = true;
+  let resultLength = rootNode.childCount;
+  Assert.equal(resultLength, maxResults);
+  for (let i = 0; i < resultLength; ++i) {
+    let node = rootNode.getChild(i);
+    Assert.equal(bookmarkGuids[i], node.bookmarkGuid, node.uri);
+  }
+  rootNode.containerOpen = false;
+
+  // using getNewQuery and getNewQueryOptions
+  query = PlacesUtils.history.getNewQuery();
+  options = PlacesUtils.history.getNewQueryOptions();
+  query.setParents(folderGuids);
+  options.sortingMode = options.SORT_BY_URI_ASCENDING;
+  options.maxResults = maxResults;
+  rootNode = PlacesUtils.history.executeQuery(query, options).root;
+  rootNode.containerOpen = true;
+  resultLength = rootNode.childCount;
+  Assert.equal(resultLength, maxResults);
+  for (let i = 0; i < resultLength; ++i) {
+    let node = rootNode.getChild(i);
+    Assert.equal(bookmarkGuids[i], node.bookmarkGuid, node.uri);
+  }
+  rootNode.containerOpen = false;
+});
+
+add_task(async function test_queryMultipleFolders_guids() {
+  // using queryStringToQuery
+  let query = {},
+    options = {};
+  let maxResults = 20;
+  let queryString = `place:${folderGuids
+    .map(guid => "parent=" + guid)
+    .join("&")}&sort=5&maxResults=${maxResults}`;
+  PlacesUtils.history.queryStringToQuery(queryString, query, options);
+  let rootNode = PlacesUtils.history.executeQuery(
+    query.value,
+    options.value
+  ).root;
+  rootNode.containerOpen = true;
+  let resultLength = rootNode.childCount;
+  Assert.equal(resultLength, maxResults);
+  for (let i = 0; i < resultLength; ++i) {
+    let node = rootNode.getChild(i);
+    Assert.equal(bookmarkGuids[i], node.bookmarkGuid, node.uri);
+  }
+  rootNode.containerOpen = false;
+
+  // using getNewQuery and getNewQueryOptions
+  query = PlacesUtils.history.getNewQuery();
+  options = PlacesUtils.history.getNewQueryOptions();
+  query.setParents(folderGuids);
+  options.sortingMode = options.SORT_BY_URI_ASCENDING;
+  options.maxResults = maxResults;
+  rootNode = PlacesUtils.history.executeQuery(query, options).root;
+  rootNode.containerOpen = true;
+  resultLength = rootNode.childCount;
+  Assert.equal(resultLength, maxResults);
+  for (let i = 0; i < resultLength; ++i) {
+    let node = rootNode.getChild(i);
+    Assert.equal(bookmarkGuids[i], node.bookmarkGuid, node.uri);
+  }
+  rootNode.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_querySerialization.js b/toolkit/components/places/tests/queries/test_querySerialization.js
new file mode 100644
index 0000000000..071b2b40c2
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_querySerialization.js
@@ -0,0 +1,746 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Tests Places query serialization.  Associated bug is
+ * https://bugzilla.mozilla.org/show_bug.cgi?id=370197
+ *
+ * The simple idea behind this test is to try out different combinations of
+ * query switches and ensure that queries are the same before serialization
+ * as they are after de-serialization.
+ *
+ * In the code below, "switch" refers to a query option -- "option" in a broad
+ * sense, not nsINavHistoryQueryOptions specifically (which is why we refer to
+ * them as switches, not options).  Both nsINavHistoryQuery and
+ * nsINavHistoryQueryOptions allow you to specify switches that affect query
+ * strings.  nsINavHistoryQuery instances have attributes hasBeginTime,
+ * hasEndTime, hasSearchTerms, and so on.  nsINavHistoryQueryOptions instances
+ * have attributes sortingMode, resultType, excludeItems, etc.
+ *
+ * Ideally we would like to test all 2^N subsets of switches, where N is the
+ * total number of switches; switches might interact in erroneous or other ways
+ * we do not expect.  However, since N is large (21 at this time), that's
+ * impractical for a single test in a suite.
+ *
+ * Instead we choose all possible subsets of a certain, smaller size.  In fact
+ * we begin by choosing CHOOSE_HOW_MANY_SWITCHES_LO and ramp up to
+ * CHOOSE_HOW_MANY_SWITCHES_HI.
+ *
+ * There are two more wrinkles.  First, for some switches we'd like to be able to
+ * test multiple values.  For example, it seems like a good idea to test both an
+ * empty string and a non-empty string for switch nsINavHistoryQuery.searchTerms.
+ * When switches have more than one value for a test run, we use the Cartesian
+ * product of their values to generate all possible combinations of values.
+ *
+ * To summarize, here's how this test works:
+ *
+ * - For n = CHOOSE_HOW_MANY_SWITCHES_LO to CHOOSE_HOW_MANY_SWITCHES_HI:
+ *   - From the total set of switches choose all possible subsets of size n.
+ *     For each of those subsets s:
+ *     - Collect the test runs of each switch in subset s and take their
+ *       Cartesian product.  For each sequence in the product:
+ *       - Create nsINavHistoryQuery and nsINavHistoryQueryOptions objects
+ *         with the chosen switches and test run values.
+ *       - Serialize the query.
+ *       - De-serialize and ensure that the de-serialized query objects equal
+ *         the originals.
+ */
+
+const CHOOSE_HOW_MANY_SWITCHES_LO = 1;
+const CHOOSE_HOW_MANY_SWITCHES_HI = 2;
+
+// The switches are represented by objects below, in arrays querySwitches and
+// queryOptionSwitches.  Use them to set up test runs.
+//
+// Some switches have special properties (where noted), but all switches must
+// have the following properties:
+//
+//   matches: A function that takes two nsINavHistoryQuery objects (in the case
+//            of nsINavHistoryQuery switches) or two nsINavHistoryQueryOptions
+//            objects (for nsINavHistoryQueryOptions switches) and returns true
+//            if the values of the switch in the two objects are equal.  This is
+//            the foundation of how we determine if two queries are equal.
+//   runs:    An array of functions.  Each function takes an nsINavHistoryQuery
+//            object and an nsINavHistoryQueryOptions object.  The functions
+//            should set the attributes of one of the two objects as appropriate
+//            to their switches.  This is how switch values are set for each test
+//            run.
+//
+// The following properties are optional:
+//
+//   desc:    An informational string to print out during runs when the switch
+//            is chosen.  Hopefully helpful if the test fails.
+
+// nsINavHistoryQuery switches
+const querySwitches = [
+  // hasBeginTime
+  {
+    // flag and subswitches are used by the flagSwitchMatches function.  Several
+    // of the nsINavHistoryQuery switches (like this one) are really guard flags
+    // that indicate if other "subswitches" are enabled.
+    flag: "hasBeginTime",
+    subswitches: ["beginTime", "beginTimeReference", "absoluteBeginTime"],
+    desc: "nsINavHistoryQuery.hasBeginTime",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.beginTime = Date.now() * 1000;
+        aQuery.beginTimeReference = Ci.nsINavHistoryQuery.TIME_RELATIVE_EPOCH;
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.beginTime = Date.now() * 1000;
+        aQuery.beginTimeReference = Ci.nsINavHistoryQuery.TIME_RELATIVE_TODAY;
+      },
+    ],
+  },
+  // hasEndTime
+  {
+    flag: "hasEndTime",
+    subswitches: ["endTime", "endTimeReference", "absoluteEndTime"],
+    desc: "nsINavHistoryQuery.hasEndTime",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.endTime = Date.now() * 1000;
+        aQuery.endTimeReference = Ci.nsINavHistoryQuery.TIME_RELATIVE_EPOCH;
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.endTime = Date.now() * 1000;
+        aQuery.endTimeReference = Ci.nsINavHistoryQuery.TIME_RELATIVE_TODAY;
+      },
+    ],
+  },
+  // hasSearchTerms
+  {
+    flag: "hasSearchTerms",
+    subswitches: ["searchTerms"],
+    desc: "nsINavHistoryQuery.hasSearchTerms",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.searchTerms = "shrimp and white wine";
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.searchTerms = "";
+      },
+    ],
+  },
+  // hasDomain
+  {
+    flag: "hasDomain",
+    subswitches: ["domain", "domainIsHost"],
+    desc: "nsINavHistoryQuery.hasDomain",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.domain = "mozilla.com";
+        aQuery.domainIsHost = false;
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.domain = "www.mozilla.com";
+        aQuery.domainIsHost = true;
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.domain = "";
+      },
+    ],
+  },
+  // hasUri
+  {
+    flag: "hasUri",
+    subswitches: ["uri"],
+    desc: "nsINavHistoryQuery.hasUri",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.uri = uri("http://mozilla.com");
+      },
+    ],
+  },
+  // hasAnnotation
+  {
+    flag: "hasAnnotation",
+    subswitches: ["annotation", "annotationIsNot"],
+    desc: "nsINavHistoryQuery.hasAnnotation",
+    matches: flagSwitchMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.annotation = "bookmarks/toolbarFolder";
+        aQuery.annotationIsNot = false;
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.annotation = "bookmarks/toolbarFolder";
+        aQuery.annotationIsNot = true;
+      },
+    ],
+  },
+  // minVisits
+  {
+    // property is used by function simplePropertyMatches.
+    property: "minVisits",
+    desc: "nsINavHistoryQuery.minVisits",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.minVisits = 0x7fffffff; // 2^31 - 1
+      },
+    ],
+  },
+  // maxVisits
+  {
+    property: "maxVisits",
+    desc: "nsINavHistoryQuery.maxVisits",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.maxVisits = 0x7fffffff; // 2^31 - 1
+      },
+    ],
+  },
+  // onlyBookmarked
+  {
+    property: "onlyBookmarked",
+    desc: "nsINavHistoryQuery.onlyBookmarked",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.onlyBookmarked = true;
+      },
+    ],
+  },
+  // getFolders
+  {
+    desc: "nsINavHistoryQuery.getParents",
+    matches(aQuery1, aQuery2) {
+      var q1Parents = aQuery1.getParents();
+      var q2Parents = aQuery2.getParents();
+      if (q1Parents.length !== q2Parents.length) {
+        return false;
+      }
+      for (let i = 0; i < q1Parents.length; i++) {
+        if (!q2Parents.includes(q1Parents[i])) {
+          return false;
+        }
+      }
+      for (let i = 0; i < q2Parents.length; i++) {
+        if (!q1Parents.includes(q2Parents[i])) {
+          return false;
+        }
+      }
+      return true;
+    },
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.setParents([]);
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.setParents([PlacesUtils.bookmarks.rootGuid]);
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.setParents([
+          PlacesUtils.bookmarks.rootGuid,
+          PlacesUtils.bookmarks.tagsGuid,
+        ]);
+      },
+    ],
+  },
+  // tags
+  {
+    desc: "nsINavHistoryQuery.getTags",
+    matches(aQuery1, aQuery2) {
+      if (aQuery1.tagsAreNot !== aQuery2.tagsAreNot) {
+        return false;
+      }
+      var q1Tags = aQuery1.tags;
+      var q2Tags = aQuery2.tags;
+      if (q1Tags.length !== q2Tags.length) {
+        return false;
+      }
+      for (let i = 0; i < q1Tags.length; i++) {
+        if (!q2Tags.includes(q1Tags[i])) {
+          return false;
+        }
+      }
+      for (let i = 0; i < q2Tags.length; i++) {
+        if (!q1Tags.includes(q2Tags[i])) {
+          return false;
+        }
+      }
+      return true;
+    },
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.tags = [];
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.tags = [""];
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.tags = [
+          "foo",
+          "七難",
+          "",
+          "いっぱいおっぱい",
+          "Abracadabra",
+          "１２３",
+          "Here's a pretty long tag name with some = signs and 1 2 3s and spaces oh jeez will it work I hope so!",
+          "アスキーでございません",
+          "あいうえお",
+        ];
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.tags = [
+          "foo",
+          "七難",
+          "",
+          "いっぱいおっぱい",
+          "Abracadabra",
+          "１２３",
+          "Here's a pretty long tag name with some = signs and 1 2 3s and spaces oh jeez will it work I hope so!",
+          "アスキーでございません",
+          "あいうえお",
+        ];
+        aQuery.tagsAreNot = true;
+      },
+    ],
+  },
+  // transitions
+  {
+    desc: "tests nsINavHistoryQuery.getTransitions",
+    matches(aQuery1, aQuery2) {
+      var q1Trans = aQuery1.getTransitions();
+      var q2Trans = aQuery2.getTransitions();
+      if (q1Trans.length !== q2Trans.length) {
+        return false;
+      }
+      for (let i = 0; i < q1Trans.length; i++) {
+        if (!q2Trans.includes(q1Trans[i])) {
+          return false;
+        }
+      }
+      for (let i = 0; i < q2Trans.length; i++) {
+        if (!q1Trans.includes(q2Trans[i])) {
+          return false;
+        }
+      }
+      return true;
+    },
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQuery.setTransitions([]);
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.setTransitions([Ci.nsINavHistoryService.TRANSITION_DOWNLOAD]);
+      },
+      function (aQuery, aQueryOptions) {
+        aQuery.setTransitions([
+          Ci.nsINavHistoryService.TRANSITION_TYPED,
+          Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+        ]);
+      },
+    ],
+  },
+];
+
+// nsINavHistoryQueryOptions switches
+const queryOptionSwitches = [
+  // sortingMode
+  {
+    desc: "nsINavHistoryQueryOptions.sortingMode",
+    matches(aOptions1, aOptions2) {
+      if (aOptions1.sortingMode === aOptions2.sortingMode) {
+        return true;
+      }
+      return false;
+    },
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.sortingMode = aQueryOptions.SORT_BY_DATE_ASCENDING;
+      },
+    ],
+  },
+  // resultType
+  {
+    // property is used by function simplePropertyMatches.
+    property: "resultType",
+    desc: "nsINavHistoryQueryOptions.resultType",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.resultType = aQueryOptions.RESULTS_AS_URI;
+      },
+    ],
+  },
+  // excludeItems
+  {
+    property: "excludeItems",
+    desc: "nsINavHistoryQueryOptions.excludeItems",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.excludeItems = true;
+      },
+    ],
+  },
+  // excludeQueries
+  {
+    property: "excludeQueries",
+    desc: "nsINavHistoryQueryOptions.excludeQueries",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.excludeQueries = true;
+      },
+    ],
+  },
+  // expandQueries
+  {
+    property: "expandQueries",
+    desc: "nsINavHistoryQueryOptions.expandQueries",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.expandQueries = true;
+      },
+    ],
+  },
+  // includeHidden
+  {
+    property: "includeHidden",
+    desc: "nsINavHistoryQueryOptions.includeHidden",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.includeHidden = true;
+      },
+    ],
+  },
+  // maxResults
+  {
+    property: "maxResults",
+    desc: "nsINavHistoryQueryOptions.maxResults",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.maxResults = 0xffffffff; // 2^32 - 1
+      },
+    ],
+  },
+  // queryType
+  {
+    property: "queryType",
+    desc: "nsINavHistoryQueryOptions.queryType",
+    matches: simplePropertyMatches,
+    runs: [
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.queryType = aQueryOptions.QUERY_TYPE_HISTORY;
+      },
+      function (aQuery, aQueryOptions) {
+        aQueryOptions.queryType = aQueryOptions.QUERY_TYPE_UNIFIED;
+      },
+    ],
+  },
+];
+
+/**
+ * Enumerates all the sequences of the cartesian product of the arrays contained
+ * in aSequences.  Examples:
+ *
+ *   cartProd([[1, 2, 3], ["a", "b"]], callback);
+ *   // callback is called 3 * 2 = 6 times with the following arrays:
+ *   // [1, "a"], [1, "b"], [2, "a"], [2, "b"], [3, "a"], [3, "b"]
+ *
+ *   cartProd([["a"], [1, 2, 3], ["X", "Y"]], callback);
+ *   // callback is called 1 * 3 * 2 = 6 times with the following arrays:
+ *   // ["a", 1, "X"], ["a", 1, "Y"], ["a", 2, "X"], ["a", 2, "Y"],
+ *   // ["a", 3, "X"], ["a", 3, "Y"]
+ *
+ *   cartProd([[1], [2], [3], [4]], callback);
+ *   // callback is called 1 * 1 * 1 * 1 = 1 time with the following array:
+ *   // [1, 2, 3, 4]
+ *
+ *   cartProd([], callback);
+ *   // callback is 0 times
+ *
+ *   cartProd([[1, 2, 3, 4]], callback);
+ *   // callback is called 4 times with the following arrays:
+ *   // [1], [2], [3], [4]
+ *
+ * @param  aSequences
+ *         an array that contains an arbitrary number of arrays
+ * @param  aCallback
+ *         a function that is passed each sequence of the product as it's
+ *         computed
+ * @return the total number of sequences in the product
+ */
+function cartProd(aSequences, aCallback) {
+  if (aSequences.length === 0) {
+    return 0;
+  }
+
+  // For each sequence in aSequences, we maintain a pointer (an array index,
+  // really) to the element we're currently enumerating in that sequence
+  var seqEltPtrs = aSequences.map(i => 0);
+
+  var numProds = 0;
+  var done = false;
+  while (!done) {
+    numProds++;
+
+    // prod = sequence in product we're currently enumerating
+    let prod = [];
+    for (let i = 0; i < aSequences.length; i++) {
+      prod.push(aSequences[i][seqEltPtrs[i]]);
+    }
+    aCallback(prod);
+
+    // The next sequence in the product differs from the current one by just a
+    // single element.  Determine which element that is.  We advance the
+    // "rightmost" element pointer to the "right" by one.  If we move past the
+    // end of that pointer's sequence, reset the pointer to the first element
+    // in its sequence and then try the sequence to the "left", and so on.
+
+    // seqPtr = index of rightmost input sequence whose element pointer is not
+    // past the end of the sequence
+    let seqPtr = aSequences.length - 1;
+    while (!done) {
+      // Advance the rightmost element pointer.
+      seqEltPtrs[seqPtr]++;
+
+      // The rightmost element pointer is past the end of its sequence.
+      if (seqEltPtrs[seqPtr] >= aSequences[seqPtr].length) {
+        seqEltPtrs[seqPtr] = 0;
+        seqPtr--;
+
+        // All element pointers are past the ends of their sequences.
+        if (seqPtr < 0) {
+          done = true;
+        }
+      } else {
+        break;
+      }
+    }
+  }
+  return numProds;
+}
+
+/**
+ * Enumerates all the subsets in aSet of size aHowMany.  There are
+ * C(aSet.length, aHowMany) such subsets.  aCallback will be passed each subset
+ * as it is generated.  Note that aSet and the subsets enumerated are -- even
+ * though they're arrays -- not sequences; the ordering of their elements is not
+ * important.  Example:
+ *
+ *   choose([1, 2, 3, 4], 2, callback);
+ *   // callback is called C(4, 2) = 6 times with the following sets (arrays):
+ *   // [1, 2], [1, 3], [1, 4], [2, 3], [2, 4], [3, 4]
+ *
+ * @param  aSet
+ *         an array from which to choose elements, aSet.length > 0
+ * @param  aHowMany
+ *         the number of elements to choose, > 0 and <= aSet.length
+ * @return the total number of sets chosen
+ */
+function choose(aSet, aHowMany, aCallback) {
+  // ptrs = indices of the elements in aSet we're currently choosing
+  var ptrs = [];
+  for (let i = 0; i < aHowMany; i++) {
+    ptrs.push(i);
+  }
+
+  var numFound = 0;
+  var done = false;
+  while (!done) {
+    numFound++;
+    aCallback(ptrs.map(p => aSet[p]));
+
+    // The next subset to be chosen differs from the current one by just a
+    // single element.  Determine which element that is.  Advance the "rightmost"
+    // pointer to the "right" by one.  If we move past the end of set, move the
+    // next non-adjacent rightmost pointer to the right by one, and reset all
+    // succeeding pointers so that they're adjacent to it.  When all pointers
+    // are clustered all the way to the right, we're done.
+
+    // Advance the rightmost pointer.
+    ptrs[ptrs.length - 1]++;
+
+    // The rightmost pointer has gone past the end of set.
+    if (ptrs[ptrs.length - 1] >= aSet.length) {
+      // Find the next rightmost pointer that is not adjacent to the current one.
+      let si = aSet.length - 2; // aSet index
+      let pi = ptrs.length - 2; // ptrs index
+      while (pi >= 0 && ptrs[pi] === si) {
+        pi--;
+        si--;
+      }
+
+      // All pointers are adjacent and clustered all the way to the right.
+      if (pi < 0) {
+        done = true;
+      } else {
+        // pi = index of rightmost pointer with a gap between it and its
+        // succeeding pointer.  Move it right and reset all succeeding pointers
+        // so that they're adjacent to it.
+        ptrs[pi]++;
+        for (let i = 0; i < ptrs.length - pi - 1; i++) {
+          ptrs[i + pi + 1] = ptrs[pi] + i + 1;
+        }
+      }
+    }
+  }
+  return numFound;
+}
+
+/**
+ * Convenience function for nsINavHistoryQuery switches that act as flags.  This
+ * is attached to switch objects.  See querySwitches array above.
+ *
+ * @param  aQuery1
+ *         an nsINavHistoryQuery object
+ * @param  aQuery2
+ *         another nsINavHistoryQuery object
+ * @return true if this switch is the same in both aQuery1 and aQuery2
+ */
+function flagSwitchMatches(aQuery1, aQuery2) {
+  if (aQuery1[this.flag] && aQuery2[this.flag]) {
+    for (let p in this.subswitches) {
+      if (p in aQuery1 && p in aQuery2) {
+        if (aQuery1[p] instanceof Ci.nsIURI) {
+          if (!aQuery1[p].equals(aQuery2[p])) {
+            return false;
+          }
+        } else if (aQuery1[p] !== aQuery2[p]) {
+          return false;
+        }
+      }
+    }
+  } else if (aQuery1[this.flag] || aQuery2[this.flag]) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Tests if aObj1 and aObj2 are equal.  This function is general and may be used
+ * for either nsINavHistoryQuery or nsINavHistoryQueryOptions objects.  aSwitches
+ * determines which set of switches is used for comparison.  Pass in either
+ * querySwitches or queryOptionSwitches.
+ *
+ * @param  aSwitches
+ *         determines which set of switches applies to aObj1 and aObj2, either
+ *         querySwitches or queryOptionSwitches
+ * @param  aObj1
+ *         an nsINavHistoryQuery or nsINavHistoryQueryOptions object
+ * @param  aObj2
+ *         another nsINavHistoryQuery or nsINavHistoryQueryOptions object
+ * @return true if aObj1 and aObj2 are equal
+ */
+function queryObjsEqual(aSwitches, aObj1, aObj2) {
+  for (let i = 0; i < aSwitches.length; i++) {
+    if (!aSwitches[i].matches(aObj1, aObj2)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * This drives the test runs.  See the comment at the top of this file.
+ *
+ * @param aHowManyLo
+ *        the size of the switch subsets to start with
+ * @param aHowManyHi
+ *        the size of the switch subsets to end with (inclusive)
+ */
+function runQuerySequences(aHowManyLo, aHowManyHi) {
+  var allSwitches = querySwitches.concat(queryOptionSwitches);
+
+  // Choose aHowManyLo switches up to aHowManyHi switches.
+  for (let howMany = aHowManyLo; howMany <= aHowManyHi; howMany++) {
+    let numIters = 0;
+    print("CHOOSING " + howMany + " SWITCHES");
+
+    // Choose all subsets of size howMany from allSwitches.
+    choose(allSwitches, howMany, function (chosenSwitches) {
+      print(numIters);
+      numIters++;
+
+      // Collect the runs.
+      // runs = [ [runs from switch 1], ..., [runs from switch howMany] ]
+      var runs = chosenSwitches.map(function (s) {
+        if (s.desc) {
+          print("  " + s.desc);
+        }
+        return s.runs;
+      });
+
+      // cartProd(runs) => [
+      //   [switch 1 run 1, switch 2 run 1, ..., switch howMany run 1 ],
+      //   ...,
+      //   [switch 1 run 1, switch 2 run 1, ..., switch howMany run N ],
+      //   ..., ...,
+      //   [switch 1 run N, switch 2 run N, ..., switch howMany run 1 ],
+      //   ...,
+      //   [switch 1 run N, switch 2 run N, ..., switch howMany run N ],
+      // ]
+      cartProd(runs, function (runSet) {
+        // Create a new query, apply the switches in runSet, and test it.
+        var query = PlacesUtils.history.getNewQuery();
+        var opts = PlacesUtils.history.getNewQueryOptions();
+        for (let i = 0; i < runSet.length; i++) {
+          runSet[i](query, opts);
+        }
+        serializeDeserialize(query, opts);
+      });
+    });
+  }
+  print("\n");
+}
+
+/**
+ * Serializes the nsINavHistoryQuery objects in aQuery and the
+ * nsINavHistoryQueryOptions object aQueryOptions, de-serializes the
+ * serialization, and ensures (using do_check_* functions) that the
+ * de-serialized objects equal the originals.
+ *
+ * @param aQuery
+ *        an nsINavHistoryQuery object
+ * @param aQueryOptions
+ *        an nsINavHistoryQueryOptions object
+ */
+function serializeDeserialize(aQuery, aQueryOptions) {
+  let queryStr = PlacesUtils.history.queryToQueryString(aQuery, aQueryOptions);
+  print("  " + queryStr);
+  let query2 = {},
+    opts2 = {};
+  PlacesUtils.history.queryStringToQuery(queryStr, query2, opts2);
+  query2 = query2.value;
+  opts2 = opts2.value;
+
+  Assert.ok(queryObjsEqual(querySwitches, aQuery, query2));
+
+  // Finally check the query options objects.
+  Assert.ok(queryObjsEqual(queryOptionSwitches, aQueryOptions, opts2));
+}
+
+/**
+ * Convenience function for switches that have simple values.  This is attached
+ * to switch objects.  See querySwitches and queryOptionSwitches arrays above.
+ *
+ * @param  aObj1
+ *         an nsINavHistoryQuery or nsINavHistoryQueryOptions object
+ * @param  aObj2
+ *         another nsINavHistoryQuery or nsINavHistoryQueryOptions object
+ * @return true if this switch is the same in both aObj1 and aObj2
+ */
+function simplePropertyMatches(aObj1, aObj2) {
+  return aObj1[this.property] === aObj2[this.property];
+}
+
+function run_test() {
+  runQuerySequences(CHOOSE_HOW_MANY_SWITCHES_LO, CHOOSE_HOW_MANY_SWITCHES_HI);
+}
diff --git a/toolkit/components/places/tests/queries/test_query_uri_liveupdate.js b/toolkit/components/places/tests/queries/test_query_uri_liveupdate.js
new file mode 100644
index 0000000000..5ada4a84d4
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_query_uri_liveupdate.js
@@ -0,0 +1,45 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_results_as_tag_query() {
+  let bms = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      { url: "http://tag1.moz.com/", tags: ["tag1"] },
+      { url: "http://tag2.moz.com/", tags: ["tag2"] },
+      { url: "place:tag=tag1" },
+    ],
+  });
+
+  let root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid,
+    false,
+    true
+  ).root;
+  Assert.equal(root.childCount, 3, "We should get 3 results");
+  let queryRoot = root.getChild(2);
+  PlacesUtils.asContainer(queryRoot).containerOpen = true;
+
+  Assert.equal(queryRoot.uri, "place:tag=tag1", "Found the query");
+  Assert.equal(queryRoot.childCount, 1, "We should get 1 result");
+  Assert.equal(
+    queryRoot.getChild(0).uri,
+    "http://tag1.moz.com/",
+    "Found the tagged bookmark"
+  );
+
+  await PlacesUtils.bookmarks.update({
+    guid: bms[2].guid,
+    url: "place:tag=tag2",
+  });
+  Assert.equal(queryRoot.uri, "place:tag=tag2", "Found the query");
+  Assert.equal(queryRoot.childCount, 1, "We should get 1 result");
+  Assert.equal(
+    queryRoot.getChild(0).uri,
+    "http://tag2.moz.com/",
+    "Found the tagged bookmark"
+  );
+
+  queryRoot.containerOpen = false;
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_redirects.js b/toolkit/components/places/tests/queries/test_redirects.js
new file mode 100644
index 0000000000..ee23f949a8
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_redirects.js
@@ -0,0 +1,351 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Array of visits we will add to the database, will be populated later
+// in the test.
+var visits = [];
+
+/**
+ * Takes a sequence of query options, and compare query results obtained through
+ * them with a custom filtered array of visits, based on the values we are
+ * expecting from the query.
+ *
+ * @param  aSequence
+ *         an array that contains query options in the form:
+ *         [includeHidden, maxResults, sortingMode]
+ */
+function check_results_callback(aSequence) {
+  // Sanity check: we should receive 3 parameters.
+  Assert.equal(aSequence.length, 3);
+  let includeHidden = aSequence[0];
+  let maxResults = aSequence[1];
+  let sortingMode = aSequence[2];
+  print(
+    "\nTESTING: includeHidden(" +
+      includeHidden +
+      ")," +
+      " maxResults(" +
+      maxResults +
+      ")," +
+      " sortingMode(" +
+      sortingMode +
+      ")."
+  );
+
+  function isHidden(aVisit) {
+    return (
+      aVisit.transType == Ci.nsINavHistoryService.TRANSITION_FRAMED_LINK ||
+      aVisit.isRedirect
+    );
+  }
+
+  // Build expectedData array.
+  let expectedData = visits.filter(function (aVisit, aIndex, aArray) {
+    // Embed visits never appear in results.
+    if (aVisit.transType == Ci.nsINavHistoryService.TRANSITION_EMBED) {
+      return false;
+    }
+
+    if (!includeHidden && isHidden(aVisit)) {
+      // If the page has any non-hidden visit, then it's visible.
+      if (
+        !visits.filter(function (refVisit) {
+          return refVisit.uri == aVisit.uri && !isHidden(refVisit);
+        }).length
+      ) {
+        return false;
+      }
+    }
+
+    return true;
+  });
+
+  // Remove duplicates, since queries are RESULTS_AS_URI (unique pages).
+  let seen = [];
+  expectedData = expectedData.filter(function (aData) {
+    if (seen.includes(aData.uri)) {
+      return false;
+    }
+    seen.push(aData.uri);
+    return true;
+  });
+
+  // Sort expectedData.
+  function getFirstIndexFor(aEntry) {
+    for (let i = 0; i < visits.length; i++) {
+      if (visits[i].uri == aEntry.uri) {
+        return i;
+      }
+    }
+    return undefined;
+  }
+  function comparator(a, b) {
+    if (sortingMode == Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING) {
+      return b.lastVisit - a.lastVisit;
+    }
+    if (
+      sortingMode == Ci.nsINavHistoryQueryOptions.SORT_BY_VISITCOUNT_DESCENDING
+    ) {
+      return b.visitCount - a.visitCount;
+    }
+    return getFirstIndexFor(a) - getFirstIndexFor(b);
+  }
+  expectedData.sort(comparator);
+
+  // Crop results to maxResults if it's defined.
+  if (maxResults) {
+    expectedData = expectedData.slice(0, maxResults);
+  }
+
+  // Create a new query with required options.
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.includeHidden = includeHidden;
+  options.sortingMode = sortingMode;
+  if (maxResults) {
+    options.maxResults = maxResults;
+  }
+
+  // Compare resultset with expectedData.
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let root = result.root;
+  root.containerOpen = true;
+  compareArrayToResult(expectedData, root);
+  root.containerOpen = false;
+}
+
+/**
+ * Enumerates all the sequences of the cartesian product of the arrays contained
+ * in aSequences.  Examples:
+ *
+ *   cartProd([[1, 2, 3], ["a", "b"]], callback);
+ *   // callback is called 3 * 2 = 6 times with the following arrays:
+ *   // [1, "a"], [1, "b"], [2, "a"], [2, "b"], [3, "a"], [3, "b"]
+ *
+ *   cartProd([["a"], [1, 2, 3], ["X", "Y"]], callback);
+ *   // callback is called 1 * 3 * 2 = 6 times with the following arrays:
+ *   // ["a", 1, "X"], ["a", 1, "Y"], ["a", 2, "X"], ["a", 2, "Y"],
+ *   // ["a", 3, "X"], ["a", 3, "Y"]
+ *
+ *   cartProd([[1], [2], [3], [4]], callback);
+ *   // callback is called 1 * 1 * 1 * 1 = 1 time with the following array:
+ *   // [1, 2, 3, 4]
+ *
+ *   cartProd([], callback);
+ *   // callback is 0 times
+ *
+ *   cartProd([[1, 2, 3, 4]], callback);
+ *   // callback is called 4 times with the following arrays:
+ *   // [1], [2], [3], [4]
+ *
+ * @param  aSequences
+ *         an array that contains an arbitrary number of arrays
+ * @param  aCallback
+ *         a function that is passed each sequence of the product as it's
+ *         computed
+ * @return the total number of sequences in the product
+ */
+function cartProd(aSequences, aCallback) {
+  if (aSequences.length === 0) {
+    return 0;
+  }
+
+  // For each sequence in aSequences, we maintain a pointer (an array index,
+  // really) to the element we're currently enumerating in that sequence
+  let seqEltPtrs = aSequences.map(i => 0);
+
+  let numProds = 0;
+  let done = false;
+  while (!done) {
+    numProds++;
+
+    // prod = sequence in product we're currently enumerating
+    let prod = [];
+    for (let i = 0; i < aSequences.length; i++) {
+      prod.push(aSequences[i][seqEltPtrs[i]]);
+    }
+    aCallback(prod);
+
+    // The next sequence in the product differs from the current one by just a
+    // single element.  Determine which element that is.  We advance the
+    // "rightmost" element pointer to the "right" by one.  If we move past the
+    // end of that pointer's sequence, reset the pointer to the first element
+    // in its sequence and then try the sequence to the "left", and so on.
+
+    // seqPtr = index of rightmost input sequence whose element pointer is not
+    // past the end of the sequence
+    let seqPtr = aSequences.length - 1;
+    while (!done) {
+      // Advance the rightmost element pointer.
+      seqEltPtrs[seqPtr]++;
+
+      // The rightmost element pointer is past the end of its sequence.
+      if (seqEltPtrs[seqPtr] >= aSequences[seqPtr].length) {
+        seqEltPtrs[seqPtr] = 0;
+        seqPtr--;
+
+        // All element pointers are past the ends of their sequences.
+        if (seqPtr < 0) {
+          done = true;
+        }
+      } else {
+        break;
+      }
+    }
+  }
+  return numProds;
+}
+
+/**
+ * Populate the visits array and add visits to the database.
+ * We will generate visit-chains like:
+ *   visit -> redirect_temp -> redirect_perm
+ */
+add_task(async function test_add_visits_to_database() {
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  // We don't really bother on this, but we need a time to add visits.
+  let timeInMicroseconds = Date.now() * 1000;
+  let visitCount = 1;
+
+  // Array of all possible transition types we could be redirected from.
+  let t = [
+    Ci.nsINavHistoryService.TRANSITION_LINK,
+    Ci.nsINavHistoryService.TRANSITION_TYPED,
+    Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+    // Embed visits are not added to the database and we don't want redirects
+    // to them, thus just avoid addition.
+    // Ci.nsINavHistoryService.TRANSITION_EMBED,
+    Ci.nsINavHistoryService.TRANSITION_FRAMED_LINK,
+    // Would make hard sorting by visit date because last_visit_date is actually
+    // calculated excluding download transitions, but the query includes
+    // downloads.
+    // TODO: Bug 488966 could fix this behavior.
+    // Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  ];
+
+  function newTimeInMicroseconds() {
+    timeInMicroseconds = timeInMicroseconds - 1000;
+    return timeInMicroseconds;
+  }
+
+  // we add a visit for each of the above transition types.
+  t.forEach(transition =>
+    visits.push({
+      isVisit: true,
+      transType: transition,
+      uri: "http://" + transition + ".example.com/",
+      title: transition + "-example",
+      isRedirect: true,
+      lastVisit: newTimeInMicroseconds(),
+      visitCount:
+        transition == Ci.nsINavHistoryService.TRANSITION_EMBED ||
+        transition == Ci.nsINavHistoryService.TRANSITION_FRAMED_LINK
+          ? 0
+          : visitCount++,
+      isInQuery: true,
+    })
+  );
+
+  // Add a REDIRECT_TEMPORARY layer of visits for each of the above visits.
+  t.forEach(transition =>
+    visits.push({
+      isVisit: true,
+      transType: Ci.nsINavHistoryService.TRANSITION_REDIRECT_TEMPORARY,
+      uri: "http://" + transition + ".redirect.temp.example.com/",
+      title: transition + "-redirect-temp-example",
+      lastVisit: newTimeInMicroseconds(),
+      isRedirect: true,
+      referrer: "http://" + transition + ".example.com/",
+      visitCount: visitCount++,
+      isInQuery: true,
+    })
+  );
+
+  // Add a REDIRECT_PERMANENT layer of visits for each of the above redirects.
+  t.forEach(transition =>
+    visits.push({
+      isVisit: true,
+      transType: Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT,
+      uri: "http://" + transition + ".redirect.perm.example.com/",
+      title: transition + "-redirect-perm-example",
+      lastVisit: newTimeInMicroseconds(),
+      isRedirect: true,
+      referrer: "http://" + transition + ".redirect.temp.example.com/",
+      visitCount: visitCount++,
+      isInQuery: true,
+    })
+  );
+
+  // Add a REDIRECT_PERMANENT layer of visits that loop to the first visit.
+  // These entries should not change visitCount or lastVisit, otherwise
+  // guessing an order would be a nightmare.
+  function getLastValue(aURI, aProperty) {
+    for (let i = 0; i < visits.length; i++) {
+      if (visits[i].uri == aURI) {
+        return visits[i][aProperty];
+      }
+    }
+    do_throw("Unknown uri.");
+    return null;
+  }
+  t.forEach(transition =>
+    visits.push({
+      isVisit: true,
+      transType: Ci.nsINavHistoryService.TRANSITION_REDIRECT_PERMANENT,
+      uri: "http://" + transition + ".example.com/",
+      title: getLastValue("http://" + transition + ".example.com/", "title"),
+      lastVisit: getLastValue(
+        "http://" + transition + ".example.com/",
+        "lastVisit"
+      ),
+      isRedirect: true,
+      referrer: "http://" + transition + ".redirect.perm.example.com/",
+      visitCount: getLastValue(
+        "http://" + transition + ".example.com/",
+        "visitCount"
+      ),
+      isInQuery: true,
+    })
+  );
+
+  // Add an unvisited bookmark in the database, it should never appear.
+  visits.push({
+    isBookmark: true,
+    uri: "http://unvisited.bookmark.com/",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    title: "Unvisited Bookmark",
+    isInQuery: false,
+  });
+
+  // Put visits in the database.
+  await task_populateDB(visits);
+});
+
+add_task(async function test_redirects() {
+  // Frecency and hidden are updated asynchronously, wait for them.
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  // This array will be used by cartProd to generate a matrix of all possible
+  // combinations.
+  let includeHidden_options = [true, false];
+  let maxResults_options = [5, 10, 20, null];
+  // These sortingMode are choosen to toggle using special queries for history
+  // menu.
+  let sorting_options = [
+    Ci.nsINavHistoryQueryOptions.SORT_BY_NONE,
+    Ci.nsINavHistoryQueryOptions.SORT_BY_VISITCOUNT_DESCENDING,
+    Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING,
+  ];
+  // Will execute check_results_callback() for each generated combination.
+  cartProd(
+    [includeHidden_options, maxResults_options, sorting_options],
+    check_results_callback
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/queries/test_result_observeHistoryDetails.js b/toolkit/components/places/tests/queries/test_result_observeHistoryDetails.js
new file mode 100644
index 0000000000..83531ee2c4
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_result_observeHistoryDetails.js
@@ -0,0 +1,155 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures that skipHistoryDetailsNotifications works as expected.
+
+function accumulateNotifications(
+  result,
+  skipHistoryDetailsNotifications = false
+) {
+  let notifications = [];
+  let resultObserver = new Proxy(NavHistoryResultObserver, {
+    get(target, name) {
+      if (name == "check") {
+        result.removeObserver(resultObserver, false);
+        return expectedNotifications =>
+          Assert.deepEqual(notifications, expectedNotifications);
+      }
+      if (name == "skipHistoryDetailsNotifications") {
+        return skipHistoryDetailsNotifications;
+      }
+      // ignore a few uninteresting notifications.
+      if (["QueryInterface", "containerStateChanged"].includes(name)) {
+        return () => {};
+      }
+      return () => {
+        notifications.push(name);
+      };
+    },
+  });
+  result.addObserver(resultObserver, false);
+  return resultObserver;
+}
+
+add_task(async function test_history_query_observe() {
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let notifications = accumulateNotifications(result);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla.org",
+    title: "test",
+  });
+  notifications.check([
+    "nodeHistoryDetailsChanged",
+    "nodeInserted",
+    "nodeTitleChanged",
+  ]);
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_history_query_no_observe() {
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let result = PlacesUtils.history.executeQuery(query, options);
+  // Even if we opt-out of notifications, this is an history query, thus the
+  // setting is pretty much ignored.
+  let notifications = accumulateNotifications(result, true);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla.org",
+    title: "test",
+  });
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla2.org",
+    title: "test",
+  });
+
+  notifications.check([
+    "nodeHistoryDetailsChanged",
+    "nodeInserted",
+    "nodeTitleChanged",
+    "nodeHistoryDetailsChanged",
+    "nodeInserted",
+    "nodeTitleChanged",
+  ]);
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_bookmarks_query_observe() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let notifications = accumulateNotifications(result);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+
+  await PlacesUtils.bookmarks.insert({
+    url: "http://mozilla.org",
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "test",
+  });
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla.org",
+    title: "title",
+  });
+
+  notifications.check([
+    "nodeHistoryDetailsChanged",
+    "nodeInserted",
+    "nodeHistoryDetailsChanged",
+  ]);
+
+  root.containerOpen = false;
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_bookmarks_query_no_observe() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let notifications = accumulateNotifications(result, true);
+  let root = PlacesUtils.asContainer(result.root);
+  root.containerOpen = true;
+
+  await PlacesUtils.bookmarks.insert({
+    url: "http://mozilla.org",
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "test",
+  });
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla.org",
+    title: "title",
+  });
+
+  notifications.check(["nodeInserted"]);
+
+  info("Change the sorting mode to one that is based on history");
+  notifications = accumulateNotifications(result, true);
+  result.sortingMode = options.SORT_BY_VISITCOUNT_DESCENDING;
+  notifications.check(["invalidateContainer"]);
+
+  notifications = accumulateNotifications(result, true);
+  await PlacesTestUtils.addVisits({
+    uri: "http://mozilla.org",
+    title: "title",
+  });
+  notifications.check(["nodeHistoryDetailsChanged"]);
+
+  root.containerOpen = false;
+  await PlacesUtils.bookmarks.eraseEverything();
+});
diff --git a/toolkit/components/places/tests/queries/test_results-as-left-pane.js b/toolkit/components/places/tests/queries/test_results-as-left-pane.js
new file mode 100644
index 0000000000..6cec733758
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_results-as-left-pane.js
@@ -0,0 +1,83 @@
+"use strict";
+
+const expectedRoots = [
+  {
+    title: "OrganizerQueryHistory",
+    uri: `place:sort=${Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING}&type=${Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY}`,
+    guid: "history____v",
+  },
+  {
+    title: "OrganizerQueryDownloads",
+    uri: `place:transition=${Ci.nsINavHistoryService.TRANSITION_DOWNLOAD}&sort=${Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING}`,
+    guid: "downloads__v",
+  },
+  {
+    title: "TagsFolderTitle",
+    uri: `place:sort=${Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING}&type=${Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAGS_ROOT}`,
+    guid: "tags_______v",
+  },
+  {
+    title: "OrganizerQueryAllBookmarks",
+    uri: `place:type=${Ci.nsINavHistoryQueryOptions.RESULTS_AS_ROOTS_QUERY}`,
+    guid: "allbms_____v",
+  },
+];
+
+const placesStrings = Services.strings.createBundle(
+  "chrome://places/locale/places.properties"
+);
+
+function getLeftPaneQuery() {
+  var query = PlacesUtils.history.getNewQuery();
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_LEFT_PANE_QUERY;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  return result.root;
+}
+
+function assertExpectedChildren(root, expectedChildren) {
+  Assert.equal(
+    root.childCount,
+    expectedChildren.length,
+    "Should have the expected number of children."
+  );
+
+  for (let i = 0; i < root.childCount; i++) {
+    Assert.ok(
+      PlacesTestUtils.ComparePlacesURIs(
+        root.getChild(i).uri,
+        expectedChildren[i].uri
+      ),
+      "Should have the correct uri for root ${i}"
+    );
+    Assert.equal(
+      root.getChild(i).title,
+      placesStrings.GetStringFromName(expectedChildren[i].title),
+      "Should have the correct title for root ${i}"
+    );
+    Assert.equal(root.getChild(i).bookmarkGuid, expectedChildren[i].guid);
+  }
+}
+
+/**
+ * This test will test the basic RESULTS_AS_ROOTS_QUERY, that simply returns,
+ * the existing bookmark roots.
+ */
+add_task(async function test_results_as_root() {
+  let root = getLeftPaneQuery();
+  root.containerOpen = true;
+
+  Assert.equal(
+    PlacesUtils.asQuery(root).queryOptions.queryType,
+    Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS,
+    "Should have a query type of QUERY_TYPE_BOOKMARKS"
+  );
+
+  assertExpectedChildren(root, expectedRoots);
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_results-as-roots.js b/toolkit/components/places/tests/queries/test_results-as-roots.js
new file mode 100644
index 0000000000..2f082d3e0b
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_results-as-roots.js
@@ -0,0 +1,114 @@
+"use strict";
+
+const MOBILE_BOOKMARKS_PREF = "browser.bookmarks.showMobileBookmarks";
+
+const expectedRoots = [
+  {
+    title: "BookmarksToolbarFolderTitle",
+    uri: `place:parent=${PlacesUtils.bookmarks.toolbarGuid}`,
+    guid: PlacesUtils.bookmarks.virtualToolbarGuid,
+  },
+  {
+    title: "BookmarksMenuFolderTitle",
+    uri: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
+    guid: PlacesUtils.bookmarks.virtualMenuGuid,
+  },
+  {
+    title: "OtherBookmarksFolderTitle",
+    uri: `place:parent=${PlacesUtils.bookmarks.unfiledGuid}`,
+    guid: PlacesUtils.bookmarks.virtualUnfiledGuid,
+  },
+];
+
+const expectedRootsWithMobile = [
+  ...expectedRoots,
+  {
+    title: "MobileBookmarksFolderTitle",
+    uri: `place:parent=${PlacesUtils.bookmarks.mobileGuid}`,
+    guid: PlacesUtils.bookmarks.virtualMobileGuid,
+  },
+];
+
+const placesStrings = Services.strings.createBundle(
+  "chrome://places/locale/places.properties"
+);
+
+function getAllBookmarksQuery() {
+  var query = PlacesUtils.history.getNewQuery();
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_VISITCOUNT_ASCENDING;
+  options.resultType = options.RESULTS_AS_ROOTS_QUERY;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  return result.root;
+}
+
+function assertExpectedChildren(root, expectedChildren) {
+  Assert.equal(
+    root.childCount,
+    expectedChildren.length,
+    "Should have the expected number of children."
+  );
+
+  for (let i = 0; i < root.childCount; i++) {
+    Assert.equal(
+      root.getChild(i).uri,
+      expectedChildren[i].uri,
+      "Should have the correct uri for root ${i}"
+    );
+    Assert.equal(
+      root.getChild(i).title,
+      placesStrings.GetStringFromName(expectedChildren[i].title),
+      "Should have the correct title for root ${i}"
+    );
+    Assert.equal(root.getChild(i).bookmarkGuid, expectedChildren[i].guid);
+  }
+}
+
+/**
+ * This test will test the basic RESULTS_AS_ROOTS_QUERY, that simply returns,
+ * the existing bookmark roots.
+ */
+add_task(async function test_results_as_root() {
+  let root = getAllBookmarksQuery();
+  root.containerOpen = true;
+
+  Assert.equal(
+    PlacesUtils.asQuery(root).queryOptions.queryType,
+    Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS,
+    "Should have a query type of QUERY_TYPE_BOOKMARKS"
+  );
+
+  assertExpectedChildren(root, expectedRoots);
+
+  root.containerOpen = false;
+});
+
+add_task(async function test_results_as_root_with_mobile() {
+  Services.prefs.setBoolPref(MOBILE_BOOKMARKS_PREF, true);
+
+  let root = getAllBookmarksQuery();
+  root.containerOpen = true;
+
+  assertExpectedChildren(root, expectedRootsWithMobile);
+
+  root.containerOpen = false;
+  Services.prefs.clearUserPref(MOBILE_BOOKMARKS_PREF);
+});
+
+add_task(async function test_results_as_root_remove_mobile_dynamic() {
+  Services.prefs.setBoolPref(MOBILE_BOOKMARKS_PREF, true);
+
+  let root = getAllBookmarksQuery();
+  root.containerOpen = true;
+
+  // Now un-set the pref, and poke the database to update the query.
+  Services.prefs.clearUserPref(MOBILE_BOOKMARKS_PREF);
+
+  assertExpectedChildren(root, expectedRoots);
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_results-as-tag-query.js b/toolkit/components/places/tests/queries/test_results-as-tag-query.js
new file mode 100644
index 0000000000..0d4670b658
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_results-as-tag-query.js
@@ -0,0 +1,63 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const testData = {
+  "http://foo.com/": ["tag1", "tag 2", "Space ☺️ Between"].sort(),
+  "http://bar.com/": ["tag1", "tag 2"].sort(),
+  "http://baz.com/": ["tag 2", "Space ☺️ Between"].sort(),
+  "http://qux.com/": ["Space ☺️ Between"],
+};
+
+const formattedTestData = [];
+for (const [uri, tagArray] of Object.entries(testData)) {
+  formattedTestData.push({
+    title: `Title of ${uri}`,
+    uri,
+    isBookmark: true,
+    isTag: true,
+    tagArray,
+  });
+}
+
+add_task(async function test_results_as_tags_root() {
+  await task_populateDB(formattedTestData);
+
+  // Construct URL - tag mapping from tag query.
+  const actualData = {};
+  for (const uri in testData) {
+    if (testData.hasOwnProperty(uri)) {
+      actualData[uri] = [];
+    }
+  }
+
+  const options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_TAGS_ROOT;
+  const query = PlacesUtils.history.getNewQuery();
+  const root = PlacesUtils.history.executeQuery(query, options).root;
+
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 3, "We should get as many results as tags.");
+  displayResultSet(root);
+
+  for (let i = 0; i < root.childCount; ++i) {
+    const node = root.getChild(i);
+    const tagName = node.title;
+    Assert.equal(
+      node.type,
+      node.RESULT_TYPE_QUERY,
+      "Result type should be RESULT_TYPE_QUERY."
+    );
+    const subRoot = node.QueryInterface(Ci.nsINavHistoryContainerResultNode);
+    subRoot.containerOpen = true;
+    for (let j = 0; j < subRoot.childCount; ++j) {
+      actualData[subRoot.getChild(j).uri].push(tagName);
+      actualData[subRoot.getChild(j).uri].sort();
+    }
+  }
+
+  Assert.deepEqual(
+    actualData,
+    testData,
+    "URI-tag mapping should be same from query and initial data."
+  );
+});
diff --git a/toolkit/components/places/tests/queries/test_results-as-visit.js b/toolkit/components/places/tests/queries/test_results-as-visit.js
new file mode 100644
index 0000000000..b10fb00bab
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_results-as-visit.js
@@ -0,0 +1,158 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+var testData = [];
+var timeInMicroseconds = PlacesUtils.toPRTime(Date.now() - 10000);
+
+function newTimeInMicroseconds() {
+  timeInMicroseconds = timeInMicroseconds + 1000;
+  return timeInMicroseconds;
+}
+
+function createTestData() {
+  function generateVisits(aPage) {
+    for (var i = 0; i < aPage.visitCount; i++) {
+      testData.push({
+        isInQuery: aPage.inQuery,
+        isVisit: true,
+        title: aPage.title,
+        uri: aPage.uri,
+        lastVisit: newTimeInMicroseconds(),
+        isTag: aPage.tags && !!aPage.tags.length,
+        tagArray: aPage.tags,
+      });
+    }
+  }
+
+  var pages = [
+    {
+      uri: "http://foo.com/",
+      title: "amo",
+      tags: ["moz"],
+      visitCount: 3,
+      inQuery: true,
+    },
+    {
+      uri: "http://moilla.com/",
+      title: "bMoz",
+      tags: ["bugzilla"],
+      visitCount: 5,
+      inQuery: true,
+    },
+    {
+      uri: "http://foo.mail.com/changeme1.html",
+      title: "c Moz",
+      visitCount: 7,
+      inQuery: true,
+    },
+    {
+      uri: "http://foo.mail.com/changeme2.html",
+      tags: ["moz"],
+      title: "",
+      visitCount: 1,
+      inQuery: false,
+    },
+    {
+      uri: "http://foo.mail.com/changeme3.html",
+      title: "zydeco",
+      visitCount: 5,
+      inQuery: false,
+    },
+  ];
+  pages.forEach(generateVisits);
+}
+
+/**
+ * This test will test Queries that use relative search terms and URI options
+ */
+add_task(async function test_results_as_visit() {
+  createTestData();
+  await task_populateDB(testData);
+  var query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "moz";
+  query.minVisits = 2;
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_VISITCOUNT_ASCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  info("Number of items in result set: " + root.childCount);
+  for (let i = 0; i < root.childCount; ++i) {
+    info(
+      "result: " + root.getChild(i).uri + " Title: " + root.getChild(i).title
+    );
+  }
+
+  // Check our inital result set
+  compareArrayToResult(testData, root);
+
+  // If that passes, check liveupdate
+  // Add to the query set
+  info("Adding item to query");
+  var tmp = [];
+  for (let i = 0; i < 2; i++) {
+    tmp.push({
+      isVisit: true,
+      uri: "http://foo.com/added.html",
+      title: "ab moz",
+    });
+  }
+  await task_populateDB(tmp);
+  for (let i = 0; i < 2; i++) {
+    Assert.equal(root.getChild(i).title, "ab moz");
+  }
+
+  // Update an existing URI
+  info("Updating Item");
+  var change2 = [
+    { isVisit: true, title: "moz", uri: "http://foo.mail.com/changeme2.html" },
+  ];
+  await task_populateDB(change2);
+  Assert.ok(isInResult(change2, root));
+
+  // Update some visits - add one and take one out of query set, and simply
+  // change one so that it still applies to the query.
+  info("Updating More Items");
+  var change3 = [
+    {
+      isVisit: true,
+      lastVisit: newTimeInMicroseconds(),
+      uri: "http://foo.mail.com/changeme1.html",
+      title: "foo",
+    },
+    {
+      isVisit: true,
+      lastVisit: newTimeInMicroseconds(),
+      uri: "http://foo.mail.com/changeme3.html",
+      title: "moz",
+      isTag: true,
+      tagArray: ["foo", "moz"],
+    },
+  ];
+  await task_populateDB(change3);
+  Assert.ok(!isInResult({ uri: "http://foo.mail.com/changeme1.html" }, root));
+  Assert.ok(isInResult({ uri: "http://foo.mail.com/changeme3.html" }, root));
+
+  // And now, delete one
+  info("Delete item outside of batch");
+  var change4 = [
+    {
+      isVisit: true,
+      lastVisit: newTimeInMicroseconds(),
+      uri: "http://moilla.com/",
+      title: "mo,z",
+    },
+  ];
+  await task_populateDB(change4);
+  Assert.ok(!isInResult(change4, root));
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_searchTerms_includeHidden.js b/toolkit/components/places/tests/queries/test_searchTerms_includeHidden.js
new file mode 100644
index 0000000000..224feb4f0c
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_searchTerms_includeHidden.js
@@ -0,0 +1,74 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Tests the interaction of includeHidden and searchTerms search options.
+
+var timeInMicroseconds = Date.now() * 1000;
+
+const VISITS = [
+  {
+    isVisit: true,
+    transType: TRANSITION_TYPED,
+    uri: "http://redirect.example.com/",
+    title: "example",
+    isRedirect: true,
+    lastVisit: timeInMicroseconds--,
+  },
+  {
+    isVisit: true,
+    transType: TRANSITION_TYPED,
+    uri: "http://target.example.com/",
+    title: "example",
+    lastVisit: timeInMicroseconds--,
+  },
+];
+
+const HIDDEN_VISITS = [
+  {
+    isVisit: true,
+    transType: TRANSITION_FRAMED_LINK,
+    uri: "http://hidden.example.com/",
+    title: "red",
+    lastVisit: timeInMicroseconds--,
+  },
+];
+
+const TEST_DATA = [
+  { searchTerms: "example", includeHidden: true, expectedResults: 2 },
+  { searchTerms: "example", includeHidden: false, expectedResults: 1 },
+  { searchTerms: "red", includeHidden: true, expectedResults: 1 },
+];
+
+add_task(async function test_initalize() {
+  await task_populateDB(VISITS);
+});
+
+add_task(async function test_searchTerms_includeHidden() {
+  for (let data of TEST_DATA) {
+    let query = PlacesUtils.history.getNewQuery();
+    query.searchTerms = data.searchTerms;
+    let options = PlacesUtils.history.getNewQueryOptions();
+    options.includeHidden = data.includeHidden;
+
+    let root = PlacesUtils.history.executeQuery(query, options).root;
+    root.containerOpen = true;
+
+    let cc = root.childCount;
+    // Live update with hidden visits.
+    await task_populateDB(HIDDEN_VISITS);
+    let cc_update = root.childCount;
+
+    root.containerOpen = false;
+
+    Assert.equal(cc, data.expectedResults);
+    Assert.equal(
+      cc_update,
+      data.expectedResults + (data.includeHidden ? 1 : 0)
+    );
+
+    await PlacesUtils.history.remove("http://hidden.example.com/");
+  }
+});
diff --git a/toolkit/components/places/tests/queries/test_search_tags.js b/toolkit/components/places/tests/queries/test_search_tags.js
new file mode 100644
index 0000000000..4f8cface07
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_search_tags.js
@@ -0,0 +1,73 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function test_search_for_tagged_bookmarks() {
+  const testURI = "http://a1.com";
+
+  let folder = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "bug 395101 test",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+  });
+
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: folder.guid,
+    title: "1 title",
+    url: testURI,
+  });
+
+  // tag the bookmarked URI
+  PlacesUtils.tagging.tagURI(uri(testURI), [
+    "elephant",
+    "walrus",
+    "giraffe",
+    "turkey",
+    "hiPPo",
+    "BABOON",
+    "alf",
+  ]);
+
+  // search for the bookmark, using a tag
+  var query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "elephant";
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+  query.setParents([folder.guid]);
+
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var rootNode = result.root;
+  rootNode.containerOpen = true;
+
+  Assert.equal(rootNode.childCount, 1);
+  Assert.equal(rootNode.getChild(0).bookmarkGuid, bookmark.guid);
+  rootNode.containerOpen = false;
+
+  // partial matches are okay
+  query.searchTerms = "wal";
+  result = PlacesUtils.history.executeQuery(query, options);
+  rootNode = result.root;
+  rootNode.containerOpen = true;
+  Assert.equal(rootNode.childCount, 1);
+  rootNode.containerOpen = false;
+
+  // case insensitive search term
+  query.searchTerms = "WALRUS";
+  result = PlacesUtils.history.executeQuery(query, options);
+  rootNode = result.root;
+  rootNode.containerOpen = true;
+  Assert.equal(rootNode.childCount, 1);
+  Assert.equal(rootNode.getChild(0).bookmarkGuid, bookmark.guid);
+  rootNode.containerOpen = false;
+
+  // case insensitive tag
+  query.searchTerms = "baboon";
+  result = PlacesUtils.history.executeQuery(query, options);
+  rootNode = result.root;
+  rootNode.containerOpen = true;
+  Assert.equal(rootNode.childCount, 1);
+  Assert.equal(rootNode.getChild(0).bookmarkGuid, bookmark.guid);
+  rootNode.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_searchterms-bookmarklets.js b/toolkit/components/places/tests/queries/test_searchterms-bookmarklets.js
new file mode 100644
index 0000000000..8eebf68cad
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_searchterms-bookmarklets.js
@@ -0,0 +1,63 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Check that bookmarklets are returned by searches with searchTerms.
+
+var testData = [
+  {
+    isInQuery: true,
+    isBookmark: true,
+    title: "bookmark 1",
+    uri: "http://mozilla.org/script/",
+  },
+
+  {
+    isInQuery: true,
+    isBookmark: true,
+    title: "bookmark 2",
+    uri: "javascript:alert('moz');",
+  },
+];
+
+add_task(async function test_initalize() {
+  await task_populateDB(testData);
+});
+
+add_test(function test_search_by_title() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "bookmark";
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  compareArrayToResult(testData, root);
+  root.containerOpen = false;
+
+  run_next_test();
+});
+
+add_test(function test_search_by_schemeToken() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "script";
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  compareArrayToResult(testData, root);
+  root.containerOpen = false;
+
+  run_next_test();
+});
+
+add_test(function test_search_by_uriAndTitle() {
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "moz";
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  compareArrayToResult(testData, root);
+  root.containerOpen = false;
+
+  run_next_test();
+});
diff --git a/toolkit/components/places/tests/queries/test_searchterms-domain.js b/toolkit/components/places/tests/queries/test_searchterms-domain.js
new file mode 100644
index 0000000000..6e14a7ef93
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_searchterms-domain.js
@@ -0,0 +1,197 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The test data for our database, note that the ordering of the results that
+// will be returned by the query (the isInQuery: true objects) is IMPORTANT.
+// see compareArrayToResult in head_queries.js for more info.
+var testData = [
+  // Test ftp protocol - vary the title length, embed search term
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    uri: "ftp://foo.com/ftp",
+    lastVisit: lastweek,
+    title: "hugelongconfmozlagurationofwordswithasearchtermsinit whoo-hoo",
+  },
+
+  // Test flat domain with annotation, search term in sentence
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://foo.com/",
+    annoName: "moz/test",
+    annoVal: "val",
+    lastVisit: lastweek,
+    title: "you know, moz is cool",
+  },
+
+  // Test subdomain included with isRedirect=true, different transtype
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "amozzie",
+    isRedirect: true,
+    uri: "http://mail.foo.com/redirect",
+    lastVisit: old,
+    referrer: "http://myreferrer.com",
+    transType: PlacesUtils.history.TRANSITION_LINK,
+  },
+
+  // Test subdomain inclued, search term at end
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    uri: "http://mail.foo.com/yiihah",
+    title: "blahmoz",
+    lastVisit: daybefore,
+  },
+
+  // Test www. style URI is included, with a tag
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isTag: true,
+    uri: "http://www.foo.com/yiihah",
+    tagArray: ["moz"],
+    lastVisit: yesterday,
+    title: "foo",
+  },
+
+  // Test https protocol
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "https://foo.com/",
+    lastVisit: today,
+  },
+
+  // Begin the invalid queries: wrong search term
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m o z",
+    uri: "http://foo.com/tooearly.php",
+    lastVisit: today,
+  },
+
+  // Test bad URI
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "http://sffoo.com/justwrong.htm",
+    lastVisit: yesterday,
+  },
+
+  // Test what we do with escaping in titles
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m%0o%0z",
+    uri: "http://foo.com/changeme1.htm",
+    lastVisit: yesterday,
+  },
+
+  // Test another invalid title - for updating later
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m,oz",
+    uri: "http://foo.com/changeme2.htm",
+    lastVisit: yesterday,
+  },
+];
+
+/**
+ * This test will test Queries that use relative search terms and domain options
+ */
+add_task(async function test_searchterms_domain() {
+  await task_populateDB(testData);
+  var query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "moz";
+  query.domain = "foo.com";
+  query.domainIsHost = false;
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_ASCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  info("Number of items in result set: " + root.childCount);
+  for (var i = 0; i < root.childCount; ++i) {
+    info(
+      "result: " + root.getChild(i).uri + " Title: " + root.getChild(i).title
+    );
+  }
+
+  // Check our inital result set
+  compareArrayToResult(testData, root);
+
+  // If that passes, check liveupdate
+  // Add to the query set
+  info("Adding item to query");
+  var change1 = [
+    {
+      isVisit: true,
+      isDetails: true,
+      uri: "http://foo.com/added.htm",
+      title: "moz",
+      transType: PlacesUtils.history.TRANSITION_LINK,
+    },
+  ];
+  await task_populateDB(change1);
+  Assert.ok(isInResult(change1, root));
+
+  // Update an existing URI
+  info("Updating Item");
+  var change2 = [
+    { isDetails: true, uri: "http://foo.com/changeme1.htm", title: "moz" },
+  ];
+  await task_populateDB(change2);
+  Assert.ok(isInResult(change2, root));
+
+  // Add one and take one out of query set, and simply change one so that it
+  // still applies to the query.
+  info("Updating More Items");
+  var change3 = [
+    { isDetails: true, uri: "http://foo.com/changeme2.htm", title: "moz" },
+    {
+      isDetails: true,
+      uri: "http://mail.foo.com/yiihah",
+      title: "moz now updated",
+    },
+    { isDetails: true, uri: "ftp://foo.com/ftp", title: "gone" },
+  ];
+  await task_populateDB(change3);
+  Assert.ok(isInResult({ uri: "http://foo.com/changeme2.htm" }, root));
+  Assert.ok(isInResult({ uri: "http://mail.foo.com/yiihah" }, root));
+  Assert.ok(!isInResult({ uri: "ftp://foo.com/ftp" }, root));
+
+  // And now, delete one
+  info("Deleting items");
+  var change4 = [{ isDetails: true, uri: "https://foo.com/", title: "mo,z" }];
+  await task_populateDB(change4);
+  Assert.ok(!isInResult(change4, root));
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_searchterms-uri.js b/toolkit/components/places/tests/queries/test_searchterms-uri.js
new file mode 100644
index 0000000000..a10735ca04
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_searchterms-uri.js
@@ -0,0 +1,125 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The test data for our database, note that the ordering of the results that
+// will be returned by the query (the isInQuery: true objects) is IMPORTANT.
+// see compareArrayToResult in head_queries.js for more info.
+var testData = [
+  // Test flat domain with annotation, search term in sentence
+  {
+    isInQuery: true,
+    isVisit: true,
+    isDetails: true,
+    isPageAnnotation: true,
+    uri: "http://foo.com/",
+    annoName: "moz/test",
+    annoVal: "val",
+    lastVisit: lastweek,
+    title: "you know, moz is cool",
+  },
+
+  // Test https protocol
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "moz",
+    uri: "https://foo.com/",
+    lastVisit: today,
+  },
+
+  // Begin the invalid queries: wrong search term
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m o z",
+    uri: "http://foo.com/wrongsearch.php",
+    lastVisit: today,
+  },
+
+  // Test subdomain inclued, search term at end
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    uri: "http://mail.foo.com/yiihah",
+    title: "blahmoz",
+    lastVisit: daybefore,
+  },
+
+  // Test ftp protocol - vary the title length, embed search term
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    uri: "ftp://foo.com/ftp",
+    lastVisit: lastweek,
+    title: "hugelongconfmozlagurationofwordswithasearchtermsinit whoo-hoo",
+  },
+
+  // Test what we do with escaping in titles
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m%0o%0z",
+    uri: "http://foo.com/changeme1.htm",
+    lastVisit: yesterday,
+  },
+
+  // Test another invalid title - for updating later
+  {
+    isInQuery: false,
+    isVisit: true,
+    isDetails: true,
+    title: "m,oz",
+    uri: "http://foo.com/changeme2.htm",
+    lastVisit: yesterday,
+  },
+];
+
+/**
+ * This test will test Queries that use relative search terms and URI options
+ */
+add_task(async function test_searchterms_uri() {
+  await task_populateDB(testData);
+  var query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = "moz";
+  query.uri = uri("http://foo.com");
+
+  // Options
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_ASCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+
+  // Results
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+
+  info("Number of items in result set: " + root.childCount);
+  for (var i = 0; i < root.childCount; ++i) {
+    info(
+      "result: " + root.getChild(i).uri + " Title: " + root.getChild(i).title
+    );
+  }
+
+  // Check our inital result set
+  compareArrayToResult(testData, root);
+
+  // live update.
+  info("change title");
+  var change1 = [{ isDetails: true, uri: "http://foo.com/", title: "mo" }];
+  await task_populateDB(change1);
+
+  Assert.ok(!isInResult({ uri: "http://foo.com/" }, root));
+  var change2 = [{ isDetails: true, uri: "http://foo.com/", title: "moz" }];
+  await task_populateDB(change2);
+  Assert.ok(isInResult({ uri: "http://foo.com/" }, root));
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/queries/test_sort-date-site-grouping.js b/toolkit/components/places/tests/queries/test_sort-date-site-grouping.js
new file mode 100644
index 0000000000..358ab45fdb
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_sort-date-site-grouping.js
@@ -0,0 +1,223 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* ***** BEGIN LICENSE BLOCK *****
+  Any copyright is dedicated to the Public Domain.
+  http://creativecommons.org/publicdomain/zero/1.0/
+ * ***** END LICENSE BLOCK ***** */
+
+// This test ensures that the date and site type of |place:| query maintains
+// its quantifications correctly. Namely, it ensures that the date part of the
+// query is not lost when the domain queries are made.
+
+// We specifically craft these entries so that if a by Date and Site sorting is
+// applied, we find one domain in the today range, and two domains in the older
+// than six months range.
+// The correspondence between item in |testData| and date range is stored in
+// leveledTestData.
+var testData = [
+  {
+    isVisit: true,
+    uri: "file:///directory/1",
+    lastVisit: today,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/1",
+    lastVisit: today,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/2",
+    lastVisit: today,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "file:///directory/2",
+    lastVisit: olderthansixmonths,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/3",
+    lastVisit: olderthansixmonths,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/4",
+    lastVisit: olderthansixmonths,
+    title: "test visit",
+    isInQuery: true,
+  },
+  {
+    isVisit: true,
+    uri: "http://example.net/1",
+    lastVisit: olderthansixmonths + 1000,
+    title: "test visit",
+    isInQuery: true,
+  },
+];
+var leveledTestData = [
+  // Today
+  [
+    [0], // Today, local files
+    [1, 2],
+  ], // Today, example.com
+  // Older than six months
+  [
+    [3], // Older than six months, local files
+    [4, 5], // Older than six months, example.com
+    [6], // Older than six months, example.net
+  ],
+];
+
+// This test data is meant for live updating. The |levels| property indicates
+// date range index and then domain index.
+var testDataAddedLater = [
+  {
+    isVisit: true,
+    uri: "http://example.com/5",
+    lastVisit: olderthansixmonths,
+    title: "test visit",
+    isInQuery: true,
+    levels: [1, 1],
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/6",
+    lastVisit: olderthansixmonths,
+    title: "test visit",
+    isInQuery: true,
+    levels: [1, 1],
+  },
+  {
+    isVisit: true,
+    uri: "http://example.com/7",
+    lastVisit: today,
+    title: "test visit",
+    isInQuery: true,
+    levels: [0, 1],
+  },
+  {
+    isVisit: true,
+    uri: "file:///directory/3",
+    lastVisit: today,
+    title: "test visit",
+    isInQuery: true,
+    levels: [0, 0],
+  },
+];
+
+add_task(async function test_sort_date_site_grouping() {
+  await task_populateDB(testData);
+
+  // On Linux, the (local files) folder is shown after sites unlike Mac/Windows.
+  // Thus, we avoid running this test on Linux but this should be re-enabled
+  // after bug 624024 is resolved.
+  let isLinux = "@mozilla.org/gnome-gconf-service;1" in Cc;
+  if (isLinux) {
+    return;
+  }
+
+  // In this test, there are three levels of results:
+  // 1st: Date queries. e.g., today, last week, or older than 6 months.
+  // 2nd: Domain queries restricted to a date. e.g. mozilla.com today.
+  // 3rd: Actual visits. e.g. mozilla.com/index.html today.
+  //
+  // We store all the third level result roots so that we can easily close all
+  // containers and test live updating into specific results.
+  let roots = [];
+
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY;
+
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  // This corresponds to the number of date ranges.
+  Assert.equal(root.childCount, leveledTestData.length);
+
+  // We pass off to |checkFirstLevel| to check the first level of results.
+  for (let index = 0; index < leveledTestData.length; index++) {
+    let node = root.getChild(index);
+    checkFirstLevel(index, node, roots);
+  }
+
+  // Test live updating.
+  for (let visit of testDataAddedLater) {
+    await task_populateDB([visit]);
+    let oldLength = testData.length;
+    let i = visit.levels[0];
+    let j = visit.levels[1];
+    testData.push(visit);
+    leveledTestData[i][j].push(oldLength);
+    compareArrayToResult(
+      leveledTestData[i][j].map(x => testData[x]),
+      roots[i][j]
+    );
+  }
+
+  for (let i = 0; i < roots.length; i++) {
+    for (let j = 0; j < roots[i].length; j++) {
+      roots[i][j].containerOpen = false;
+    }
+  }
+
+  root.containerOpen = false;
+});
+
+function checkFirstLevel(index, node, roots) {
+  PlacesUtils.asContainer(node).containerOpen = true;
+
+  Assert.ok(PlacesUtils.nodeIsDay(node));
+  PlacesUtils.asQuery(node);
+  let query = node.query;
+  let options = node.queryOptions;
+
+  Assert.ok(query.hasBeginTime && query.hasEndTime);
+
+  // Here we check the second level of results.
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  roots.push([]);
+  root.containerOpen = true;
+
+  Assert.equal(root.childCount, leveledTestData[index].length);
+  for (var secondIndex = 0; secondIndex < root.childCount; secondIndex++) {
+    let child = PlacesUtils.asQuery(root.getChild(secondIndex));
+    checkSecondLevel(index, secondIndex, child, roots);
+  }
+  root.containerOpen = false;
+  node.containerOpen = false;
+}
+
+function checkSecondLevel(index, secondIndex, child, roots) {
+  let query = child.query;
+  let options = child.queryOptions;
+
+  Assert.ok(query.hasDomain);
+  Assert.ok(query.hasBeginTime && query.hasEndTime);
+
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  // We should now have that roots[index][secondIndex] is set to the second
+  // level's results root.
+  roots[index].push(root);
+
+  // We pass off to compareArrayToResult to check the third level of
+  // results.
+  root.containerOpen = true;
+  compareArrayToResult(
+    leveledTestData[index][secondIndex].map(x => testData[x]),
+    root
+  );
+  // We close |root|'s container later so that we can test live
+  // updates into it.
+}
diff --git a/toolkit/components/places/tests/queries/test_sorting.js b/toolkit/components/places/tests/queries/test_sorting.js
new file mode 100644
index 0000000000..fe4508de9b
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_sorting.js
@@ -0,0 +1,968 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var tests = [];
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_NONE,
+
+  async setup() {
+    info("Sorting test 1: SORT BY NONE");
+
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://example.com/b",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "y",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "z",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "x",
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = this._unsortedData;
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    // no reverse sorting for SORT BY NONE
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING,
+
+  async setup() {
+    info("Sorting test 2: SORT BY TITLE");
+
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://example.com/b1",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "y",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "z",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "x",
+        isInQuery: true,
+      },
+
+      // if titles are equal, should fall back to URI
+      {
+        isBookmark: true,
+        uri: "http://example.com/b2",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "y",
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[2],
+      this._unsortedData[0],
+      this._unsortedData[3],
+      this._unsortedData[1],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_ASCENDING,
+
+  async setup() {
+    info("Sorting test 3: SORT BY DATE");
+
+    var timeInMicroseconds = Date.now() * 1000;
+    this._unsortedData = [
+      {
+        isVisit: true,
+        isDetails: true,
+        isBookmark: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 0,
+        uri: "http://example.com/c1",
+        lastVisit: timeInMicroseconds - 2000,
+        title: "x1",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        isBookmark: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+        uri: "http://example.com/a",
+        lastVisit: timeInMicroseconds - 1000,
+        title: "z",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        isBookmark: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 2,
+        uri: "http://example.com/b",
+        lastVisit: timeInMicroseconds - 3000,
+        title: "y",
+        isInQuery: true,
+      },
+
+      // if dates are equal, should fall back to title
+      {
+        isVisit: true,
+        isDetails: true,
+        isBookmark: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 3,
+        uri: "http://example.com/c2",
+        lastVisit: timeInMicroseconds - 2000,
+        title: "x2",
+        isInQuery: true,
+      },
+
+      // if dates and title are equal, should fall back to bookmark index
+      {
+        isVisit: true,
+        isDetails: true,
+        isBookmark: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 4,
+        uri: "http://example.com/c2",
+        lastVisit: timeInMicroseconds - 2000,
+        title: "x2",
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[2],
+      this._unsortedData[0],
+      this._unsortedData[3],
+      this._unsortedData[4],
+      this._unsortedData[1],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_URI_ASCENDING,
+
+  async setup() {
+    info("Sorting test 4: SORT BY URI");
+
+    var timeInMicroseconds = Date.now() * 1000;
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        isDetails: true,
+        lastVisit: timeInMicroseconds,
+        uri: "http://example.com/b",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 0,
+        title: "y",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+        title: "x",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 2,
+        title: "z",
+        isInQuery: true,
+      },
+
+      // if URIs are equal, should fall back to date
+      {
+        isBookmark: true,
+        isDetails: true,
+        lastVisit: timeInMicroseconds + 1000,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 3,
+        title: "x",
+        isInQuery: true,
+      },
+
+      // if no URI (e.g., node is a folder), should fall back to title
+      {
+        isFolder: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 4,
+        title: "y",
+        isInQuery: true,
+      },
+
+      // if URIs and dates are equal, should fall back to bookmark index
+      {
+        isBookmark: true,
+        isDetails: true,
+        lastVisit: timeInMicroseconds + 1000,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 5,
+        title: "x",
+        isInQuery: true,
+      },
+
+      // if no URI and titles are equal, should fall back to bookmark index
+      {
+        isFolder: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 6,
+        title: "y",
+        isInQuery: true,
+      },
+
+      // if no URI and titles are equal, should fall back to title
+      {
+        isFolder: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 7,
+        title: "z",
+        isInQuery: true,
+      },
+
+      // Separator should go after folders.
+      {
+        isSeparator: true,
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 8,
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[4],
+      this._unsortedData[6],
+      this._unsortedData[7],
+      this._unsortedData[8],
+      this._unsortedData[2],
+      this._unsortedData[0],
+      this._unsortedData[1],
+      this._unsortedData[3],
+      this._unsortedData[5],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_URI_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_VISITCOUNT_ASCENDING,
+
+  async setup() {
+    info("Sorting test 5: SORT BY VISITCOUNT");
+
+    var timeInMicroseconds = Date.now() * 1000;
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        lastVisit: timeInMicroseconds,
+        title: "z",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 0,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        lastVisit: timeInMicroseconds,
+        title: "x",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/b1",
+        lastVisit: timeInMicroseconds,
+        title: "y1",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 2,
+        isInQuery: true,
+      },
+
+      // if visitCounts are equal, should fall back to date
+      {
+        isBookmark: true,
+        uri: "http://example.com/b2",
+        lastVisit: timeInMicroseconds + 1000,
+        title: "y2a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 3,
+        isInQuery: true,
+      },
+
+      // if visitCounts and dates are equal, should fall back to bookmark index
+      {
+        isBookmark: true,
+        uri: "http://example.com/b2",
+        lastVisit: timeInMicroseconds + 1000,
+        title: "y2b",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 4,
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[0],
+      this._unsortedData[2],
+      this._unsortedData[3],
+      this._unsortedData[4],
+      this._unsortedData[1],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+    // add visits to increase visit count
+    await PlacesTestUtils.addVisits([
+      {
+        uri: uri("http://example.com/a"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+      {
+        uri: uri("http://example.com/b1"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+      {
+        uri: uri("http://example.com/b1"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+      {
+        uri: uri("http://example.com/b2"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds + 1000,
+      },
+      {
+        uri: uri("http://example.com/b2"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds + 1000,
+      },
+      {
+        uri: uri("http://example.com/c"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+      {
+        uri: uri("http://example.com/c"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+      {
+        uri: uri("http://example.com/c"),
+        transition: TRANSITION_TYPED,
+        visitDate: timeInMicroseconds,
+      },
+    ]);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode =
+      Ci.nsINavHistoryQueryOptions.SORT_BY_VISITCOUNT_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_ASCENDING,
+
+  async setup() {
+    info("Sorting test 7: SORT BY DATEADDED");
+
+    var timeInMicroseconds = Date.now() * 1000;
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://example.com/b1",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 0,
+        title: "y1",
+        dateAdded: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+        title: "z",
+        dateAdded: timeInMicroseconds - 2000,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 2,
+        title: "x",
+        dateAdded: timeInMicroseconds,
+        isInQuery: true,
+      },
+
+      // if dateAddeds are equal, should fall back to title
+      {
+        isBookmark: true,
+        uri: "http://example.com/b2",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 3,
+        title: "y2",
+        dateAdded: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+
+      // if dateAddeds and titles are equal, should fall back to bookmark index
+      {
+        isBookmark: true,
+        uri: "http://example.com/b3",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 4,
+        title: "y3",
+        dateAdded: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[1],
+      this._unsortedData[0],
+      this._unsortedData[3],
+      this._unsortedData[4],
+      this._unsortedData[2],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode =
+      Ci.nsINavHistoryQueryOptions.SORT_BY_DATEADDED_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_LASTMODIFIED_ASCENDING,
+
+  async setup() {
+    info("Sorting test 8: SORT BY LASTMODIFIED");
+
+    var timeInMicroseconds = Date.now() * 1000;
+    var timeAddedInMicroseconds = timeInMicroseconds - 10000;
+
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://example.com/b1",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 0,
+        title: "y1",
+        dateAdded: timeAddedInMicroseconds,
+        lastModified: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/a",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 1,
+        title: "z",
+        dateAdded: timeAddedInMicroseconds,
+        lastModified: timeInMicroseconds - 2000,
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://example.com/c",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 2,
+        title: "x",
+        dateAdded: timeAddedInMicroseconds,
+        lastModified: timeInMicroseconds,
+        isInQuery: true,
+      },
+
+      // if lastModifieds are equal, should fall back to title
+      {
+        isBookmark: true,
+        uri: "http://example.com/b2",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 3,
+        title: "y2",
+        dateAdded: timeAddedInMicroseconds,
+        lastModified: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+
+      // if lastModifieds and titles are equal, should fall back to bookmark
+      // index
+      {
+        isBookmark: true,
+        uri: "http://example.com/b3",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: 4,
+        title: "y3",
+        dateAdded: timeAddedInMicroseconds,
+        lastModified: timeInMicroseconds - 1000,
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[1],
+      this._unsortedData[0],
+      this._unsortedData[3],
+      this._unsortedData[4],
+      this._unsortedData[2],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode =
+      Ci.nsINavHistoryQueryOptions.SORT_BY_LASTMODIFIED_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_TAGS_ASCENDING,
+
+  async setup() {
+    info("Sorting test 9: SORT BY TAGS");
+
+    this._unsortedData = [
+      {
+        isBookmark: true,
+        uri: "http://url2.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title x",
+        isTag: true,
+        tagArray: ["x", "y", "z"],
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://url1a.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title y1",
+        isTag: true,
+        tagArray: ["a", "b"],
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://url3a.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title w1",
+        isInQuery: true,
+      },
+
+      {
+        isBookmark: true,
+        uri: "http://url0.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title z",
+        isTag: true,
+        tagArray: ["a", "y", "z"],
+        isInQuery: true,
+      },
+
+      // if tags are equal, should fall back to title
+      {
+        isBookmark: true,
+        uri: "http://url1b.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title y2",
+        isTag: true,
+        tagArray: ["b", "a"],
+        isInQuery: true,
+      },
+
+      // if tags are equal, should fall back to title
+      {
+        isBookmark: true,
+        uri: "http://url3b.com/",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+        title: "title w2",
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[2],
+      this._unsortedData[5],
+      this._unsortedData[1],
+      this._unsortedData[4],
+      this._unsortedData[3],
+      this._unsortedData[0],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    // Query
+    var query = PlacesUtils.history.getNewQuery();
+    query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+    query.onlyBookmarked = true;
+
+    // query options
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    // Results - this gets the result set and opens it for reading and modification.
+    var result = PlacesUtils.history.executeQuery(query, options);
+    var root = result.root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_TAGS_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+// SORT_BY_FRECENCY_*
+
+tests.push({
+  _sortingMode: Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_ASCENDING,
+
+  async setup() {
+    info("Sorting test 13: SORT BY FRECENCY ");
+
+    let timeInMicroseconds = PlacesUtils.toPRTime(Date.now() - 10000);
+
+    function newTimeInMicroseconds() {
+      timeInMicroseconds = timeInMicroseconds + 1000;
+      return timeInMicroseconds;
+    }
+
+    this._unsortedData = [
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://moz.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "I",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://moz.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "I",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://moz.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "I",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://is.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "love",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://best.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "moz",
+        isInQuery: true,
+      },
+
+      {
+        isVisit: true,
+        isDetails: true,
+        uri: "http://best.com/",
+        lastVisit: newTimeInMicroseconds(),
+        title: "moz",
+        isInQuery: true,
+      },
+    ];
+
+    this._sortedData = [
+      this._unsortedData[3],
+      this._unsortedData[5],
+      this._unsortedData[2],
+    ];
+
+    // This function in head_queries.js creates our database with the above data
+    await task_populateDB(this._unsortedData);
+  },
+
+  check() {
+    var query = PlacesUtils.history.getNewQuery();
+    var options = PlacesUtils.history.getNewQueryOptions();
+    options.sortingMode = this._sortingMode;
+
+    var root = PlacesUtils.history.executeQuery(query, options).root;
+    root.containerOpen = true;
+    compareArrayToResult(this._sortedData, root);
+    root.containerOpen = false;
+  },
+
+  check_reverse() {
+    this._sortingMode =
+      Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_DESCENDING;
+    this._sortedData.reverse();
+    this.check();
+  },
+});
+
+add_task(async function test_sorting() {
+  for (let test of tests) {
+    await test.setup();
+    await PlacesTestUtils.promiseAsyncUpdates();
+    test.check();
+    // sorting reversed, usually SORT_BY have ASC and DESC
+    test.check_reverse();
+    // Execute cleanup tasks
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesUtils.history.clear();
+  }
+});
diff --git a/toolkit/components/places/tests/queries/test_tags.js b/toolkit/components/places/tests/queries/test_tags.js
new file mode 100644
index 0000000000..17ad3478ce
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_tags.js
@@ -0,0 +1,626 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Tests bookmark queries with tags.  See bug 399799.
+ */
+
+"use strict";
+
+add_task(async function tags_getter_setter() {
+  info("Tags getter/setter should work correctly");
+  info("Without setting tags, tags getter should return empty array");
+  var [query] = makeQuery();
+  Assert.equal(query.tags.length, 0);
+
+  info("Setting tags to an empty array, tags getter should return empty array");
+  [query] = makeQuery([]);
+  Assert.equal(query.tags.length, 0);
+
+  info("Setting a few tags, tags getter should return correct array");
+  var tags = ["bar", "baz", "foo"];
+  [query] = makeQuery(tags);
+  setsAreEqual(query.tags, tags, true);
+
+  info("Setting some dupe tags, tags getter return unique tags");
+  [query] = makeQuery(["foo", "foo", "bar", "foo", "baz", "bar"]);
+  setsAreEqual(query.tags, ["bar", "baz", "foo"], true);
+});
+
+add_task(async function invalid_setter_calls() {
+  info("Invalid calls to tags setter should fail");
+  try {
+    var query = PlacesUtils.history.getNewQuery();
+    query.tags = null;
+    do_throw("Passing null to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    query = PlacesUtils.history.getNewQuery();
+    query.tags = "this should not work";
+    do_throw("Passing a string to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery([null]);
+    do_throw("Passing one-element array with null to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery([undefined]);
+    do_throw("Passing one-element array with undefined to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery(["foo", null, "bar"]);
+    do_throw("Passing mixture of tags and null to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery(["foo", undefined, "bar"]);
+    do_throw("Passing mixture of tags and undefined to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery([1, 2, 3]);
+    do_throw("Passing numbers to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery(["foo", 1, 2, 3]);
+    do_throw("Passing mixture of tags and numbers to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    var str = PlacesUtils.toISupportsString("foo");
+    query = PlacesUtils.history.getNewQuery();
+    query.tags = str;
+    do_throw("Passing nsISupportsString to SetTags should fail");
+  } catch (exc) {}
+
+  try {
+    makeQuery([str]);
+    do_throw("Passing array of nsISupportsStrings to SetTags should fail");
+  } catch (exc) {}
+});
+
+add_task(async function not_setting_tags() {
+  info("Not setting tags at all should not affect query URI");
+  checkQueryURI();
+});
+
+add_task(async function empty_array_tags() {
+  info("Setting tags with an empty array should not affect query URI");
+  checkQueryURI([]);
+});
+
+add_task(async function set_tags() {
+  info("Setting some tags should result in correct query URI");
+  checkQueryURI([
+    "foo",
+    "七難",
+    "",
+    "いっぱいおっぱい",
+    "Abracadabra",
+    "１２３",
+    "Here's a pretty long tag name with some = signs and 1 2 3s and spaces oh jeez will it work I hope so!",
+    "アスキーでございません",
+    "あいうえお",
+  ]);
+});
+
+add_task(async function no_tags_tagsAreNot() {
+  info(
+    "Not setting tags at all but setting tagsAreNot should " +
+      "affect query URI"
+  );
+  checkQueryURI(null, true);
+});
+
+add_task(async function empty_array_tags_tagsAreNot() {
+  info(
+    "Setting tags with an empty array and setting tagsAreNot " +
+      "should affect query URI"
+  );
+  checkQueryURI([], true);
+});
+
+add_task(async function () {
+  info(
+    "Setting some tags and setting tagsAreNot should result in " +
+      "correct query URI"
+  );
+  checkQueryURI(
+    [
+      "foo",
+      "七難",
+      "",
+      "いっぱいおっぱい",
+      "Abracadabra",
+      "１２３",
+      "Here's a pretty long tag name with some = signs and 1 2 3s and spaces oh jeez will it work I hope so!",
+      "アスキーでございません",
+      "あいうえお",
+    ],
+    true
+  );
+});
+
+add_task(async function tag() {
+  info("Querying on tag associated with a URI should return that URI");
+  await task_doWithBookmark(["foo", "bar", "baz"], function (aURI) {
+    var [query, opts] = makeQuery(["foo"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["bar"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["baz"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+  });
+});
+
+add_task(async function many_tags() {
+  info("Querying on many tags associated with a URI should return that URI");
+  await task_doWithBookmark(["foo", "bar", "baz"], function (aURI) {
+    var [query, opts] = makeQuery(["foo", "bar"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["foo", "baz"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["bar", "baz"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["foo", "bar", "baz"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+  });
+});
+
+add_task(async function repeated_tag() {
+  info("Specifying the same tag multiple times should not matter");
+  await task_doWithBookmark(["foo", "bar", "baz"], function (aURI) {
+    var [query, opts] = makeQuery(["foo", "foo"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["foo", "foo", "foo", "bar", "bar", "baz"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+  });
+});
+
+add_task(async function many_tags_no_bookmark() {
+  info(
+    "Querying on many tags associated with a URI and tags not associated " +
+      "with that URI should not return that URI"
+  );
+  await task_doWithBookmark(["foo", "bar", "baz"], function (aURI) {
+    var [query, opts] = makeQuery(["foo", "bogus"]);
+    executeAndCheckQueryResults(query, opts, []);
+    [query, opts] = makeQuery(["foo", "bar", "bogus"]);
+    executeAndCheckQueryResults(query, opts, []);
+    [query, opts] = makeQuery(["foo", "bar", "baz", "bogus"]);
+    executeAndCheckQueryResults(query, opts, []);
+  });
+});
+
+add_task(async function nonexistent_tags() {
+  info("Querying on nonexistent tag should return no results");
+  await task_doWithBookmark(["foo", "bar", "baz"], function (aURI) {
+    var [query, opts] = makeQuery(["bogus"]);
+    executeAndCheckQueryResults(query, opts, []);
+    [query, opts] = makeQuery(["bogus", "gnarly"]);
+    executeAndCheckQueryResults(query, opts, []);
+  });
+});
+
+add_task(async function tagsAreNot() {
+  info("Querying bookmarks using tagsAreNot should work correctly");
+  var urisAndTags = {
+    "http://example.com/1": ["foo", "bar"],
+    "http://example.com/2": ["baz", "qux"],
+    "http://example.com/3": null,
+  };
+
+  info("Add bookmarks and tag the URIs");
+  for (let [pURI, tags] of Object.entries(urisAndTags)) {
+    let nsiuri = uri(pURI);
+    await addBookmark(nsiuri);
+    if (tags) {
+      PlacesUtils.tagging.tagURI(nsiuri, tags);
+    }
+  }
+
+  info('  Querying for "foo" should match only /2 and /3');
+  var [query, opts] = makeQuery(["foo"], true);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    "http://example.com/2",
+    "http://example.com/3",
+  ]);
+
+  info('  Querying for "foo" and "bar" should match only /2 and /3');
+  [query, opts] = makeQuery(["foo", "bar"], true);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    "http://example.com/2",
+    "http://example.com/3",
+  ]);
+
+  info('  Querying for "foo" and "bogus" should match only /2 and /3');
+  [query, opts] = makeQuery(["foo", "bogus"], true);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    "http://example.com/2",
+    "http://example.com/3",
+  ]);
+
+  info('  Querying for "foo" and "baz" should match only /3');
+  [query, opts] = makeQuery(["foo", "baz"], true);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    "http://example.com/3",
+  ]);
+
+  info('  Querying for "bogus" should match all');
+  [query, opts] = makeQuery(["bogus"], true);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    "http://example.com/1",
+    "http://example.com/2",
+    "http://example.com/3",
+  ]);
+
+  // Clean up.
+  for (let [pURI, tags] of Object.entries(urisAndTags)) {
+    let nsiuri = uri(pURI);
+    if (tags) {
+      PlacesUtils.tagging.untagURI(nsiuri, tags);
+    }
+  }
+  await task_cleanDatabase();
+});
+
+add_task(async function duplicate_tags() {
+  info(
+    "Duplicate existing tags (i.e., multiple tag folders with " +
+      "same name) should not throw off query results"
+  );
+  var tagName = "foo";
+
+  info("Add bookmark and tag it normally");
+  await addBookmark(TEST_URI);
+  PlacesUtils.tagging.tagURI(TEST_URI, [tagName]);
+
+  info("Manually create tag folder with same name as tag and insert bookmark");
+  let dupTag = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.tagsGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: tagName,
+  });
+
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: dupTag.guid,
+    title: "title",
+    url: TEST_URI,
+  });
+
+  info("Querying for tag should match URI");
+  var [query, opts] = makeQuery([tagName]);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    TEST_URI.spec,
+  ]);
+
+  PlacesUtils.tagging.untagURI(TEST_URI, [tagName]);
+  await task_cleanDatabase();
+});
+
+add_task(async function folder_named_as_tag() {
+  info(
+    "Regular folders with the same name as tag should not throw " +
+      "off query results"
+  );
+  var tagName = "foo";
+
+  info("Add bookmark and tag it");
+  await addBookmark(TEST_URI);
+  PlacesUtils.tagging.tagURI(TEST_URI, [tagName]);
+
+  info("Create folder with same name as tag");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: tagName,
+  });
+
+  info("Querying for tag should match URI");
+  var [query, opts] = makeQuery([tagName]);
+  queryResultsAre(PlacesUtils.history.executeQuery(query, opts).root, [
+    TEST_URI.spec,
+  ]);
+
+  PlacesUtils.tagging.untagURI(TEST_URI, [tagName]);
+  await task_cleanDatabase();
+});
+
+add_task(async function ORed_queries() {
+  info("Multiple queries ORed together should work");
+  var urisAndTags = {
+    "http://example.com/1": [],
+    "http://example.com/2": [],
+  };
+
+  // Search with lots of tags to make sure tag parameter substitution in SQL
+  // can handle it with more than one query.
+  for (let i = 0; i < 11; i++) {
+    urisAndTags["http://example.com/1"].push("/1 tag " + i);
+    urisAndTags["http://example.com/2"].push("/2 tag " + i);
+  }
+
+  info("Add bookmarks and tag the URIs");
+  for (let [pURI, tags] of Object.entries(urisAndTags)) {
+    let nsiuri = uri(pURI);
+    await addBookmark(nsiuri);
+    if (tags) {
+      PlacesUtils.tagging.tagURI(nsiuri, tags);
+    }
+  }
+
+  // Clean up.
+  for (let [pURI, tags] of Object.entries(urisAndTags)) {
+    let nsiuri = uri(pURI);
+    if (tags) {
+      PlacesUtils.tagging.untagURI(nsiuri, tags);
+    }
+  }
+  await task_cleanDatabase();
+});
+
+add_task(async function tag_casing() {
+  info(
+    "Querying on associated tags should return " +
+      "correct results irrespective of casing of tags."
+  );
+  await task_doWithBookmark(["fOo", "bAr"], function (aURI) {
+    let [query, opts] = makeQuery(["Foo"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["Foo", "Bar"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["Foo"], true);
+    executeAndCheckQueryResults(query, opts, []);
+    [query, opts] = makeQuery(["Bogus"], true);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+  });
+});
+
+add_task(async function tag_casing_l10n() {
+  info(
+    "Querying on associated tags should return " +
+      "correct results irrespective of casing of tags with international strings."
+  );
+  // \u041F is a lowercase \u043F
+  await task_doWithBookmark(
+    ["\u041F\u0442\u0438\u0446\u044B"],
+    function (aURI) {
+      let [query, opts] = makeQuery(["\u041F\u0442\u0438\u0446\u044B"]);
+      executeAndCheckQueryResults(query, opts, [aURI.spec]);
+      [query, opts] = makeQuery(["\u043F\u0442\u0438\u0446\u044B"]);
+      executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    }
+  );
+  await task_doWithBookmark(
+    ["\u043F\u0442\u0438\u0446\u044B"],
+    function (aURI) {
+      let [query, opts] = makeQuery(["\u041F\u0442\u0438\u0446\u044B"]);
+      executeAndCheckQueryResults(query, opts, [aURI.spec]);
+      [query, opts] = makeQuery(["\u043F\u0442\u0438\u0446\u044B"]);
+      executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    }
+  );
+});
+
+add_task(async function tag_special_char() {
+  info(
+    "Querying on associated tags should return " +
+      "correct results even if tags contain special characters."
+  );
+  await task_doWithBookmark(["Space ☺️ Between"], function (aURI) {
+    let [query, opts] = makeQuery(["Space ☺️ Between"]);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+    [query, opts] = makeQuery(["Space ☺️ Between"], true);
+    executeAndCheckQueryResults(query, opts, []);
+    [query, opts] = makeQuery(["Bogus"], true);
+    executeAndCheckQueryResults(query, opts, [aURI.spec]);
+  });
+});
+
+// The tag keys in query URIs, i.e., "place:tag=foo&!tags=1"
+//                                          ---     -----
+const QUERY_KEY_TAG = "tag";
+const QUERY_KEY_NOT_TAGS = "!tags";
+
+const TEST_URI = uri("http://example.com/");
+
+/**
+ * Adds a bookmark.
+ *
+ * @param aURI
+ *        URI of the page (an nsIURI)
+ */
+function addBookmark(aURI) {
+  return PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: aURI.spec,
+    url: aURI,
+  });
+}
+
+/**
+ * Asynchronous task that removes all pages from history and bookmarks.
+ */
+async function task_cleanDatabase(aCallback) {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+}
+
+/**
+ * Sets up a query with the specified tags, converts it to a URI, and makes sure
+ * the URI is what we expect it to be.
+ *
+ * @param aTags
+ *        The query's tags will be set to those in this array
+ * @param aTagsAreNot
+ *        The query's tagsAreNot property will be set to this
+ */
+function checkQueryURI(aTags, aTagsAreNot) {
+  var pairs = (aTags || []).sort().map(t => QUERY_KEY_TAG + "=" + encodeTag(t));
+  if (aTagsAreNot) {
+    pairs.push(QUERY_KEY_NOT_TAGS + "=1");
+  }
+  var expURI = "place:" + pairs.join("&");
+  var [query, opts] = makeQuery(aTags, aTagsAreNot);
+  var actualURI = queryURI(query, opts);
+  info("Query URI should be what we expect for the given tags");
+  Assert.equal(actualURI, expURI);
+}
+
+/**
+ * Asynchronous task that executes a callback task in a "scoped" database state.
+ * A bookmark is added and tagged before the callback is called, and afterward
+ * the database is cleared.
+ *
+ * @param aTags
+ *        A bookmark will be added and tagged with this array of tags
+ * @param aCallback
+ *        A task function that will be called after the bookmark has been tagged
+ */
+async function task_doWithBookmark(aTags, aCallback) {
+  await addBookmark(TEST_URI);
+  PlacesUtils.tagging.tagURI(TEST_URI, aTags);
+  await aCallback(TEST_URI);
+  PlacesUtils.tagging.untagURI(TEST_URI, aTags);
+  await task_cleanDatabase();
+}
+
+/**
+ * queryToQueryString() encodes every character in the query URI that doesn't
+ * match /[a-zA-Z]/.  There's no simple JavaScript function that does the same,
+ * but encodeURIComponent() comes close, only missing some punctuation.  This
+ * function takes care of all of that.
+ *
+ * @param  aTag
+ *         A tag name to encode
+ * @return A UTF-8 escaped string suitable for inclusion in a query URI
+ */
+function encodeTag(aTag) {
+  return encodeURIComponent(aTag).replace(
+    /[-_.!~*'()]/g, // '
+    s => "%" + s.charCodeAt(0).toString(16)
+  );
+}
+
+/**
+ * Executes the given query and compares the results to the given URIs.
+ * See queryResultsAre().
+ *
+ * @param aQuery
+ *        An nsINavHistoryQuery
+ * @param aQueryOpts
+ *        An nsINavHistoryQueryOptions
+ * @param aExpectedURIs
+ *        Array of URIs (as strings) that aResultRoot should contain
+ */
+function executeAndCheckQueryResults(aQuery, aQueryOpts, aExpectedURIs) {
+  var root = PlacesUtils.history.executeQuery(aQuery, aQueryOpts).root;
+  root.containerOpen = true;
+  queryResultsAre(root, aExpectedURIs);
+  root.containerOpen = false;
+}
+
+/**
+ * Returns new query and query options objects.  The query's tags will be
+ * set to aTags.  aTags may be null, in which case setTags() is not called at
+ * all on the query.
+ *
+ * @param  aTags
+ *         The query's tags will be set to those in this array
+ * @param  aTagsAreNot
+ *         The query's tagsAreNot property will be set to this
+ * @return [query, queryOptions]
+ */
+function makeQuery(aTags, aTagsAreNot) {
+  aTagsAreNot = !!aTagsAreNot;
+  info(
+    "Making a query " +
+      (aTags
+        ? "with tags " + aTags.toSource()
+        : "without calling setTags() at all") +
+      " and with tagsAreNot=" +
+      aTagsAreNot
+  );
+  var query = PlacesUtils.history.getNewQuery();
+  query.tagsAreNot = aTagsAreNot;
+  if (aTags) {
+    query.tags = aTags;
+    var uniqueTags = [];
+    aTags.forEach(function (t) {
+      if (typeof t === "string" && !uniqueTags.includes(t)) {
+        uniqueTags.push(t);
+      }
+    });
+    uniqueTags.sort();
+  }
+
+  info("Made query should be correct for tags and tagsAreNot");
+  if (uniqueTags) {
+    setsAreEqual(query.tags, uniqueTags, true);
+  }
+  var expCount = uniqueTags ? uniqueTags.length : 0;
+  Assert.equal(query.tags.length, expCount);
+  Assert.equal(query.tagsAreNot, aTagsAreNot);
+
+  return [query, PlacesUtils.history.getNewQueryOptions()];
+}
+
+/**
+ * Ensures that the URIs of aResultRoot are the same as those in aExpectedURIs.
+ *
+ * @param aResultRoot
+ *        The nsINavHistoryContainerResultNode root of an nsINavHistoryResult
+ * @param aExpectedURIs
+ *        Array of URIs (as strings) that aResultRoot should contain
+ */
+function queryResultsAre(aResultRoot, aExpectedURIs) {
+  var rootWasOpen = aResultRoot.containerOpen;
+  if (!rootWasOpen) {
+    aResultRoot.containerOpen = true;
+  }
+  var actualURIs = [];
+  for (let i = 0; i < aResultRoot.childCount; i++) {
+    actualURIs.push(aResultRoot.getChild(i).uri);
+  }
+  setsAreEqual(actualURIs, aExpectedURIs);
+  if (!rootWasOpen) {
+    aResultRoot.containerOpen = false;
+  }
+}
+
+/**
+ * Converts the given query into its query URI.
+ *
+ * @param  aQuery
+ *         An nsINavHistoryQuery
+ * @param  aQueryOpts
+ *         An nsINavHistoryQueryOptions
+ * @return The query's URI
+ */
+function queryURI(aQuery, aQueryOpts) {
+  return PlacesUtils.history.queryToQueryString(aQuery, aQueryOpts);
+}
+
+/**
+ * Ensures that the arrays contain the same elements and, optionally, in the
+ * same order.
+ */
+function setsAreEqual(aArr1, aArr2, aIsOrdered) {
+  Assert.equal(aArr1.length, aArr2.length);
+  if (aIsOrdered) {
+    for (let i = 0; i < aArr1.length; i++) {
+      Assert.equal(aArr1[i], aArr2[i]);
+    }
+  } else {
+    aArr1.forEach(u => Assert.ok(aArr2.includes(u)));
+    aArr2.forEach(u => Assert.ok(aArr1.includes(u)));
+  }
+}
diff --git a/toolkit/components/places/tests/queries/test_transitions.js b/toolkit/components/places/tests/queries/test_transitions.js
new file mode 100644
index 0000000000..3055f28e9f
--- /dev/null
+++ b/toolkit/components/places/tests/queries/test_transitions.js
@@ -0,0 +1,175 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* ***** BEGIN LICENSE BLOCK *****
+  Any copyright is dedicated to the Public Domain.
+  http://creativecommons.org/publicdomain/zero/1.0/
+ * ***** END LICENSE BLOCK ***** */
+var testData = [
+  {
+    isVisit: true,
+    title: "page 0",
+    uri: "http://mozilla.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_TYPED,
+  },
+  {
+    isVisit: true,
+    title: "page 1",
+    uri: "http://google.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  },
+  {
+    isVisit: true,
+    title: "page 2",
+    uri: "http://microsoft.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  },
+  {
+    isVisit: true,
+    title: "page 3",
+    uri: "http://en.wikipedia.org/",
+    transType: Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+  },
+  {
+    isVisit: true,
+    title: "page 4",
+    uri: "http://fr.wikipedia.org/",
+    transType: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  },
+  {
+    isVisit: true,
+    title: "page 5",
+    uri: "http://apple.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_TYPED,
+  },
+  {
+    isVisit: true,
+    title: "page 6",
+    uri: "http://campus-bike-store.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  },
+  {
+    isVisit: true,
+    title: "page 7",
+    uri: "http://uwaterloo.ca/",
+    transType: Ci.nsINavHistoryService.TRANSITION_TYPED,
+  },
+  {
+    isVisit: true,
+    title: "page 8",
+    uri: "http://pugcleaner.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+  },
+  {
+    isVisit: true,
+    title: "page 9",
+    uri: "http://de.wikipedia.org/",
+    transType: Ci.nsINavHistoryService.TRANSITION_TYPED,
+  },
+  {
+    isVisit: true,
+    title: "arewefastyet",
+    uri: "http://arewefastyet.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+  },
+  {
+    isVisit: true,
+    title: "arewefastyet",
+    uri: "http://arewefastyet.com/",
+    transType: Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+  },
+];
+// sets of indices of testData array by transition type
+var testDataTyped = [0, 5, 7, 9];
+var testDataDownload = [1, 2, 4, 6, 10];
+var testDataBookmark = [3, 8, 11];
+
+add_task(async function test_transitions() {
+  let timeNow = Date.now();
+  for (let item of testData) {
+    await PlacesTestUtils.addVisits({
+      uri: uri(item.uri),
+      transition: item.transType,
+      visitDate: timeNow++ * 1000,
+      title: item.title,
+    });
+  }
+
+  // dump_table("moz_places");
+  // dump_table("moz_historyvisits");
+
+  var numSortFunc = function (a, b) {
+    return a - b;
+  };
+  var arrs = testDataTyped
+    .concat(testDataDownload)
+    .concat(testDataBookmark)
+    .sort(numSortFunc);
+
+  // Four tests which compare the result of a query to an expected set.
+  var data = arrs.filter(function (index) {
+    return (
+      testData[index].uri.match(/arewefastyet\.com/) &&
+      testData[index].transType == Ci.nsINavHistoryService.TRANSITION_DOWNLOAD
+    );
+  });
+
+  compareQueryToTestData(
+    "place:domain=arewefastyet.com&transition=" +
+      Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+    data.slice()
+  );
+
+  compareQueryToTestData(
+    "place:transition=" + Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+    testDataDownload.slice()
+  );
+
+  compareQueryToTestData(
+    "place:transition=" + Ci.nsINavHistoryService.TRANSITION_TYPED,
+    testDataTyped.slice()
+  );
+
+  compareQueryToTestData(
+    "place:transition=" +
+      Ci.nsINavHistoryService.TRANSITION_DOWNLOAD +
+      "&transition=" +
+      Ci.nsINavHistoryService.TRANSITION_BOOKMARK,
+    data
+  );
+
+  // Tests the live update property of transitions.
+  var query = {};
+  var options = {};
+  PlacesUtils.history.queryStringToQuery(
+    "place:transition=" + Ci.nsINavHistoryService.TRANSITION_DOWNLOAD,
+    query,
+    options
+  );
+  var result = PlacesUtils.history.executeQuery(query.value, options.value);
+  var root = result.root;
+  root.containerOpen = true;
+  Assert.equal(testDataDownload.length, root.childCount);
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://getfirefox.com"),
+    transition: TRANSITION_DOWNLOAD,
+  });
+  Assert.equal(testDataDownload.length + 1, root.childCount);
+  root.containerOpen = false;
+});
+
+/*
+ * Takes a query and a set of indices. The indices correspond to elements
+ * of testData that are the result of the query.
+ */
+function compareQueryToTestData(queryStr, data) {
+  var query = {};
+  var options = {};
+  PlacesUtils.history.queryStringToQuery(queryStr, query, options);
+  var result = PlacesUtils.history.executeQuery(query.value, options.value);
+  var root = result.root;
+  for (var i = 0; i < data.length; i++) {
+    data[i] = testData[data[i]];
+    data[i].isInQuery = true;
+  }
+  compareArrayToResult(data, root);
+}
diff --git a/toolkit/components/places/tests/queries/xpcshell.ini b/toolkit/components/places/tests/queries/xpcshell.ini
new file mode 100644
index 0000000000..b968a46b5f
--- /dev/null
+++ b/toolkit/components/places/tests/queries/xpcshell.ini
@@ -0,0 +1,34 @@
+[DEFAULT]
+head = head_queries.js
+skip-if = toolkit == 'android'
+
+[test_415716.js]
+[test_abstime-annotation-domain.js]
+[test_abstime-annotation-uri.js]
+[test_async.js]
+[test_bookmarks.js]
+[test_containersQueries_sorting.js]
+[test_downloadHistory_liveUpdate.js]
+[test_excludeQueries.js]
+[test_history_queries_tags_liveUpdate.js]
+[test_history_queries_titles_liveUpdate.js]
+[test_onlyBookmarked.js]
+[test_options_inherit.js]
+[test_query_uri_liveupdate.js]
+[test_queryMultipleFolder.js]
+[test_querySerialization.js]
+[test_redirects.js]
+[test_result_observeHistoryDetails.js]
+[test_results-as-left-pane.js]
+[test_results-as-roots.js]
+[test_results-as-tag-query.js]
+[test_results-as-visit.js]
+[test_search_tags.js]
+[test_searchterms-domain.js]
+[test_searchterms-uri.js]
+[test_searchterms-bookmarklets.js]
+[test_sort-date-site-grouping.js]
+[test_sorting.js]
+[test_tags.js]
+[test_transitions.js]
+[test_searchTerms_includeHidden.js]
diff --git a/toolkit/components/places/tests/sync/head_sync.js b/toolkit/components/places/tests/sync/head_sync.js
new file mode 100644
index 0000000000..d6c8910ec3
--- /dev/null
+++ b/toolkit/components/places/tests/sync/head_sync.js
@@ -0,0 +1,449 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+
+var { CanonicalJSON } = ChromeUtils.import(
+  "resource://gre/modules/CanonicalJSON.jsm"
+);
+var { Log } = ChromeUtils.importESModule("resource://gre/modules/Log.sys.mjs");
+
+var { PlacesSyncUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesSyncUtils.sys.mjs"
+);
+var { SyncedBookmarksMirror } = ChromeUtils.importESModule(
+  "resource://gre/modules/SyncedBookmarksMirror.sys.mjs"
+);
+var { CommonUtils } = ChromeUtils.importESModule(
+  "resource://services-common/utils.sys.mjs"
+);
+var { FileTestUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/FileTestUtils.sys.mjs"
+);
+var {
+  HTTP_400,
+  HTTP_401,
+  HTTP_402,
+  HTTP_403,
+  HTTP_404,
+  HTTP_405,
+  HTTP_406,
+  HTTP_407,
+  HTTP_408,
+  HTTP_409,
+  HTTP_410,
+  HTTP_411,
+  HTTP_412,
+  HTTP_413,
+  HTTP_414,
+  HTTP_415,
+  HTTP_417,
+  HTTP_500,
+  HTTP_501,
+  HTTP_502,
+  HTTP_503,
+  HTTP_504,
+  HTTP_505,
+  HttpError,
+  HttpServer,
+} = ChromeUtils.import("resource://testing-common/httpd.js");
+
+// These titles are defined in Database::CreateBookmarkRoots
+const BookmarksMenuTitle = "menu";
+const BookmarksToolbarTitle = "toolbar";
+const UnfiledBookmarksTitle = "unfiled";
+const MobileBookmarksTitle = "mobile";
+
+function run_test() {
+  let bufLog = Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror");
+  bufLog.level = Log.Level.All;
+
+  let sqliteLog = Log.repository.getLogger("Sqlite");
+  sqliteLog.level = Log.Level.Error;
+
+  let formatter = new Log.BasicFormatter();
+  let appender = new Log.DumpAppender(formatter);
+  appender.level = Log.Level.All;
+
+  for (let log of [bufLog, sqliteLog]) {
+    log.addAppender(appender);
+  }
+
+  do_get_profile();
+  run_next_test();
+}
+
+// A test helper to insert local roots directly into Places, since the public
+// bookmarks APIs no longer support custom roots.
+async function insertLocalRoot({ guid, title }) {
+  await PlacesUtils.withConnectionWrapper(
+    "insertLocalRoot",
+    async function (db) {
+      let dateAdded = PlacesUtils.toPRTime(new Date());
+      await db.execute(
+        `
+        INSERT INTO moz_bookmarks(guid, type, parent, position, title,
+                                  dateAdded, lastModified)
+        VALUES(:guid, :type, (SELECT id FROM moz_bookmarks
+                              WHERE guid = :parentGuid),
+               (SELECT COUNT(*) FROM moz_bookmarks
+                WHERE parent = (SELECT id FROM moz_bookmarks
+                                WHERE guid = :parentGuid)),
+               :title, :dateAdded, :dateAdded)`,
+        {
+          guid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          parentGuid: PlacesUtils.bookmarks.rootGuid,
+          title,
+          dateAdded,
+        }
+      );
+    }
+  );
+}
+
+// Returns a `CryptoWrapper`-like object that wraps the Sync record cleartext.
+// This exists to avoid importing `record.js` from Sync.
+function makeRecord(cleartext) {
+  return new Proxy(
+    { cleartext },
+    {
+      get(target, property, receiver) {
+        if (property == "cleartext") {
+          return target.cleartext;
+        }
+        if (property == "cleartextToString") {
+          return () => JSON.stringify(target.cleartext);
+        }
+        return target.cleartext[property];
+      },
+      set(target, property, value, receiver) {
+        if (property == "cleartext") {
+          target.cleartext = value;
+        } else if (property != "cleartextToString") {
+          target.cleartext[property] = value;
+        }
+      },
+      has(target, property) {
+        return property == "cleartext" || property in target.cleartext;
+      },
+      deleteProperty(target, property) {},
+      ownKeys(target) {
+        return ["cleartext", ...Reflect.ownKeys(target)];
+      },
+    }
+  );
+}
+
+async function storeRecords(buf, records, options) {
+  await buf.store(records.map(makeRecord), options);
+}
+
+async function storeChangesInMirror(buf, changesToUpload) {
+  let cleartexts = [];
+  for (let recordId in changesToUpload) {
+    changesToUpload[recordId].synced = true;
+    cleartexts.push(changesToUpload[recordId].cleartext);
+  }
+  await storeRecords(buf, cleartexts, { needsMerge: false });
+  await PlacesSyncUtils.bookmarks.pushChanges(changesToUpload);
+}
+
+function inspectChangeRecords(changeRecords) {
+  let results = { updated: [], deleted: [] };
+  for (let [id, record] of Object.entries(changeRecords)) {
+    (record.tombstone ? results.deleted : results.updated).push(id);
+  }
+  results.updated.sort();
+  results.deleted.sort();
+  return results;
+}
+
+async function promiseManyDatesAdded(guids) {
+  let datesAdded = new Map();
+  let db = await PlacesUtils.promiseDBConnection();
+  for (let chunk of PlacesUtils.chunkArray(guids, 100)) {
+    let rows = await db.executeCached(
+      `
+      SELECT guid, dateAdded FROM moz_bookmarks
+      WHERE guid IN (${new Array(chunk.length).fill("?").join(",")})`,
+      chunk
+    );
+    if (rows.length != chunk.length) {
+      throw new TypeError("Can't fetch date added for nonexistent items");
+    }
+    for (let row of rows) {
+      let dateAdded = row.getResultByName("dateAdded") / 1000;
+      datesAdded.set(row.getResultByName("guid"), dateAdded);
+    }
+  }
+  return datesAdded;
+}
+
+async function fetchLocalTree(rootGuid) {
+  function bookmarkNodeToInfo(node) {
+    let { guid, index, title, typeCode: type } = node;
+    let itemInfo = { guid, index, title, type };
+    if (node.annos) {
+      let syncableAnnos = node.annos.filter(anno =>
+        [PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI].includes(
+          anno.name
+        )
+      );
+      if (syncableAnnos.length) {
+        itemInfo.annos = syncableAnnos;
+      }
+    }
+    if (node.uri) {
+      itemInfo.url = node.uri;
+    }
+    if (node.keyword) {
+      itemInfo.keyword = node.keyword;
+    }
+    if (node.children) {
+      itemInfo.children = node.children.map(bookmarkNodeToInfo);
+    }
+    if (node.tags) {
+      itemInfo.tags = node.tags.split(",").sort();
+    }
+    return itemInfo;
+  }
+  let root = await PlacesUtils.promiseBookmarksTree(rootGuid);
+  return bookmarkNodeToInfo(root);
+}
+
+async function assertLocalTree(rootGuid, expected, message) {
+  let actual = await fetchLocalTree(rootGuid);
+  if (!ObjectUtils.deepEqual(actual, expected)) {
+    info(
+      `Expected structure for ${rootGuid}: ${CanonicalJSON.stringify(expected)}`
+    );
+    info(
+      `Actual structure for ${rootGuid}:   ${CanonicalJSON.stringify(actual)}`
+    );
+    throw new Assert.constructor.AssertionError({ actual, expected, message });
+  }
+}
+
+function makeLivemarkServer() {
+  let server = new HttpServer();
+  server.registerPrefixHandler("/feed/", do_get_file("./livemark.xml"));
+  server.start(-1);
+  return {
+    server,
+    get site() {
+      let { identity } = server;
+      let host = identity.primaryHost.includes(":")
+        ? `[${identity.primaryHost}]`
+        : identity.primaryHost;
+      return `${identity.primaryScheme}://${host}:${identity.primaryPort}`;
+    },
+    stopServer() {
+      return new Promise(resolve => server.stop(resolve));
+    },
+  };
+}
+
+function shuffle(array) {
+  let results = [];
+  for (let i = 0; i < array.length; ++i) {
+    let randomIndex = Math.floor(Math.random() * (i + 1));
+    results[i] = results[randomIndex];
+    results[randomIndex] = array[i];
+  }
+  return results;
+}
+
+async function fetchAllKeywords(info) {
+  let entries = [];
+  await PlacesUtils.keywords.fetch(info, entry => entries.push(entry));
+  return entries;
+}
+
+async function openMirror(name, options = {}) {
+  let buf = await SyncedBookmarksMirror.open({
+    path: `${name}_buf.sqlite`,
+    recordStepTelemetry(...args) {
+      if (options.recordStepTelemetry) {
+        options.recordStepTelemetry.call(this, ...args);
+      }
+    },
+    recordValidationTelemetry(...args) {
+      if (options.recordValidationTelemetry) {
+        options.recordValidationTelemetry.call(this, ...args);
+      }
+    },
+  });
+  return buf;
+}
+
+function BookmarkObserver({ ignoreDates = true, skipTags = false } = {}) {
+  this.notifications = [];
+  this.ignoreDates = ignoreDates;
+  this.skipTags = skipTags;
+  this.handlePlacesEvents = this.handlePlacesEvents.bind(this);
+}
+
+BookmarkObserver.prototype = {
+  handlePlacesEvents(events) {
+    for (let event of events) {
+      switch (event.type) {
+        case "bookmark-added": {
+          if (this.skipTags && event.isTagging) {
+            continue;
+          }
+          let params = {
+            itemId: event.id,
+            parentId: event.parentId,
+            index: event.index,
+            type: event.itemType,
+            urlHref: event.url,
+            title: event.title,
+            guid: event.guid,
+            parentGuid: event.parentGuid,
+            source: event.source,
+          };
+          if (!this.ignoreDates) {
+            params.dateAdded = event.dateAdded;
+          }
+          this.notifications.push({ name: "bookmark-added", params });
+          break;
+        }
+        case "bookmark-removed": {
+          if (this.skipTags && event.isTagging) {
+            continue;
+          }
+          // Since we are now skipping tags on the listener side we don't
+          // prevent unTagging notifications from going out. These events cause empty
+          // tags folders to be removed which creates another bookmark-removed notification
+          if (
+            this.skipTags &&
+            event.parentGuid == PlacesUtils.bookmarks.tagsGuid
+          ) {
+            continue;
+          }
+          let params = {
+            itemId: event.id,
+            parentId: event.parentId,
+            index: event.index,
+            type: event.itemType,
+            urlHref: event.url || null,
+            title: event.title,
+            guid: event.guid,
+            parentGuid: event.parentGuid,
+            source: event.source,
+          };
+          this.notifications.push({ name: "bookmark-removed", params });
+          break;
+        }
+        case "bookmark-moved": {
+          const params = {
+            itemId: event.id,
+            type: event.itemType,
+            urlHref: event.url,
+            source: event.source,
+            guid: event.guid,
+            newIndex: event.index,
+            newParentGuid: event.parentGuid,
+            oldIndex: event.oldIndex,
+            oldParentGuid: event.oldParentGuid,
+            isTagging: event.isTagging,
+          };
+          this.notifications.push({ name: "bookmark-moved", params });
+          break;
+        }
+        case "bookmark-guid-changed": {
+          const params = {
+            itemId: event.id,
+            type: event.itemType,
+            urlHref: event.url,
+            guid: event.guid,
+            parentGuid: event.parentGuid,
+            source: event.source,
+            isTagging: event.isTagging,
+          };
+          this.notifications.push({ name: "bookmark-guid-changed", params });
+          break;
+        }
+        case "bookmark-title-changed": {
+          const params = {
+            itemId: event.id,
+            guid: event.guid,
+            title: event.title,
+            parentGuid: event.parentGuid,
+          };
+          this.notifications.push({ name: "bookmark-title-changed", params });
+          break;
+        }
+        case "bookmark-url-changed": {
+          const params = {
+            itemId: event.id,
+            type: event.itemType,
+            urlHref: event.url,
+            guid: event.guid,
+            parentGuid: event.parentGuid,
+            source: event.source,
+            isTagging: event.isTagging,
+          };
+          this.notifications.push({ name: "bookmark-url-changed", params });
+          break;
+        }
+      }
+    }
+  },
+
+  check(expectedNotifications) {
+    PlacesUtils.observers.removeListener(
+      [
+        "bookmark-added",
+        "bookmark-removed",
+        "bookmark-moved",
+        "bookmark-guid-changed",
+        "bookmark-title-changed",
+        "bookmark-url-changed",
+      ],
+      this.handlePlacesEvents
+    );
+    if (!ObjectUtils.deepEqual(this.notifications, expectedNotifications)) {
+      info(`Expected notifications: ${JSON.stringify(expectedNotifications)}`);
+      info(`Actual notifications: ${JSON.stringify(this.notifications)}`);
+      throw new Assert.constructor.AssertionError({
+        actual: this.notifications,
+        expected: expectedNotifications,
+      });
+    }
+  },
+};
+
+function expectBookmarkChangeNotifications(options) {
+  let observer = new BookmarkObserver(options);
+  PlacesUtils.observers.addListener(
+    [
+      "bookmark-added",
+      "bookmark-removed",
+      "bookmark-moved",
+      "bookmark-guid-changed",
+      "bookmark-title-changed",
+      "bookmark-url-changed",
+    ],
+    observer.handlePlacesEvents
+  );
+  return observer;
+}
+
+// Copies a support file to a temporary fixture file, allowing the support
+// file to be reused for multiple tests.
+async function setupFixtureFile(fixturePath) {
+  let fixtureFile = do_get_file(fixturePath);
+  let tempFile = FileTestUtils.getTempFile(fixturePath);
+  await IOUtils.copy(fixtureFile.path, tempFile.path);
+  return tempFile;
+}
diff --git a/toolkit/components/places/tests/sync/mirror_corrupt.sqlite b/toolkit/components/places/tests/sync/mirror_corrupt.sqlite
new file mode 100644
index 0000000000..ed3613447c
--- /dev/null
+++ b/toolkit/components/places/tests/sync/mirror_corrupt.sqlite
@@ -0,0 +1 @@
+Not a database!
diff --git a/toolkit/components/places/tests/sync/mirror_v1.sqlite b/toolkit/components/places/tests/sync/mirror_v1.sqlite
new file mode 100644
index 0000000000..f0b8853616
Binary files /dev/null and b/toolkit/components/places/tests/sync/mirror_v1.sqlite differ
diff --git a/toolkit/components/places/tests/sync/mirror_v5.sqlite b/toolkit/components/places/tests/sync/mirror_v5.sqlite
new file mode 100644
index 0000000000..2a798ae908
Binary files /dev/null and b/toolkit/components/places/tests/sync/mirror_v5.sqlite differ
diff --git a/toolkit/components/places/tests/sync/sync_utils_bookmarks.html b/toolkit/components/places/tests/sync/sync_utils_bookmarks.html
new file mode 100644
index 0000000000..53ad366b1f
--- /dev/null
+++ b/toolkit/components/places/tests/sync/sync_utils_bookmarks.html
@@ -0,0 +1,18 @@
+
+
+
+Bookmarks
+
      Bookmarks Menu
      
+
+
      
+    
      Mozilla
+    
      Mozilla home
+    
      Bookmarks Toolbar
      
+    
      Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar
+    
      
+        
      Firefox
+        
      Firefox home
+    
      
+
      
diff --git a/toolkit/components/places/tests/sync/sync_utils_bookmarks.json b/toolkit/components/places/tests/sync/sync_utils_bookmarks.json
new file mode 100644
index 0000000000..961140843d
--- /dev/null
+++ b/toolkit/components/places/tests/sync/sync_utils_bookmarks.json
@@ -0,0 +1,94 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1449080379324000,
+  "lastModified": 1471365727344000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1449080379324000,
+      "lastModified": 1471365683893000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "NnvGl3CRA4hC",
+          "title": "Mozilla",
+          "index": 0,
+          "dateAdded": 1471365662585000,
+          "lastModified": 1471365667573000,
+          "id": 6,
+          "charset": "UTF-8",
+          "annos": [
+            {
+              "name": "bookmarkProperties/description",
+              "flags": 0,
+              "expires": 4,
+              "value": "Mozilla home"
+            }
+          ],
+          "type": "text/x-moz-place",
+          "uri": "https://www.mozilla.org/"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1449080379324000,
+      "lastModified": 1471365683893000,
+      "id": 3,
+      "annos": [
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "value": "Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "APzP8MupzA8l",
+          "title": "Firefox",
+          "index": 0,
+          "dateAdded": 1471365681801000,
+          "lastModified": 1471365687887000,
+          "id": 7,
+          "charset": "UTF-8",
+          "tags": "browser",
+          "annos": [
+            {
+              "name": "bookmarkProperties/description",
+              "flags": 0,
+              "expires": 4,
+              "value": "Firefox home"
+            }
+          ],
+          "type": "text/x-moz-place",
+          "uri": "https://www.mozilla.org/en-US/firefox/",
+          "keyword": "fx"
+        }
+      ]
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1449080379324000,
+      "lastModified": 1471365629626000,
+      "id": 5,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder"
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/sync/test_bookmark_abort_merging.js b/toolkit/components/places/tests/sync/test_bookmark_abort_merging.js
new file mode 100644
index 0000000000..877feb99f4
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_abort_merging.js
@@ -0,0 +1,220 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var { AsyncShutdown } = ChromeUtils.importESModule(
+  "resource://gre/modules/AsyncShutdown.sys.mjs"
+);
+
+add_task(async function test_transaction_in_progress() {
+  let buf = await openMirror("transaction_in_progress");
+
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+  ]);
+
+  // This transaction should block merging until the transaction is committed.
+  info("Open transaction on Places connection");
+  await buf.db.execute("BEGIN EXCLUSIVE");
+
+  await Assert.rejects(
+    buf.apply(),
+    ex => ex.name == "MergeConflictError",
+    "Should not merge when a transaction is in progress"
+  );
+
+  info("Commit open transaction");
+  await buf.db.execute("COMMIT");
+
+  info("Merging should succeed after committing");
+  await buf.apply();
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_abort_store() {
+  let buf = await openMirror("abort_store");
+
+  let controller = new AbortController();
+  controller.abort();
+  await Assert.rejects(
+    storeRecords(
+      buf,
+      [
+        {
+          id: "menu",
+          parentid: "places",
+          type: "folder",
+          children: [],
+        },
+      ],
+      { signal: controller.signal }
+    ),
+    ex => ex.name == "InterruptedError",
+    "Should abort storing when signaled"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_abort_merging() {
+  let buf = await openMirror("abort_merging");
+
+  let controller = new AbortController();
+  controller.abort();
+  await Assert.rejects(
+    buf.apply({ signal: controller.signal }),
+    ex => ex.name == "InterruptedError",
+    "Should abort merge when signaled"
+  );
+
+  // Even though the merger is already finalized on the Rust side, the DB
+  // connection is still open on the JS side. Finalizing `buf` closes it.
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_blocker_state() {
+  let barrier = new AsyncShutdown.Barrier("Test");
+  let buf = await SyncedBookmarksMirror.open({
+    path: "blocker_state_buf.sqlite",
+    finalizeAt: barrier.client,
+    recordStepTelemetry(...args) {},
+    recordValidationTelemetry(...args) {},
+  });
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+  ]);
+
+  await buf.tryApply(buf.finalizeController.signal);
+  await barrier.wait();
+
+  let state = buf.progress.fetchState();
+  let names = [];
+  for (let s of state.steps) {
+    equal(typeof s.at, "number", `Should report timestamp for ${s.step}`);
+    switch (s.step) {
+      case "fetchLocalTree":
+        greaterOrEqual(
+          s.took,
+          0,
+          "Should report time taken to fetch local tree"
+        );
+        deepEqual(
+          s.counts,
+          [
+            { name: "items", count: 6 },
+            { name: "deletions", count: 0 },
+          ],
+          "Should report number of items in local tree"
+        );
+        break;
+
+      case "fetchRemoteTree":
+        greaterOrEqual(
+          s.took,
+          0,
+          "Should report time taken to fetch remote tree"
+        );
+        deepEqual(
+          s.counts,
+          [
+            { name: "items", count: 6 },
+            { name: "deletions", count: 0 },
+          ],
+          "Should report number of items in remote tree"
+        );
+        break;
+
+      case "merge":
+        greaterOrEqual(s.took, 0, "Should report time taken to merge");
+        deepEqual(
+          s.counts,
+          [{ name: "items", count: 6 }],
+          "Should report merge stats"
+        );
+        break;
+
+      case "apply":
+        greaterOrEqual(s.took, 0, "Should report time taken to apply");
+        ok(!("counts" in s), "Should not report counts for applying");
+        break;
+
+      case "notifyObservers":
+        greaterOrEqual(
+          s.took,
+          0,
+          "Should report time taken to notify observers"
+        );
+        ok(!("counts" in s), "Should not report counts for observers");
+        break;
+
+      case "fetchLocalChangeRecords":
+        greaterOrEqual(
+          s.took,
+          0,
+          "Should report time taken to fetch records for upload"
+        );
+        deepEqual(
+          s.counts,
+          [{ name: "items", count: 4 }],
+          "Should report number of records to upload"
+        );
+        break;
+
+      case "finalize":
+        ok(!("took" in s), "Should not report time taken to finalize");
+        ok(!("counts" in s), "Should not report counts for finalizing");
+    }
+    names.push(s.step);
+  }
+  deepEqual(
+    names,
+    [
+      "fetchLocalTree",
+      "fetchRemoteTree",
+      "merge",
+      "apply",
+      "notifyObservers",
+      "fetchLocalChangeRecords",
+      "finalize",
+    ],
+    "Should report merge progress after waiting on blocker"
+  );
+  ok(
+    buf.finalizeController.signal.aborted,
+    "Should abort finalize signal on shutdown"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_chunking.js b/toolkit/components/places/tests/sync/test_bookmark_chunking.js
new file mode 100644
index 0000000000..3652502a3d
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_chunking.js
@@ -0,0 +1,165 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// These tests ensure we correctly chunk statements that exceed SQLite's
+// binding parameter limit.
+
+// Inserts 1500 unfiled bookmarks. Using `PlacesUtils.bookmarks.insertTree`
+// is an order of magnitude slower, so we write bookmarks directly into the
+// database.
+async function insertManyUnfiledBookmarks(db, url) {
+  await db.executeCached(
+    `
+    INSERT OR IGNORE INTO moz_places(id, url, url_hash, rev_host, hidden,
+                                     frecency, guid)
+    VALUES((SELECT id FROM moz_places
+            WHERE url_hash = hash(:url) AND
+                  url = :url), :url, hash(:url), :revHost, 0, -1,
+           generate_guid())`,
+    { url: url.href, revHost: PlacesUtils.getReversedHost(url) }
+  );
+
+  let guids = [];
+
+  for (let position = 0; position < 1500; ++position) {
+    let title = position.toString(10);
+    let guid = title.padStart(12, "A");
+    await db.executeCached(
+      `
+      INSERT INTO moz_bookmarks(guid, parent, fk, position, type, title,
+                                syncStatus, syncChangeCounter)
+      VALUES(:guid, (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid),
+             (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND
+                                              url = :url),
+             :position, :type, :title, :syncStatus, 1)`,
+      {
+        guid,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        position,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        title,
+        syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      }
+    );
+    guids.push(guid);
+  }
+
+  return guids;
+}
+
+add_task(async function test_merged_item_chunking() {
+  let buf = await openMirror("merged_item_chunking");
+
+  info("Set up local tree with 1500 bookmarks");
+  let localGuids = await buf.db.executeTransaction(function () {
+    let url = new URL("http://example.com/a");
+    return insertManyUnfiledBookmarks(buf.db, url);
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Set up remote tree with 1500 bookmarks");
+  let toolbarRecord = makeRecord({
+    id: "toolbar",
+    parentid: "places",
+    type: "folder",
+    children: [],
+  });
+  let records = [toolbarRecord];
+  for (let i = 0; i < 1500; ++i) {
+    let title = i.toString(10);
+    let guid = title.padStart(12, "B");
+    toolbarRecord.children.push(guid);
+    records.push(
+      makeRecord({
+        id: guid,
+        parentid: "toolbar",
+        type: "bookmark",
+        title,
+        bmkUri: "http://example.com/b",
+      })
+    );
+  }
+  await buf.store(shuffle(records));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.unfiledGuid],
+    "Should leave unfiled with new remote structure unmerged"
+  );
+
+  let localChildRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+    "toolbar"
+  );
+  deepEqual(
+    localChildRecordIds,
+    toolbarRecord.children,
+    "Should apply all remote toolbar children"
+  );
+
+  let guidsToUpload = Object.keys(changesToUpload);
+  deepEqual(
+    guidsToUpload.sort(),
+    ["unfiled", ...localGuids].sort(),
+    "Should upload unfiled and all new local children"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_deletion_chunking() {
+  let buf = await openMirror("deletion_chunking");
+
+  info("Set up local tree with 1500 bookmarks");
+  let guids = await buf.db.executeTransaction(function () {
+    let url = new URL("http://example.com/a");
+    return insertManyUnfiledBookmarks(buf.db, url);
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Delete them all on the server");
+  let records = [
+    makeRecord({
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: [],
+    }),
+  ];
+  for (let guid of guids) {
+    records.push(
+      makeRecord({
+        id: guid,
+        deleted: true,
+      })
+    );
+  }
+  await buf.store(shuffle(records));
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+  deepEqual(changesToUpload, {}, "Should take all remote deletions");
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Shouldn't store tombstones for remote deletions");
+
+  let localChildRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+    "unfiled"
+  );
+  deepEqual(
+    localChildRecordIds,
+    [],
+    "Should delete all unfiled children locally"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_corruption.js b/toolkit/components/places/tests/sync/test_bookmark_corruption.js
new file mode 100644
index 0000000000..5f0b0afeef
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_corruption.js
@@ -0,0 +1,3290 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+async function reparentItem(db, guid, newParentGuid = null) {
+  await db.execute(
+    `
+     UPDATE moz_bookmarks SET
+       parent = IFNULL((SELECT id FROM moz_bookmarks
+                        WHERE guid = :newParentGuid), 0)
+     WHERE guid = :guid`,
+    { newParentGuid, guid }
+  );
+}
+
+async function getCountOfBookmarkRows(db) {
+  let queryRows = await db.execute("SELECT COUNT(*) FROM moz_bookmarks");
+  Assert.equal(queryRows.length, 1);
+  return queryRows[0].getResultByIndex(0);
+}
+
+add_task(async function test_multiple_parents() {
+  let buf = await openMirror("multiple_parents");
+  let now = Date.now();
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      modified: now / 1000 - 10,
+      children: ["bookmarkAAAA"],
+    },
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      modified: now / 1000 - 5,
+      children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkCCCC"],
+    },
+    {
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      modified: now / 1000 - 3,
+      children: ["bookmarkBBBB"],
+    },
+    {
+      id: "mobile",
+      parentid: "places",
+      type: "folder",
+      modified: now / 1000,
+      children: ["bookmarkCCCC"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "toolbar",
+      type: "bookmark",
+      title: "A",
+      modified: now / 1000 - 10,
+      bmkUri: "http://example.com/a",
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "mobile",
+      type: "bookmark",
+      title: "B",
+      modified: now / 1000 - 3,
+      bmkUri: "http://example.com/b",
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    localTimeSeconds: now / 1000,
+    remoteTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      "bookmarkAAAA",
+      "bookmarkBBBB",
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.mobileGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave items with new remote structure unmerged"
+  );
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    PlacesUtils.bookmarks.mobileGuid,
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+  ]);
+  deepEqual(changesToUpload, {
+    menu: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "menu",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+        title: BookmarksMenuTitle,
+        children: ["bookmarkAAAA"],
+      },
+    },
+    toolbar: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "toolbar",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.toolbarGuid),
+        title: BookmarksToolbarTitle,
+        children: [],
+      },
+    },
+    unfiled: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "unfiled",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+        title: UnfiledBookmarksTitle,
+        children: ["bookmarkBBBB"],
+      },
+    },
+    mobile: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "mobile",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.mobileGuid),
+        title: MobileBookmarksTitle,
+        children: [],
+      },
+    },
+    bookmarkAAAA: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkAAAA",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: datesAdded.get("bookmarkAAAA"),
+        bmkUri: "http://example.com/a",
+        title: "A",
+      },
+    },
+    bookmarkBBBB: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkBBBB",
+        type: "bookmark",
+        parentid: "unfiled",
+        hasDupe: true,
+        parentName: UnfiledBookmarksTitle,
+        dateAdded: datesAdded.get("bookmarkBBBB"),
+        bmkUri: "http://example.com/b",
+        title: "B",
+      },
+    },
+  });
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should parent (A B) correctly"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+
+  let newChangesToUpload = await buf.apply({
+    localTimeSeconds: now / 1000,
+    remoteTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    newChangesToUpload,
+    {},
+    "Should not upload any changes after updating mirror"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_reupload_replace() {
+  let buf = await openMirror("reupload_replace");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+      },
+    ],
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "folderBBBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: [],
+      },
+    ],
+    { needsMerge: false }
+  );
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: [
+        "bookmarkAAAA",
+        "folderBBBBBB",
+        "queryCCCCCCC",
+        "queryDDDDDDD",
+      ],
+    },
+    {
+      // A has an invalid URL, but exists locally, so we should reupload a valid
+      // local copy. This discards _all_ remote changes to A.
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A (remote)",
+      bmkUri: "!@#$%",
+    },
+    {
+      id: "folderBBBBBB",
+      parentid: "menu",
+      type: "folder",
+      title: "B (remote)",
+      children: ["bookmarkEEEE"],
+    },
+    {
+      // E is a bookmark with an invalid URL that doesn't exist locally, so we'll
+      // delete it.
+      id: "bookmarkEEEE",
+      parentid: "folderBBBBBB",
+      type: "bookmark",
+      title: "E (remote)",
+      bmkUri: "!@#$%",
+    },
+    {
+      // C is a legacy tag query, so we'll rewrite its URL and reupload it.
+      id: "queryCCCCCCC",
+      parentid: "menu",
+      type: "query",
+      title: "C (remote)",
+      bmkUri: "place:type=7&folder=999",
+      folderName: "taggy",
+    },
+    {
+      // D is a query with an invalid URL, so we'll delete it.
+      id: "queryDDDDDDD",
+      parentid: "menu",
+      type: "query",
+      title: "D",
+      bmkUri: "^&*()",
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      "bookmarkAAAA",
+      "bookmarkEEEE",
+      "folderBBBBBB",
+      PlacesUtils.bookmarks.menuGuid,
+      "queryCCCCCCC",
+      "queryDDDDDDD",
+    ],
+    "Should leave invalid A, E, D; reuploaded C; B, menu with new remote structure unmerged"
+  );
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    "bookmarkAAAA",
+  ]);
+  deepEqual(changesToUpload, {
+    menu: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "menu",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+        title: BookmarksMenuTitle,
+        children: ["bookmarkAAAA", "folderBBBBBB", "queryCCCCCCC"],
+      },
+    },
+    bookmarkAAAA: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkAAAA",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: datesAdded.get("bookmarkAAAA"),
+        bmkUri: "http://example.com/a",
+        title: "A",
+      },
+    },
+    folderBBBBBB: {
+      // B is reuploaded because we deleted its child E.
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "folderBBBBBB",
+        type: "folder",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: undefined,
+        title: "B (remote)",
+        children: [],
+      },
+    },
+    queryCCCCCCC: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "queryCCCCCCC",
+        type: "query",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: undefined,
+        bmkUri: "place:tag=taggy",
+        title: "C (remote)",
+        folderName: "taggy",
+      },
+    },
+    queryDDDDDDD: {
+      tombstone: true,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "queryDDDDDDD",
+        deleted: true,
+      },
+    },
+    bookmarkEEEE: {
+      tombstone: true,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkEEEE",
+        deleted: true,
+      },
+    },
+  });
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["bookmarkEEEE", "queryDDDDDDD"],
+    "Should store local tombstones for (E D)"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_corrupt_local_roots() {
+  let buf = await openMirror("corrupt_roots");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkBBBB"],
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "toolbar",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+  ]);
+
+  try {
+    info("Move local menu into unfiled");
+    await reparentItem(
+      buf.db,
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.unfiledGuid
+    );
+    await Assert.rejects(
+      buf.apply(),
+      /The Places roots are invalid/,
+      "Should abort merge if local tree has misparented syncable root"
+    );
+
+    info("Move local Places root into toolbar");
+    await buf.db.executeTransaction(async function () {
+      await reparentItem(
+        buf.db,
+        PlacesUtils.bookmarks.menuGuid,
+        PlacesUtils.bookmarks.rootGuid
+      );
+      await reparentItem(
+        buf.db,
+        PlacesUtils.bookmarks.rootGuid,
+        PlacesUtils.bookmarks.toolbarGuid
+      );
+    });
+    await Assert.rejects(
+      buf.apply(),
+      /The Places roots are invalid/,
+      "Should abort merge if local tree has misparented Places root"
+    );
+  } finally {
+    info("Restore local roots");
+    await buf.db.executeTransaction(async function () {
+      await reparentItem(buf.db, PlacesUtils.bookmarks.rootGuid);
+      await reparentItem(
+        buf.db,
+        PlacesUtils.bookmarks.menuGuid,
+        PlacesUtils.bookmarks.rootGuid
+      );
+    });
+  }
+
+  info("Apply remote with restored roots");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  deepEqual(changesToUpload, {}, "Should not reupload any local records");
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should parent (A B) correctly with restored roots"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_corrupt_remote_roots() {
+  let buf = await openMirror("corrupt_remote_roots");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes: Menu > Unfiled");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["unfiled", "bookmarkAAAA"],
+    },
+    {
+      id: "unfiled",
+      parentid: "menu",
+      type: "folder",
+      children: ["bookmarkBBBB"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      id: "toolbar",
+      deleted: true,
+    },
+  ]);
+
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave deleted roots unmerged"
+  );
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          title: BookmarksMenuTitle,
+          children: ["bookmarkAAAA"],
+        },
+      },
+      unfiled: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "unfiled",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+          title: UnfiledBookmarksTitle,
+          children: ["bookmarkBBBB"],
+        },
+      },
+      toolbar: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "toolbar",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.toolbarGuid),
+          title: BookmarksToolbarTitle,
+          children: [],
+        },
+      },
+    },
+    "Should reupload invalid roots"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should not corrupt local roots"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Should not store local tombstones");
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_missing_children() {
+  let buf = await openMirror("missing_childen");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes: A > ([B] C [D E])");
+  {
+    await storeRecords(
+      buf,
+      shuffle([
+        {
+          id: "menu",
+          parentid: "places",
+          type: "folder",
+          children: [
+            "bookmarkBBBB",
+            "bookmarkCCCC",
+            "bookmarkDDDD",
+            "bookmarkEEEE",
+          ],
+        },
+        {
+          id: "bookmarkCCCC",
+          parentid: "menu",
+          type: "bookmark",
+          bmkUri: "http://example.com/c",
+          title: "C",
+        },
+      ])
+    );
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      [PlacesUtils.bookmarks.menuGuid],
+      "Should leave menu with new remote structure unmerged"
+    );
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["menu"],
+        deleted: [],
+      },
+      "Should reupload menu without missing children (B D E)"
+    );
+    await assertLocalTree(
+      PlacesUtils.bookmarks.menuGuid,
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: BookmarksMenuTitle,
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "C",
+            url: "http://example.com/c",
+          },
+        ],
+      },
+      "Menu children should be (C)"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  info("Add (B E) to remote");
+  {
+    await storeRecords(
+      buf,
+      shuffle([
+        {
+          id: "bookmarkBBBB",
+          parentid: "menu",
+          type: "bookmark",
+          title: "B",
+          bmkUri: "http://example.com/b",
+        },
+        {
+          id: "bookmarkEEEE",
+          parentid: "menu",
+          type: "bookmark",
+          title: "E",
+          bmkUri: "http://example.com/e",
+        },
+      ])
+    );
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["bookmarkBBBB", "bookmarkEEEE"],
+      "Should leave B, E with new remote structure unmerged"
+    );
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["bookmarkBBBB", "bookmarkEEEE", "menu"],
+        deleted: [],
+      },
+      "Should reupload menu and restored children"
+    );
+    await assertLocalTree(
+      PlacesUtils.bookmarks.menuGuid,
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: BookmarksMenuTitle,
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "C",
+            url: "http://example.com/c",
+          },
+          {
+            guid: "bookmarkBBBB",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 1,
+            title: "B",
+            url: "http://example.com/b",
+          },
+          {
+            guid: "bookmarkEEEE",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 2,
+            title: "E",
+            url: "http://example.com/e",
+          },
+        ],
+      },
+      "Menu children should be (C B E)"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  info("Add D to remote");
+  {
+    await storeRecords(buf, [
+      {
+        id: "bookmarkDDDD",
+        parentid: "menu",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+    ]);
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["bookmarkDDDD"],
+      "Should leave D with new remote structure unmerged"
+    );
+
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["bookmarkDDDD", "menu"],
+        deleted: [],
+      },
+      "Should reupload complete menu"
+    );
+    await assertLocalTree(
+      PlacesUtils.bookmarks.menuGuid,
+      {
+        guid: PlacesUtils.bookmarks.menuGuid,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        index: 0,
+        title: BookmarksMenuTitle,
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 0,
+            title: "C",
+            url: "http://example.com/c",
+          },
+          {
+            guid: "bookmarkBBBB",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 1,
+            title: "B",
+            url: "http://example.com/b",
+          },
+          {
+            guid: "bookmarkEEEE",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 2,
+            title: "E",
+            url: "http://example.com/e",
+          },
+          {
+            guid: "bookmarkDDDD",
+            type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+            index: 3,
+            title: "D",
+            url: "http://example.com/d",
+          },
+        ],
+      },
+      "Menu children should be (C B E D)"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_new_orphan_without_local_parent() {
+  let buf = await openMirror("new_orphan_without_local_parent");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // A doesn't exist locally, so we move the bookmarks into "unfiled" without
+  // reuploading. When the partial uploader returns and uploads A, we'll
+  // move the bookmarks to the correct folder.
+  info("Make remote changes: [A] > (B C D)");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B (remote)",
+        bmkUri: "http://example.com/b-remote",
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C (remote)",
+        bmkUri: "http://example.com/c-remote",
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "D (remote)",
+        bmkUri: "http://example.com/d-remote",
+      },
+    ])
+  );
+
+  info("Apply remote with (B C D)");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      [
+        "bookmarkBBBB",
+        "bookmarkCCCC",
+        "bookmarkDDDD",
+        PlacesUtils.bookmarks.unfiledGuid,
+      ],
+      "Should leave orphans B, C, D unmerged"
+    );
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD", "unfiled"],
+        deleted: [],
+      },
+      "Should reupload orphans (B C D)"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.unfiledGuid,
+    {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: UnfiledBookmarksTitle,
+      children: [
+        {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "B (remote)",
+          url: "http://example.com/b-remote",
+        },
+        {
+          guid: "bookmarkCCCC",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "C (remote)",
+          url: "http://example.com/c-remote",
+        },
+        {
+          guid: "bookmarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "D (remote)",
+          url: "http://example.com/d-remote",
+        },
+      ],
+    },
+    "Should move (B C D) to unfiled"
+  );
+
+  // A is an orphan because we don't have E locally, but we should move
+  // (B C D) into A.
+  info("Add [E] > A to remote");
+  await storeRecords(buf, [
+    {
+      id: "folderAAAAAA",
+      parentid: "folderEEEEEE",
+      type: "folder",
+      title: "A",
+      children: ["bookmarkDDDD", "bookmarkCCCC", "bookmarkBBBB"],
+    },
+  ]);
+
+  info("Apply remote with A");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["folderAAAAAA"],
+      "Should leave A with new remote structure unmerged"
+    );
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: [
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkDDDD",
+          "folderAAAAAA",
+          "unfiled",
+        ],
+        deleted: [],
+      },
+      "Should reupload A and its children"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.unfiledGuid,
+    {
+      guid: PlacesUtils.bookmarks.unfiledGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 3,
+      title: UnfiledBookmarksTitle,
+      children: [
+        {
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A",
+          children: [
+            {
+              guid: "bookmarkDDDD",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "D (remote)",
+              url: "http://example.com/d-remote",
+            },
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "C (remote)",
+              url: "http://example.com/c-remote",
+            },
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 2,
+              title: "B (remote)",
+              url: "http://example.com/b-remote",
+            },
+          ],
+        },
+      ],
+    },
+    "Should move (D C B) into A"
+  );
+
+  info("Add E to remote");
+  await storeRecords(buf, [
+    {
+      id: "folderEEEEEE",
+      parentid: "menu",
+      type: "folder",
+      title: "E",
+      children: ["folderAAAAAA"],
+    },
+  ]);
+
+  info("Apply remote with E");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["folderEEEEEE"],
+      "Should leave E with new remote structure unmerged"
+    );
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["folderAAAAAA", "folderEEEEEE", "menu", "unfiled"],
+        deleted: [],
+      },
+      "Should move E out of unfiled into menu"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "folderEEEEEE",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "E",
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkDDDD",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "D (remote)",
+                  url: "http://example.com/d-remote",
+                },
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "C (remote)",
+                  url: "http://example.com/c-remote",
+                },
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 2,
+                  title: "B (remote)",
+                  url: "http://example.com/b-remote",
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+    "Should move Menu > E > A"
+  );
+
+  info("Add Menu > E to remote");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["folderEEEEEE"],
+    },
+  ]);
+
+  info("Apply remote with menu");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: [],
+        deleted: [],
+      },
+      "Should not reupload after forming complete tree"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderEEEEEE",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "E",
+              children: [
+                {
+                  guid: "folderAAAAAA",
+                  type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                  index: 0,
+                  title: "A",
+                  children: [
+                    {
+                      guid: "bookmarkDDDD",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 0,
+                      title: "D (remote)",
+                      url: "http://example.com/d-remote",
+                    },
+                    {
+                      guid: "bookmarkCCCC",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 1,
+                      title: "C (remote)",
+                      url: "http://example.com/c-remote",
+                    },
+                    {
+                      guid: "bookmarkBBBB",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 2,
+                      title: "B (remote)",
+                      url: "http://example.com/b-remote",
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should form complete tree after applying E"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_into_orphaned() {
+  let buf = await openMirror("move_into_orphaned");
+
+  info("Set up mirror: Menu > (A B (C > (D (E > F))))");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      },
+      {
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "C",
+        children: [
+          {
+            guid: "bookmarkDDDD",
+            title: "D",
+            url: "http://example.com/d",
+          },
+          {
+            guid: "folderEEEEEE",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "E",
+            children: [
+              {
+                guid: "bookmarkFFFF",
+                title: "F",
+                url: "http://example.com/f",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB", "folderCCCCCC"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "menu",
+        type: "folder",
+        title: "C",
+        children: ["bookmarkDDDD", "folderEEEEEE"],
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderCCCCCC",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+      {
+        id: "folderEEEEEE",
+        parentid: "folderCCCCCC",
+        type: "folder",
+        title: "E",
+        children: ["bookmarkFFFF"],
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "folderEEEEEE",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete D, add E > I");
+  await PlacesUtils.bookmarks.remove("bookmarkDDDD");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkIIII",
+    parentGuid: "folderEEEEEE",
+    title: "I (local)",
+    url: "http://example.com/i",
+  });
+
+  // G doesn't exist on the server.
+  info("Make remote changes: ([G] > A (C > (D H E))), (C > H)");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkAAAA",
+        parentid: "folderGGGGGG",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "folderGGGGGG",
+        type: "folder",
+        title: "C",
+        children: ["bookmarkDDDD", "bookmarkHHHH", "folderEEEEEE"],
+      },
+      {
+        id: "bookmarkHHHH",
+        parentid: "folderCCCCCC",
+        type: "bookmark",
+        title: "H (remote)",
+        bmkUri: "http://example.com/h-remote",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkAAAA", "folderCCCCCC"],
+    "Should leave orphaned A, C with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [
+        "bookmarkAAAA",
+        "bookmarkIIII",
+        "folderCCCCCC",
+        "folderEEEEEE",
+        "menu",
+      ],
+      deleted: ["bookmarkDDDD"],
+    },
+    "Should upload records for (A I C E); tombstone for D"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              // A remains in its original place, since we don't use the `parentid`,
+              // and we don't have a record for G.
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "B",
+              url: "http://example.com/b",
+            },
+            {
+              // C exists on the server, so we take its children and order. D was
+              // deleted locally, and doesn't exist remotely. C is also a child of
+              // G, but we don't have a record for it on the server.
+              guid: "folderCCCCCC",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 2,
+              title: "C",
+              children: [
+                {
+                  guid: "bookmarkHHHH",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "H (remote)",
+                  url: "http://example.com/h-remote",
+                },
+                {
+                  guid: "folderEEEEEE",
+                  type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                  index: 1,
+                  title: "E",
+                  children: [
+                    {
+                      guid: "bookmarkFFFF",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 0,
+                      title: "F",
+                      url: "http://example.com/f",
+                    },
+                    {
+                      guid: "bookmarkIIII",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 1,
+                      title: "I (local)",
+                      url: "http://example.com/i",
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should treat local tree as canonical if server is missing new parent"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["bookmarkDDDD"],
+    "Should store local tombstone for D"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_new_orphan_with_local_parent() {
+  let buf = await openMirror("new_orphan_with_local_parent");
+
+  info("Set up mirror: A > (B D)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB", "bookmarkEEEE"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Simulate a partial write by another device that uploaded only B and C. A
+  // exists locally, so we can move B and C into the correct folder, but not
+  // the correct positions.
+  info("Set up remote with orphans: [A] > (C D)");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkDDDD",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      title: "D (remote)",
+      bmkUri: "http://example.com/d-remote",
+    },
+    {
+      id: "bookmarkCCCC",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      title: "C (remote)",
+      bmkUri: "http://example.com/c-remote",
+    },
+  ]);
+
+  info("Apply remote with (C D)");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["bookmarkCCCC", "bookmarkDDDD"],
+      "Should leave orphaned C, D unmerged"
+    );
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: ["bookmarkCCCC", "bookmarkDDDD", "folderAAAAAA"],
+        deleted: [],
+      },
+      "Should reupload orphans (C D) and folder A"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "B",
+                  url: "http://example.com/b",
+                },
+                {
+                  guid: "bookmarkEEEE",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "E",
+                  url: "http://example.com/e",
+                },
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 2,
+                  title: "C (remote)",
+                  url: "http://example.com/c-remote",
+                },
+                {
+                  guid: "bookmarkDDDD",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 3,
+                  title: "D (remote)",
+                  url: "http://example.com/d-remote",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should move (C D) to end of A"
+  );
+
+  // The partial uploader returns and uploads A.
+  info("Add A to remote");
+  await storeRecords(buf, [
+    {
+      id: "folderAAAAAA",
+      parentid: "menu",
+      type: "folder",
+      title: "A",
+      children: [
+        "bookmarkCCCC",
+        "bookmarkDDDD",
+        "bookmarkEEEE",
+        "bookmarkBBBB",
+      ],
+    },
+  ]);
+
+  info("Apply remote with A");
+  {
+    let changesToUpload = await buf.apply();
+    deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+    let idsToUpload = inspectChangeRecords(changesToUpload);
+    deepEqual(
+      idsToUpload,
+      {
+        updated: [],
+        deleted: [],
+      },
+      "Should not reupload orphan A"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+  }
+
+  await assertLocalTree(
+    "folderAAAAAA",
+    {
+      guid: "folderAAAAAA",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "A",
+      children: [
+        {
+          guid: "bookmarkCCCC",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "C (remote)",
+          url: "http://example.com/c-remote",
+        },
+        {
+          guid: "bookmarkDDDD",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "D (remote)",
+          url: "http://example.com/d-remote",
+        },
+        {
+          guid: "bookmarkEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "E",
+          url: "http://example.com/e",
+        },
+        {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 3,
+          title: "B",
+          url: "http://example.com/b",
+        },
+      ],
+    },
+    "Should update child positions once A exists in mirror"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_tombstone_as_child() {
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let buf = await openMirror("tombstone_as_child");
+  // Setup the mirror such that an incoming folder references a tombstone
+  // as a child.
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkAAAA", "bookmarkTTTT", "bookmarkBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "Bookmark A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "Bookmark B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkTTTT",
+        deleted: true,
+      },
+    ]),
+    { needsMerge: true }
+  );
+
+  let changesToUpload = await buf.apply();
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(idsToUpload.deleted, [], "no new tombstones were created.");
+  deepEqual(idsToUpload.updated, ["folderAAAAAA"], "parent is re-uploaded");
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkAAAA",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  url: "http://example.com/a",
+                  index: 0,
+                  title: "Bookmark A",
+                },
+                {
+                  // Note that this was the 3rd child specified on the server record,
+                  // but we we've correctly moved it back to being the second after
+                  // ignoring the tombstone.
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  url: "http://example.com/b",
+                  index: 1,
+                  title: "Bookmark B",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should have ignored tombstone record"
+  );
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Should not store local tombstones");
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_non_syncable_items() {
+  let buf = await openMirror("non_syncable_items");
+
+  info("Insert local orphaned left pane queries");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        guid: "folderLEFTPQ",
+        url: "place:folder=SOMETHING",
+        title: "Some query",
+      },
+      {
+        guid: "folderLEFTPC",
+        url: "place:folder=SOMETHING_ELSE",
+        title: "A query under 'All Bookmarks'",
+      },
+    ],
+  });
+
+  info(
+    "Insert syncable local items (A > B) that exist in non-syncable remote root H"
+  );
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // A is non-syncable remotely, but B doesn't exist remotely, so we'll
+        // remove A from the merged structure, and move B to the menu.
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            title: "B",
+            url: "http://example.com/b",
+          },
+        ],
+      },
+    ],
+  });
+
+  info("Insert non-syncable local root C and items (C > (D > E) F)");
+  await insertLocalRoot({
+    guid: "rootCCCCCCCC",
+    title: "C",
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: "rootCCCCCCCC",
+    children: [
+      {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        children: [
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+        ],
+      },
+      {
+        guid: "bookmarkFFFF",
+        url: "http://example.com/f",
+        title: "F",
+      },
+    ],
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      // H is a non-syncable root that only exists remotely.
+      id: "rootHHHHHHHH",
+      type: "folder",
+      parentid: "places",
+      title: "H",
+      children: ["folderAAAAAA"],
+    },
+    {
+      // A is a folder with children that's non-syncable remotely, and syncable
+      // locally. We should remove A and its descendants locally, since its parent
+      // H is known to be non-syncable remotely.
+      id: "folderAAAAAA",
+      parentid: "rootHHHHHHHH",
+      type: "folder",
+      title: "A",
+      children: ["bookmarkFFFF", "bookmarkIIII"],
+    },
+    {
+      // F exists in two different non-syncable folders: C locally, and A
+      // remotely.
+      id: "bookmarkFFFF",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      title: "F",
+      bmkUri: "http://example.com/f",
+    },
+    {
+      id: "bookmarkIIII",
+      parentid: "folderAAAAAA",
+      type: "query",
+      title: "I",
+      bmkUri: "http://example.com/i",
+    },
+    {
+      // The complete left pane root. We should remove all left pane queries
+      // locally, even though they're syncable, since the left pane root is
+      // known to be non-syncable.
+      id: "folderLEFTPR",
+      type: "folder",
+      parentid: "places",
+      title: "",
+      children: ["folderLEFTPQ", "folderLEFTPF"],
+    },
+    {
+      id: "folderLEFTPQ",
+      parentid: "folderLEFTPR",
+      type: "query",
+      title: "Some query",
+      bmkUri: "place:folder=SOMETHING",
+    },
+    {
+      id: "folderLEFTPF",
+      parentid: "folderLEFTPR",
+      type: "folder",
+      title: "All Bookmarks",
+      children: ["folderLEFTPC"],
+    },
+    {
+      id: "folderLEFTPC",
+      parentid: "folderLEFTPF",
+      type: "query",
+      title: "A query under 'All Bookmarks'",
+      bmkUri: "place:folder=SOMETHING_ELSE",
+    },
+    {
+      // D, J, and G are syncable remotely, but D is non-syncable locally. Since
+      // J and G don't exist locally, and are syncable remotely, we'll remove D
+      // from the merged structure, and move J and G to unfiled.
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: ["folderDDDDDD", "bookmarkGGGG"],
+    },
+    {
+      id: "folderDDDDDD",
+      parentid: "unfiled",
+      type: "folder",
+      title: "D",
+      children: ["bookmarkJJJJ"],
+    },
+    {
+      id: "bookmarkJJJJ",
+      parentid: "folderDDDDDD",
+      type: "bookmark",
+      title: "J",
+      bmkUri: "http://example.com/j",
+    },
+    {
+      id: "bookmarkGGGG",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "G",
+      bmkUri: "http://example.com/g",
+    },
+  ]);
+
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      "bookmarkFFFF",
+      "bookmarkIIII",
+      "bookmarkJJJJ",
+      "folderAAAAAA",
+      "folderDDDDDD",
+      "folderLEFTPC",
+      "folderLEFTPF",
+      "folderLEFTPQ",
+      "folderLEFTPR",
+      PlacesUtils.bookmarks.menuGuid,
+      "rootHHHHHHHH",
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave non-syncable items and roots with new remote structure unmerged"
+  );
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    "bookmarkBBBB",
+    "bookmarkJJJJ",
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      folderAAAAAA: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderAAAAAA",
+          deleted: true,
+        },
+      },
+      folderDDDDDD: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderDDDDDD",
+          deleted: true,
+        },
+      },
+      folderLEFTPQ: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderLEFTPQ",
+          deleted: true,
+        },
+      },
+      folderLEFTPC: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderLEFTPC",
+          deleted: true,
+        },
+      },
+      folderLEFTPR: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderLEFTPR",
+          deleted: true,
+        },
+      },
+      folderLEFTPF: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderLEFTPF",
+          deleted: true,
+        },
+      },
+      rootHHHHHHHH: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "rootHHHHHHHH",
+          deleted: true,
+        },
+      },
+      bookmarkFFFF: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkFFFF",
+          deleted: true,
+        },
+      },
+      bookmarkIIII: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkIIII",
+          deleted: true,
+        },
+      },
+      bookmarkBBBB: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkBBBB",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: datesAdded.get("bookmarkBBBB"),
+          bmkUri: "http://example.com/b",
+          title: "B",
+        },
+      },
+      bookmarkJJJJ: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkJJJJ",
+          type: "bookmark",
+          parentid: "unfiled",
+          hasDupe: true,
+          parentName: UnfiledBookmarksTitle,
+          dateAdded: undefined,
+          bmkUri: "http://example.com/j",
+          title: "J",
+        },
+      },
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          title: BookmarksMenuTitle,
+          children: ["bookmarkBBBB"],
+        },
+      },
+      unfiled: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "unfiled",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+          title: UnfiledBookmarksTitle,
+          children: ["bookmarkJJJJ", "bookmarkGGGG"],
+        },
+      },
+    },
+    "Should upload new structure and tombstones for non-syncable items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkJJJJ",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "J",
+              url: "http://example.com/j",
+            },
+            {
+              guid: "bookmarkGGGG",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "G",
+              url: "http://example.com/g",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should exclude non-syncable items from new local structure"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    [
+      "bookmarkFFFF",
+      "bookmarkIIII",
+      "folderAAAAAA",
+      "folderDDDDDD",
+      "folderLEFTPC",
+      "folderLEFTPF",
+      "folderLEFTPQ",
+      "folderLEFTPR",
+      "rootHHHHHHHH",
+    ],
+    "Should store local tombstones for non-syncable items"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// See what happens when a left-pane root and a left-pane query are on the server
+add_task(async function test_left_pane_root() {
+  let buf = await openMirror("lpr");
+
+  let initialTree = await fetchLocalTree(PlacesUtils.bookmarks.rootGuid);
+
+  // This test is expected to not touch bookmarks at all, and if it did
+  // happen to create a new item that's not under our syncable roots, then
+  // just checking the result of fetchLocalTree wouldn't pick that up - so
+  // as an additional safety check, count how many bookmark rows exist.
+  let numRows = await getCountOfBookmarkRows(buf.db);
+
+  // Add a left pane root, a left-pane query and a left-pane folder to the
+  // mirror, all correctly parented.
+  // Because we can determine this is a complete tree that's outside our
+  // syncable trees, we expect none of them to be applied.
+  await storeRecords(
+    buf,
+    shuffle(
+      [
+        {
+          id: "folderLEFTPR",
+          type: "folder",
+          parentid: "places",
+          title: "",
+          children: ["folderLEFTPQ", "folderLEFTPF"],
+        },
+        {
+          id: "folderLEFTPQ",
+          type: "query",
+          parentid: "folderLEFTPR",
+          title: "Some query",
+          bmkUri: "place:folder=SOMETHING",
+        },
+        {
+          id: "folderLEFTPF",
+          type: "folder",
+          parentid: "folderLEFTPR",
+          title: "All Bookmarks",
+          children: ["folderLEFTPC"],
+        },
+        {
+          id: "folderLEFTPC",
+          type: "query",
+          parentid: "folderLEFTPF",
+          title: "A query under 'All Bookmarks'",
+          bmkUri: "place:folder=SOMETHING_ELSE",
+        },
+      ],
+      { needsMerge: true }
+    )
+  );
+
+  await buf.apply();
+
+  // should have ignored everything.
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, initialTree);
+
+  // and a check we didn't write *any* items to the places database, even
+  // outside of our user roots.
+  Assert.equal(await getCountOfBookmarkRows(buf.db), numRows);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// See what happens when a left-pane query (without the left-pane root) is on
+// the server
+add_task(async function test_left_pane_query() {
+  let buf = await openMirror("lpq");
+
+  let initialTree = await fetchLocalTree(PlacesUtils.bookmarks.rootGuid);
+
+  // This test is expected to not touch bookmarks at all, and if it did
+  // happen to create a new item that's not under our syncable roots, then
+  // just checking the result of fetchLocalTree wouldn't pick that up - so
+  // as an additional safety check, count how many bookmark rows exist.
+  let numRows = await getCountOfBookmarkRows(buf.db);
+
+  // Add the left pane root and left-pane folders to the mirror, correctly parented.
+  // We should not apply it because we made a policy decision to not apply
+  // orphaned queries (bug 1433182)
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "folderLEFTPQ",
+        type: "query",
+        parentid: "folderLEFTPR",
+        title: "Some query",
+        bmkUri: "place:folder=SOMETHING",
+      },
+    ],
+    { needsMerge: true }
+  );
+
+  await buf.apply();
+
+  // should have ignored everything.
+  await assertLocalTree(PlacesUtils.bookmarks.rootGuid, initialTree);
+
+  // and further check we didn't apply it as mis-rooted.
+  Assert.equal(await getCountOfBookmarkRows(buf.db), numRows);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_partial_cycle() {
+  let buf = await openMirror("partial_cycle");
+
+  info("Set up mirror: Menu > A > B > C");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "folderBBBBBB",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "B",
+            children: [
+              {
+                guid: "bookmarkCCCC",
+                url: "http://example.com/c",
+                title: "C",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["folderBBBBBB"],
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "folderAAAAAA",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Try to create a cycle: move A into B, and B into the menu, but don't upload
+  // a record for the menu.
+  info("Make remote changes: A > C");
+  await storeRecords(buf, [
+    {
+      id: "folderAAAAAA",
+      parentid: "menu",
+      type: "folder",
+      title: "A (remote)",
+      children: ["bookmarkCCCC"],
+    },
+    {
+      id: "folderBBBBBB",
+      parentid: "folderAAAAAA",
+      type: "folder",
+      title: "B (remote)",
+      children: ["folderAAAAAA"],
+    },
+  ]);
+
+  await Assert.rejects(
+    buf.apply(),
+    /Item  can't contain itself/,
+    "Should abort merge if remote tree parents form `parentid` cycle"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_complete_cycle() {
+  let buf = await openMirror("complete_cycle");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // This test is order-dependent. We shouldn't recurse infinitely, but,
+  // depending on the order of the records, we might ignore the circular
+  // subtree because there's nothing linking it back to the rest of the
+  // tree.
+  info("Make remote changes: Menu > A > B > C > A");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["folderAAAAAA"],
+    },
+    {
+      id: "folderAAAAAA",
+      parentid: "menu",
+      type: "folder",
+      title: "A",
+      children: ["folderBBBBBB"],
+    },
+    {
+      id: "folderBBBBBB",
+      parentid: "folderAAAAAA",
+      type: "folder",
+      title: "B",
+      children: ["folderCCCCCC"],
+    },
+    {
+      id: "folderCCCCCC",
+      parentid: "folderBBBBBB",
+      type: "folder",
+      title: "C",
+      children: ["folderDDDDDD"],
+    },
+    {
+      id: "folderDDDDDD",
+      parentid: "folderCCCCCC",
+      type: "folder",
+      title: "D",
+      children: ["folderAAAAAA"],
+    },
+  ]);
+
+  await Assert.rejects(
+    buf.apply(),
+    /Item  can't contain itself/,
+    "Should abort merge if remote tree parents form cycle through `children`"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_invalid_guid() {
+  let now = new Date();
+
+  let buf = await openMirror("invalid_guid");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA", "bad!guid~", "bookmarkBBBB"],
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+    {
+      id: "bad!guid~",
+      parentid: "menu",
+      type: "bookmark",
+      title: "C",
+      bmkUri: "http://example.com/c",
+      dateAdded: now.getTime(),
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+  ]);
+
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bad!guid~", PlacesUtils.bookmarks.menuGuid],
+    "Should leave bad GUID and menu with new remote structure unmerged"
+  );
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+
+  let recordIdsToUpload = Object.keys(changesToUpload);
+  let newGuid = recordIdsToUpload.find(
+    recordId => !["bad!guid~", "menu"].includes(recordId)
+  );
+
+  equal(
+    recordIdsToUpload.length,
+    3,
+    "Should reupload menu, C, and tombstone for bad GUID"
+  );
+
+  deepEqual(
+    changesToUpload["bad!guid~"],
+    {
+      tombstone: true,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bad!guid~",
+        deleted: true,
+      },
+    },
+    "Should upload tombstone for C's invalid GUID"
+  );
+
+  deepEqual(
+    changesToUpload[newGuid],
+    {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: newGuid,
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: now.getTime(),
+        bmkUri: "http://example.com/c",
+        title: "C",
+      },
+    },
+    "Should reupload C with new GUID"
+  );
+
+  deepEqual(
+    changesToUpload.menu,
+    {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "menu",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+        title: BookmarksMenuTitle,
+        children: ["bookmarkAAAA", newGuid, "bookmarkBBBB"],
+      },
+    },
+    "Should reupload menu with new child GUID for C"
+  );
+
+  await assertLocalTree(PlacesUtils.bookmarks.menuGuid, {
+    guid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    index: 0,
+    title: BookmarksMenuTitle,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 0,
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: newGuid,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 1,
+        title: "C",
+        url: "http://example.com/c",
+      },
+      {
+        guid: "bookmarkBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        index: 2,
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["bad!guid~"],
+    "Should store local tombstone for C's invalid GUID"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_sync_status_mismatches() {
+  let dateAdded = new Date();
+
+  let buf = await openMirror("sync_status_mismatches");
+
+  info("Ensure mirror is up-to-date with Places");
+  let initialChangesToUpload = await buf.apply();
+
+  deepEqual(
+    Object.keys(initialChangesToUpload).sort(),
+    ["menu", "mobile", "toolbar", "unfiled"],
+    "Should upload roots on first merge"
+  );
+
+  await storeChangesInMirror(buf, initialChangesToUpload);
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        // A is NORMAL in Places, but doesn't exist in the mirror.
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+        dateAdded,
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        // B is NEW in Places and exists in the mirror.
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+        dateAdded,
+      },
+    ],
+  });
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkBBBB"],
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkCCCC"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "unfiled",
+        type: "bookmark",
+        bmkUri: "http://example.com/b",
+        title: "B",
+      },
+      {
+        // C is flagged as merged in the mirror, but doesn't exist in Places.
+        id: "bookmarkCCCC",
+        parentid: "toolbar",
+        type: "bookmark",
+        bmkUri: "http://example.com/c",
+        title: "C",
+      },
+    ],
+    { needsMerge: false }
+  );
+
+  info("Apply mirror");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      bookmarkAAAA: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAAA",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: dateAdded.getTime(),
+          bmkUri: "http://example.com/a",
+          title: "A",
+        },
+      },
+      bookmarkBBBB: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkBBBB",
+          type: "bookmark",
+          parentid: "unfiled",
+          hasDupe: true,
+          parentName: UnfiledBookmarksTitle,
+          dateAdded: dateAdded.getTime(),
+          bmkUri: "http://example.com/b",
+          title: "B",
+        },
+      },
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          title: BookmarksMenuTitle,
+          children: ["bookmarkAAAA"],
+        },
+      },
+      unfiled: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "unfiled",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+          title: UnfiledBookmarksTitle,
+          children: ["bookmarkBBBB"],
+        },
+      },
+    },
+    "Should flag (A B) and their parents for upload"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should parent C correctly"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_invalid_local_urls() {
+  let buf = await openMirror("invalid_local_urls");
+
+  info("Skip uploading local roots on first merge");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Set up local tree");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // A has an invalid URL locally and doesn't exist remotely, so we
+        // should delete it without uploading a tombstone.
+        guid: "bookmarkAAAA",
+        title: "A (local)",
+        url: "http://example.com/a",
+      },
+      {
+        // B has an invalid URL locally and has a valid URL remotely, so
+        // we should replace our local copy with the remote one.
+        guid: "bookmarkBBBB",
+        title: "B (local)",
+        url: "http://example.com/b",
+      },
+      {
+        // C has an invalid URL on both sides, so we should delete it locally
+        // and upload a tombstone.
+        guid: "bookmarkCCCC",
+        title: "A (local)",
+        url: "http://example.com/c",
+      },
+    ],
+  });
+
+  // The public API doesn't let us insert invalid URLs (for good reason!), so
+  // we update them directly in Places.
+  info("Invalidate local URLs");
+  await buf.db.executeTransaction(async function () {
+    const invalidURLs = [
+      {
+        guid: "bookmarkAAAA",
+        invalidURL: "!@#$%",
+      },
+      {
+        guid: "bookmarkBBBB",
+        invalidURL: "^&*(",
+      },
+      {
+        guid: "bookmarkCCCC",
+        invalidURL: ")-+!@",
+      },
+    ];
+    for (let params of invalidURLs) {
+      await buf.db.execute(
+        `UPDATE moz_places SET
+           url = :invalidURL,
+           url_hash = hash(:invalidURL)
+         WHERE id = (SELECT fk FROM moz_bookmarks WHERE guid = :guid)`,
+        params
+      );
+    }
+  });
+
+  info("Set up remote tree");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkBBBB", "bookmarkCCCC", "bookmarkDDDD"],
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B (remote)",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      // C should be marked as `VALIDITY_REPLACE` in the mirror database.
+      id: "bookmarkCCCC",
+      parentid: "menu",
+      type: "bookmark",
+      title: "C (remote)",
+      bmkUri: ")(*&^",
+    },
+    {
+      // D has an invalid URL remotely and doesn't exist locally, so we
+      // should replace it with a tombstone.
+      id: "bookmarkDDDD",
+      parentid: "menu",
+      type: "bookmark",
+      title: "D (remote)",
+      bmkUri: "^%$#@",
+    },
+  ]);
+
+  info("Apply mirror");
+  let changesToUpload = await buf.apply();
+
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          title: BookmarksMenuTitle,
+          children: ["bookmarkBBBB"],
+        },
+      },
+      bookmarkCCCC: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkCCCC",
+          deleted: true,
+        },
+      },
+      bookmarkDDDD: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkDDDD",
+          deleted: true,
+        },
+      },
+    },
+    "Should reupload menu and tombstones for (C D)"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "B (remote)",
+          url: "http://example.com/b",
+        },
+      ],
+    },
+    "Should replace B with remote and delete (A C)"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [],
+    "Should flag all items as merged after upload"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_deduping.js b/toolkit/components/places/tests/sync/test_bookmark_deduping.js
new file mode 100644
index 0000000000..0c6c79496a
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_deduping.js
@@ -0,0 +1,1290 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_duping_local_newer() {
+  let mergeTelemetryCounts;
+  let buf = await openMirror("duping_local_newer", {
+    recordStepTelemetry(name, took, counts) {
+      if (name == "merge") {
+        mergeTelemetryCounts = counts.filter(({ count }) => count > 0);
+      }
+    },
+  });
+  let localModified = new Date();
+
+  info("Start with empty local and mirror with merged items");
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAA5"],
+        dateAdded: localModified.getTime(),
+      },
+      {
+        id: "bookmarkAAA5",
+        parentid: "menu",
+        type: "bookmark",
+        bmkUri: "http://example.com/a",
+        title: "A",
+        dateAdded: localModified.getTime(),
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Add newer local dupes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAA1",
+        title: "A",
+        url: "http://example.com/a",
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+      {
+        guid: "bookmarkAAA2",
+        title: "A",
+        url: "http://example.com/a",
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+      {
+        guid: "bookmarkAAA3",
+        title: "A",
+        url: "http://example.com/a",
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+    ],
+  });
+
+  info("Add older remote dupes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA", "bookmarkAAA4", "bookmarkAAA5"],
+      modified: localModified / 1000 - 5,
+    },
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      bmkUri: "http://example.com/a",
+      title: "A",
+      keyword: "kw",
+      tags: ["remote", "tags"],
+      modified: localModified / 1000 - 5,
+    },
+    {
+      id: "bookmarkAAA4",
+      parentid: "menu",
+      type: "bookmark",
+      bmkUri: "http://example.com/a",
+      title: "A",
+      modified: localModified / 1000 - 5,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: localModified / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkAAA4", "bookmarkAAAA", PlacesUtils.bookmarks.menuGuid],
+    "Should leave A4, A, menu with new remote structure unmerged"
+  );
+  deepEqual(
+    mergeTelemetryCounts,
+    [
+      { name: "items", count: 9 },
+      { name: "dupes", count: 2 },
+    ],
+    "Should record telemetry with dupe counts"
+  );
+
+  let menuInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.menuGuid
+  );
+  deepEqual(
+    changesToUpload,
+    {
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: menuInfo.dateAdded.getTime(),
+          title: menuInfo.title,
+          children: [
+            "bookmarkAAAA",
+            "bookmarkAAA4",
+            "bookmarkAAA3",
+            "bookmarkAAA5",
+          ],
+        },
+      },
+      // Note that we always reupload the deduped local item, because content
+      // matching doesn't account for attributes like keywords, synced annos, or
+      // tags.
+      bookmarkAAAA: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAAA",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: localModified.getTime(),
+          title: "A",
+          bmkUri: "http://example.com/a",
+        },
+      },
+      // Unchanged from local.
+      bookmarkAAA4: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAA4",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: localModified.getTime(),
+          title: "A",
+          bmkUri: "http://example.com/a",
+        },
+      },
+      bookmarkAAA3: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAA3",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: localModified.getTime(),
+          title: "A",
+          bmkUri: "http://example.com/a",
+        },
+      },
+      bookmarkAAA5: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAA5",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: localModified.getTime(),
+          title: "A",
+          bmkUri: "http://example.com/a",
+        },
+      },
+    },
+    "Should uploaded newer deduped local items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A",
+          url: "http://example.com/a",
+        },
+        {
+          guid: "bookmarkAAA4",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "A",
+          url: "http://example.com/a",
+        },
+        {
+          guid: "bookmarkAAA3",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "A",
+          url: "http://example.com/a",
+        },
+        {
+          guid: "bookmarkAAA5",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 3,
+          title: "A",
+          url: "http://example.com/a",
+        },
+      ],
+    },
+    "Should dedupe local multiple bookmarks with similar contents"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_duping_remote_newer() {
+  let buf = await openMirror("duping_remote_new");
+  let localModified = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Shouldn't dupe to `folderA11111` because its sync status is "NORMAL".
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            // Shouldn't dupe to `bookmarkG111`.
+            guid: "bookmarkGGGG",
+            url: "http://example.com/g",
+            title: "G",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkGGGG"],
+      },
+      {
+        id: "bookmarkGGGG",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Insert local dupes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Should dupe to `folderB11111`.
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        dateAdded: localModified,
+        lastModified: localModified,
+        children: [
+          {
+            // Should dupe to `bookmarkC222`.
+            guid: "bookmarkC111",
+            url: "http://example.com/c",
+            title: "C",
+            dateAdded: localModified,
+            lastModified: localModified,
+          },
+          {
+            // Should dupe to `separatorF11` because the positions are the same.
+            guid: "separatorFFF",
+            type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+            dateAdded: localModified,
+            lastModified: localModified,
+          },
+        ],
+      },
+      {
+        // Shouldn't dupe to `separatorE11`, because the positions are different.
+        guid: "separatorEEE",
+        type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+      {
+        // Shouldn't dupe to `bookmarkC222` because the parents are different.
+        guid: "bookmarkCCCC",
+        url: "http://example.com/c",
+        title: "C",
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+      {
+        // Should dupe to `queryD111111`.
+        guid: "queryDDDDDDD",
+        url: "place:maxResults=10&sort=8",
+        title: "Most Visited",
+        dateAdded: localModified,
+        lastModified: localModified,
+      },
+    ],
+  });
+
+  // Make sure we still dedupe this even though it doesn't have SYNC_STATUS.NEW
+  PlacesTestUtils.setBookmarkSyncFields({
+    guid: "folderBBBBBB",
+    syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN,
+  });
+
+  // Not a candidate for `bookmarkH111` because we didn't dupe `folderAAAAAA`.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: "folderAAAAAA",
+    guid: "bookmarkHHHH",
+    url: "http://example.com/h",
+    title: "H",
+    dateAdded: localModified,
+    lastModified: localModified,
+  });
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "folderAAAAAA",
+          "folderB11111",
+          "folderA11111",
+          "separatorE11",
+          "queryD111111",
+        ],
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "folderB11111",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkC222", "separatorF11"],
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "bookmarkC222",
+        parentid: "folderB11111",
+        type: "bookmark",
+        bmkUri: "http://example.com/c",
+        title: "C",
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "separatorF11",
+        parentid: "folderB11111",
+        type: "separator",
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "folderA11111",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkG111"],
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "bookmarkG111",
+        parentid: "folderA11111",
+        type: "bookmark",
+        bmkUri: "http://example.com/g",
+        title: "G",
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "separatorE11",
+        parentid: "menu",
+        type: "separator",
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+      {
+        id: "queryD111111",
+        parentid: "menu",
+        type: "query",
+        bmkUri: "place:maxResults=10&sort=8",
+        title: "Most Visited",
+        dateAdded: localModified.getTime(),
+        modified: localModified / 1000 + 5,
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: localModified / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [
+        "bookmarkCCCC",
+        "bookmarkHHHH",
+        "folderAAAAAA",
+        "menu",
+        "separatorEEE",
+      ],
+      deleted: [],
+    },
+    "Should not upload deduped local records"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkGGGG",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "G",
+                  url: "http://example.com/g",
+                },
+                {
+                  guid: "bookmarkHHHH",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "H",
+                  url: "http://example.com/h",
+                },
+              ],
+            },
+            {
+              guid: "folderB11111",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 1,
+              title: "B",
+              children: [
+                {
+                  guid: "bookmarkC222",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "C",
+                  url: "http://example.com/c",
+                },
+                {
+                  guid: "separatorF11",
+                  type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+                  index: 1,
+                  title: "",
+                },
+              ],
+            },
+            {
+              guid: "folderA11111",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 2,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkG111",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "G",
+                  url: "http://example.com/g",
+                },
+              ],
+            },
+            {
+              guid: "separatorE11",
+              type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+              index: 3,
+              title: "",
+            },
+            {
+              guid: "queryD111111",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 4,
+              title: "Most Visited",
+              url: "place:maxResults=10&sort=8",
+            },
+            {
+              guid: "separatorEEE",
+              type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+              index: 5,
+              title: "",
+            },
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 6,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should dedupe matching NEW bookmarks"
+  );
+
+  ok(
+    (
+      await PlacesTestUtils.fetchBookmarkSyncFields(
+        "menu________",
+        "folderB11111",
+        "bookmarkC222",
+        "separatorF11",
+        "folderA11111",
+        "bookmarkG111",
+        "separatorE11",
+        "queryD111111"
+      )
+    ).every(info => info.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL)
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_duping_both() {
+  let buf = await openMirror("duping_both");
+  let now = Date.now();
+
+  info("Start with empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Add local dupes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // `folderAAAAA1` is older than `folderAAAAAA`, but we should still flag
+        // it for upload because it has a new structure (`bookmarkCCCC`).
+        guid: "folderAAAAA1",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        dateAdded: new Date(now - 10000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            // Shouldn't upload, since `bookmarkBBBB` is newer.
+            guid: "bookmarkBBB1",
+            title: "B",
+            url: "http://example.com/b",
+            dateAdded: new Date(now - 10000),
+            lastModified: new Date(now - 5000),
+          },
+          {
+            // Should upload, since `bookmarkCCCC` doesn't exist on the server and
+            // has no content matches.
+            guid: "bookmarkCCCC",
+            title: "C",
+            url: "http://example.com/c",
+            dateAdded: new Date(now - 10000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        // `folderDDDDD1` should keep complete local structure, but we'll still
+        // flag it for reupload because it's newer than `folderDDDDDD`.
+        guid: "folderDDDDD1",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        dateAdded: new Date(now - 10000),
+        lastModified: new Date(now + 5000),
+        children: [
+          {
+            guid: "bookmarkEEE1",
+            title: "E",
+            url: "http://example.com/e",
+            dateAdded: new Date(now - 10000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        // `folderFFFFF1` should keep complete remote value and structure, so
+        // we shouldn't upload it or its children.
+        guid: "folderFFFFF1",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "F",
+        dateAdded: new Date(now - 10000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            guid: "bookmarkGGG1",
+            title: "G",
+            url: "http://example.com/g",
+            dateAdded: new Date(now - 10000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+    ],
+  });
+
+  info("Add remote dupes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["folderAAAAAA", "folderDDDDDD", "folderFFFFFF"],
+    },
+    {
+      id: "folderAAAAAA",
+      parentid: "menu",
+      type: "folder",
+      title: "A",
+      dateAdded: now - 10000,
+      modified: now / 1000 + 5,
+      children: ["bookmarkBBBB"],
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      bmkUri: "http://example.com/b",
+      title: "B",
+      dateAdded: now - 10000,
+      modified: now / 1000 + 5,
+    },
+    {
+      id: "folderDDDDDD",
+      parentid: "menu",
+      type: "folder",
+      title: "D",
+      dateAdded: now - 10000,
+      modified: now / 1000 - 5,
+      children: ["bookmarkEEEE"],
+    },
+    {
+      id: "bookmarkEEEE",
+      parentid: "folderDDDDDD",
+      type: "bookmark",
+      bmkUri: "http://example.com/e",
+      title: "E",
+      dateAdded: now - 10000,
+      modified: now / 1000 + 5,
+    },
+    {
+      id: "folderFFFFFF",
+      parentid: "menu",
+      type: "folder",
+      title: "F",
+      dateAdded: now - 10000,
+      modified: now / 1000 + 5,
+      children: ["bookmarkGGGG", "bookmarkHHHH"],
+    },
+    {
+      id: "bookmarkGGGG",
+      parentid: "folderFFFFFF",
+      type: "bookmark",
+      bmkUri: "http://example.com/g",
+      title: "G",
+      dateAdded: now - 10000,
+      modified: now / 1000 - 5,
+    },
+    {
+      id: "bookmarkHHHH",
+      parentid: "folderFFFFFF",
+      type: "bookmark",
+      bmkUri: "http://example.com/h",
+      title: "H",
+      dateAdded: now - 10000,
+      modified: now / 1000 + 5,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: now / 1000,
+  });
+
+  let menuInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.menuGuid
+  );
+  deepEqual(
+    changesToUpload,
+    {
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: menuInfo.dateAdded.getTime(),
+          title: menuInfo.title,
+          children: ["folderAAAAAA", "folderDDDDDD", "folderFFFFFF"],
+        },
+      },
+      folderAAAAAA: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderAAAAAA",
+          type: "folder",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: now - 10000,
+          title: "A",
+          children: ["bookmarkBBBB", "bookmarkCCCC"],
+        },
+      },
+      bookmarkCCCC: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkCCCC",
+          type: "bookmark",
+          parentid: "folderAAAAAA",
+          hasDupe: true,
+          parentName: "A",
+          dateAdded: now - 10000,
+          title: "C",
+          bmkUri: "http://example.com/c",
+        },
+      },
+      folderDDDDDD: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderDDDDDD",
+          type: "folder",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: menuInfo.title,
+          dateAdded: now - 10000,
+          title: "D",
+          children: ["bookmarkEEEE"],
+        },
+      },
+    },
+    "Should upload new and newer locally deduped items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A",
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: "folderDDDDDD",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "D",
+          children: [
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "E",
+              url: "http://example.com/e",
+            },
+          ],
+        },
+        {
+          guid: "folderFFFFFF",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 2,
+          title: "F",
+          children: [
+            {
+              guid: "bookmarkGGGG",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "G",
+              url: "http://example.com/g",
+            },
+            {
+              guid: "bookmarkHHHH",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "H",
+              url: "http://example.com/h",
+            },
+          ],
+        },
+      ],
+    },
+    "Should change local GUIDs for mixed older and newer items"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_applying_two_empty_folders_doesnt_smush() {
+  let buf = await openMirror("applying_two_empty_folders_doesnt_smush");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: ["emptyempty01", "emptyempty02"],
+      },
+      {
+        id: "emptyempty01",
+        parentid: "mobile",
+        type: "folder",
+        title: "Empty",
+      },
+      {
+        id: "emptyempty02",
+        parentid: "mobile",
+        type: "folder",
+        title: "Empty",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [],
+      deleted: [],
+    },
+    "Should not upload records for remote-only value changes"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.mobileGuid,
+    {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+      children: [
+        {
+          guid: "emptyempty01",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "Empty",
+        },
+        {
+          guid: "emptyempty02",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "Empty",
+        },
+      ],
+    },
+    "Should not smush 1 and 2"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_applying_two_empty_folders_matches_only_one() {
+  let buf = await openMirror("applying_two_empty_folders_doesnt_smush");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    children: [
+      {
+        guid: "emptyempty02",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "Empty",
+      },
+      {
+        guid: "emptyemptyL0",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "Empty",
+      },
+    ],
+  });
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: ["emptyempty01", "emptyempty02", "emptyempty03"],
+      },
+      {
+        id: "emptyempty01",
+        parentid: "mobile",
+        type: "folder",
+        title: "Empty",
+      },
+      {
+        id: "emptyempty02",
+        parentid: "mobile",
+        type: "folder",
+        title: "Empty",
+      },
+      {
+        id: "emptyempty03",
+        parentid: "mobile",
+        type: "folder",
+        title: "Empty",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.mobileGuid],
+    "Should leave mobile with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["mobile"],
+      deleted: [],
+    },
+    "Should not upload records after applying empty folders"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.mobileGuid,
+    {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "mobile",
+      children: [
+        {
+          guid: "emptyempty01",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "Empty",
+        },
+        {
+          guid: "emptyempty02",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "Empty",
+        },
+        {
+          guid: "emptyempty03",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 2,
+          title: "Empty",
+        },
+      ],
+    },
+    "Should apply 1 and dedupe L0 to 3"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Bug 747699.
+add_task(async function test_duping_mobile_bookmarks() {
+  let buf = await openMirror("duping_mobile_bookmarks");
+
+  info("Set up empty mirror with localized mobile root title");
+  let mobileInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.mobileGuid
+  );
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    title: "Favoritos do celular",
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkAAA1",
+    parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    title: "A",
+    url: "http://example.com/a",
+  });
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "mobile",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.mobileGuid],
+    "Should leave mobile with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["mobile"],
+      deleted: [],
+    },
+    "Should not upload records after applying deduped mobile bookmark"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.mobileGuid,
+    {
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 4,
+      title: "Favoritos do celular",
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A",
+          url: "http://example.com/a",
+        },
+      ],
+    },
+    "Should dedupe A1 to A with different parent title"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  // Restore the original mobile root title.
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.mobileGuid,
+    title: mobileInfo.title,
+  });
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_duping_invalid() {
+  // To check if invalid items are prevented from deduping
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAA1",
+        title: "A",
+        url: "http://example.com/a",
+      },
+    ],
+  });
+
+  let buf = await openMirror("duping_invalid");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAA2"],
+    },
+    {
+      id: "bookmarkAAA2",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A",
+      bmkUri: "http://example.com/a",
+    },
+  ]);
+
+  // Invalidate bookmarkAAA2 so that it does not dedupe to bookmarkAAA1
+  await buf.db.execute(
+    `UPDATE items SET
+       validity = :validity
+     WHERE guid = :guid`,
+    {
+      validity: Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE,
+      guid: "bookmarkAAA2",
+    }
+  );
+
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    changesToUpload.menu.cleartext.children,
+    ["bookmarkAAA1"],
+    "Should upload A1 in menu"
+  );
+  ok(
+    !changesToUpload.bookmarkAAA1.tombstone,
+    "Should not upload tombstone for A1"
+  );
+  ok(changesToUpload.bookmarkAAA2.tombstone, "Should upload tombstone for A2");
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAA1",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A",
+          url: "http://example.com/a",
+        },
+      ],
+    },
+    "No deduping of invalid items"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_deletion.js b/toolkit/components/places/tests/sync/test_bookmark_deletion.js
new file mode 100644
index 0000000000..fd29252e74
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_deletion.js
@@ -0,0 +1,1602 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_complex_orphaning() {
+  let now = Date.now();
+
+  let mergeTelemetryCounts;
+  let buf = await openMirror("complex_orphaning", {
+    recordStepTelemetry(name, took, counts) {
+      if (name == "merge") {
+        mergeTelemetryCounts = counts.filter(({ count }) => count > 0);
+      }
+    },
+  });
+
+  // On iOS, the mirror exists as a separate table. On Desktop, we have a
+  // shadow mirror of synced local bookmarks without new changes.
+  info("Set up mirror: ((Toolbar > A > B) (Menu > G > C > D))");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "folderBBBBBB",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "B",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "toolbar",
+        type: "folder",
+        title: "A",
+        children: ["folderBBBBBB"],
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "folderAAAAAA",
+        type: "folder",
+        title: "B",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderGGGGGG",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "G",
+        children: [
+          {
+            guid: "folderCCCCCC",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "C",
+            children: [
+              {
+                guid: "folderDDDDDD",
+                type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                title: "D",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderGGGGGG"],
+      },
+      {
+        id: "folderGGGGGG",
+        parentid: "menu",
+        type: "folder",
+        title: "G",
+        children: ["folderCCCCCC"],
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "folderGGGGGG",
+        type: "folder",
+        title: "C",
+        children: ["folderDDDDDD"],
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "folderCCCCCC",
+        type: "folder",
+        title: "D",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete D, add B > E");
+  await PlacesUtils.bookmarks.remove("folderDDDDDD");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEEE",
+    parentGuid: "folderBBBBBB",
+    title: "E",
+    url: "http://example.com/e",
+  });
+
+  info("Make remote changes: delete B, add D > F");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "folderBBBBBB",
+        deleted: true,
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "toolbar",
+        type: "folder",
+        title: "A",
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "folderCCCCCC",
+        type: "folder",
+        children: ["bookmarkFFFF"],
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkFFFF", "folderAAAAAA", "folderDDDDDD"],
+    "Should leave deleted D; A and F with new remote structure unmerged"
+  );
+  deepEqual(
+    mergeTelemetryCounts,
+    [
+      { name: "items", count: 10 },
+      { name: "localDeletes", count: 1 },
+      { name: "remoteDeletes", count: 1 },
+    ],
+    "Should record telemetry with structure change counts"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkEEEE", "bookmarkFFFF", "folderAAAAAA", "folderCCCCCC"],
+      deleted: ["folderDDDDDD"],
+    },
+    "Should upload new records for (A > E), (C > F); tombstone for D"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderGGGGGG",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "G",
+              children: [
+                {
+                  guid: "folderCCCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                  index: 0,
+                  title: "C",
+                  children: [
+                    {
+                      // D was deleted, so F moved to C, the closest surviving parent.
+                      guid: "bookmarkFFFF",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 0,
+                      title: "F",
+                      url: "http://example.com/f",
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  // B was deleted, so E moved to A.
+                  guid: "bookmarkEEEE",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "E",
+                  url: "http://example.com/e",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should move orphans to closest surviving parent"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["folderDDDDDD"],
+    "Should store local tombstone for D"
+  );
+  Assert.ok(
+    is_time_ordered(now, tombstones[0].dateRemoved.getTime()),
+    "Tombstone timestamp should be recent"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_locally_modified_remotely_deleted() {
+  let mergeTelemetryCounts;
+  let buf = await openMirror("locally_modified_remotely_deleted", {
+    recordStepTelemetry(name, took, counts) {
+      if (name == "merge") {
+        mergeTelemetryCounts = counts.filter(({ count }) => count > 0);
+      }
+    },
+  });
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            title: "C",
+            url: "http://example.com/c",
+          },
+          {
+            guid: "folderDDDDDD",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "D",
+            children: [
+              {
+                guid: "bookmarkEEEE",
+                title: "E",
+                url: "http://example.com/e",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "folderBBBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC", "folderDDDDDD"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "folderBBBBBB",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkEEEE"],
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: change A; B > ((D > F) G)");
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkAAAA",
+    title: "A (local)",
+    url: "http://example.com/a-local",
+  });
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkFFFF",
+    parentGuid: "folderDDDDDD",
+    title: "F (local)",
+    url: "http://example.com/f-local",
+  });
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkGGGG",
+    parentGuid: "folderBBBBBB",
+    title: "G (local)",
+    url: "http://example.com/g-local",
+  });
+
+  info("Make remote changes: delete A, B");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: [],
+    },
+    {
+      id: "bookmarkAAAA",
+      deleted: true,
+    },
+    {
+      id: "folderBBBBBB",
+      deleted: true,
+    },
+    {
+      id: "bookmarkCCCC",
+      deleted: true,
+    },
+    {
+      id: "folderDDDDDD",
+      deleted: true,
+    },
+    {
+      id: "bookmarkEEEE",
+      deleted: true,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkAAAA", PlacesUtils.bookmarks.menuGuid],
+    "Should leave revived A and menu with new remote structure unmerged"
+  );
+  deepEqual(
+    mergeTelemetryCounts,
+    [
+      { name: "items", count: 8 },
+      { name: "localRevives", count: 1 },
+      { name: "remoteDeletes", count: 2 },
+    ],
+    "Should record telemetry for local item and remote folder deletions"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkAAAA", "bookmarkFFFF", "bookmarkGGGG", "menu"],
+      deleted: [],
+    },
+    "Should upload A, relocated local orphans, and menu"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A (local)",
+          url: "http://example.com/a-local",
+        },
+        {
+          guid: "bookmarkFFFF",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "F (local)",
+          url: "http://example.com/f-local",
+        },
+        {
+          guid: "bookmarkGGGG",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "G (local)",
+          url: "http://example.com/g-local",
+        },
+      ],
+    },
+    "Should restore A and relocate (F G) to menu"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Should not store local tombstones");
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_locally_deleted_remotely_modified() {
+  let now = Date.now();
+
+  let mergeTelemetryCounts;
+  let buf = await openMirror("locally_deleted_remotely_modified", {
+    recordStepTelemetry(name, took, counts) {
+      if (name == "merge") {
+        mergeTelemetryCounts = counts.filter(({ count }) => count > 0);
+      }
+    },
+  });
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            title: "C",
+            url: "http://example.com/c",
+          },
+          {
+            guid: "folderDDDDDD",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "D",
+            children: [
+              {
+                guid: "bookmarkEEEE",
+                title: "E",
+                url: "http://example.com/e",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "folderBBBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC", "folderDDDDDD"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "folderBBBBBB",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkEEEE"],
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: delete A, B");
+  await PlacesUtils.bookmarks.remove("bookmarkAAAA");
+  await PlacesUtils.bookmarks.remove("folderBBBBBB");
+
+  info("Make remote changes: change A; B > ((D > F) G)");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A (remote)",
+      bmkUri: "http://example.com/a-remote",
+    },
+    {
+      id: "folderBBBBBB",
+      parentid: "menu",
+      type: "folder",
+      title: "B (remote)",
+      children: ["bookmarkCCCC", "folderDDDDDD", "bookmarkGGGG"],
+    },
+    {
+      id: "folderDDDDDD",
+      parentid: "folderBBBBBB",
+      type: "folder",
+      title: "D",
+      children: ["bookmarkEEEE", "bookmarkFFFF"],
+    },
+    {
+      id: "bookmarkFFFF",
+      parentid: "folderDDDDDD",
+      type: "bookmark",
+      title: "F (remote)",
+      bmkUri: "http://example.com/f-remote",
+    },
+    {
+      id: "bookmarkGGGG",
+      parentid: "folderBBBBBB",
+      type: "bookmark",
+      title: "G (remote)",
+      bmkUri: "http://example.com/g-remote",
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkFFFF", "bookmarkGGGG", "folderBBBBBB", "folderDDDDDD"],
+    "Should leave deleted B and D; relocated F and G unmerged"
+  );
+  deepEqual(
+    mergeTelemetryCounts,
+    [
+      { name: "items", count: 8 },
+      { name: "remoteRevives", count: 1 },
+      { name: "localDeletes", count: 2 },
+    ],
+    "Should record telemetry for remote item and local folder deletions"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkFFFF", "bookmarkGGGG", "menu"],
+      deleted: ["bookmarkCCCC", "bookmarkEEEE", "folderBBBBBB", "folderDDDDDD"],
+    },
+    "Should upload relocated remote orphans and menu"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A (remote)",
+          url: "http://example.com/a-remote",
+        },
+        {
+          guid: "bookmarkFFFF",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "F (remote)",
+          url: "http://example.com/f-remote",
+        },
+        {
+          guid: "bookmarkGGGG",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "G (remote)",
+          url: "http://example.com/g-remote",
+        },
+      ],
+    },
+    "Should restore A and relocate (F G) to menu"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["bookmarkCCCC", "bookmarkEEEE", "folderBBBBBB", "folderDDDDDD"],
+    "Should store local tombstones for deleted items; remove for undeleted"
+  );
+  Assert.ok(
+    tombstones.every(({ dateRemoved }) =>
+      is_time_ordered(now, dateRemoved.getTime())
+    ),
+    "Local tombstone timestamps should be recent"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_to_new_then_delete() {
+  let buf = await openMirror("move_to_new_then_delete");
+
+  info("Set up mirror: A > B > (C D)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "folderBBBBBB",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "B",
+            children: [
+              {
+                guid: "bookmarkCCCC",
+                url: "http://example.com/c",
+                title: "C",
+              },
+              {
+                guid: "bookmarkDDDD",
+                url: "http://example.com/d",
+                title: "D",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["folderBBBBBB"],
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "folderAAAAAA",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC", "bookmarkDDDD"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: E > A, delete E");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "folderEEEEEE",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "E",
+  });
+  await PlacesUtils.bookmarks.update({
+    guid: "folderAAAAAA",
+    parentGuid: "folderEEEEEE",
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+  });
+  // E isn't synced, so we shouldn't upload a tombstone.
+  await PlacesUtils.bookmarks.remove("folderEEEEEE");
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkCCCC",
+      parentid: "folderBBBBBB",
+      type: "bookmark",
+      title: "C (remote)",
+      bmkUri: "http://example.com/c-remote",
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkCCCC", PlacesUtils.bookmarks.toolbarGuid],
+    "Should leave revived C and toolbar with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkCCCC", "menu", "toolbar"],
+      deleted: ["bookmarkDDDD", "folderAAAAAA", "folderBBBBBB"],
+    },
+    "Should upload records for Menu > C, Toolbar"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C (remote)",
+              url: "http://example.com/c-remote",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should move C to closest surviving parent"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["bookmarkDDDD", "folderAAAAAA", "folderBBBBBB"],
+    "Should store local tombstones for (D A B)"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_nonexistent_on_one_side() {
+  let buf = await openMirror("nonexistent_on_one_side");
+
+  info("Set up empty mirror");
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // A doesn't exist in the mirror.
+  info("Create local tombstone for nonexistent remote item A");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkAAAA",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "A",
+    url: "http://example.com/a",
+    // Pretend a bookmark restore added A, so that we'll write a tombstone when
+    // we remove it.
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE,
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkAAAA");
+
+  // B doesn't exist in Places, and we don't currently persist tombstones (bug
+  // 1343103), so we should ignore it.
+  info("Create remote tombstone for nonexistent local item B");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkBBBB",
+      deleted: true,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged"
+  );
+
+  let menuInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.menuGuid
+  );
+
+  // We should still upload a record for the menu, since we changed its
+  // children when we added then removed A.
+  deepEqual(changesToUpload, {
+    menu: {
+      tombstone: false,
+      counter: 2,
+      synced: false,
+      cleartext: {
+        id: "menu",
+        type: "folder",
+        parentid: "places",
+        hasDupe: true,
+        parentName: "",
+        dateAdded: menuInfo.dateAdded.getTime(),
+        title: BookmarksMenuTitle,
+        children: [],
+      },
+    },
+  });
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_clear_folder_then_delete() {
+  let buf = await openMirror("clear_folder_then_delete");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+          },
+        ],
+      },
+      {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        children: [
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+          {
+            guid: "bookmarkFFFF",
+            url: "http://example.com/f",
+            title: "F",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "folderDDDDDD"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB", "bookmarkCCCC"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkEEEE", "bookmarkFFFF"],
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes: Menu > E, Mobile > F, delete D");
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkEEEE",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    index: 0,
+  });
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkFFFF",
+    parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    index: 0,
+  });
+  await PlacesUtils.bookmarks.remove("folderDDDDDD");
+
+  info("Make remote changes: Menu > B, Unfiled > C, delete A");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkBBBB", "folderDDDDDD"],
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkCCCC"],
+    },
+    {
+      id: "bookmarkCCCC",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "C",
+      bmkUri: "http://example.com/c",
+    },
+    {
+      id: "folderAAAAAA",
+      deleted: true,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid, PlacesUtils.bookmarks.mobileGuid],
+    "Should leave menu and mobile with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkEEEE", "bookmarkFFFF", "menu", "mobile"],
+      deleted: ["folderDDDDDD"],
+    },
+    "Should upload locally moved and deleted items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "E",
+              url: "http://example.com/e",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "F",
+              url: "http://example.com/f",
+            },
+          ],
+        },
+      ],
+    },
+    "Should not orphan moved children of a deleted folder"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["folderDDDDDD"],
+    "Should store local tombstone for D"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_newer_move_to_deleted() {
+  let buf = await openMirror("test_newer_move_to_deleted");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+        ],
+      },
+      {
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "C",
+        children: [
+          {
+            guid: "bookmarkDDDD",
+            url: "http://example.com/d",
+            title: "D",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "folderCCCCCC"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "menu",
+        type: "folder",
+        title: "C",
+        children: ["bookmarkDDDD"],
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderCCCCCC",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let now = Date.now();
+
+  // A will have a newer local timestamp. However, we should *not* revert
+  // remotely moving B to the toolbar. (Locally, B exists in A, but we
+  // deleted the now-empty A remotely).
+  info("Make local changes: A > E, Toolbar > D, delete C");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEEE",
+    parentGuid: "folderAAAAAA",
+    title: "E",
+    url: "http://example.com/e",
+    dateAdded: new Date(now),
+    lastModified: new Date(now),
+  });
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkDDDD",
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    index: 0,
+    lastModified: new Date(now),
+  });
+  await PlacesUtils.bookmarks.remove("folderCCCCCC");
+
+  // C will have a newer remote timestamp. However, we should *not* revert
+  // locally moving D to the toolbar. (Locally, D exists in C, but we
+  // deleted the now-empty C locally).
+  info("Make remote changes: C > F, Toolbar > B, delete A");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["folderCCCCCC"],
+    },
+    {
+      id: "folderCCCCCC",
+      parentid: "menu",
+      type: "folder",
+      title: "C",
+      children: ["bookmarkDDDD", "bookmarkFFFF"],
+      modified: now / 1000 + 5,
+    },
+    {
+      id: "bookmarkFFFF",
+      parentid: "folderCCCCCC",
+      type: "bookmark",
+      title: "F",
+      bmkUri: "http://example.com/f",
+    },
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkBBBB"],
+      modified: now / 1000 - 5,
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "toolbar",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+      modified: now / 1000 - 5,
+    },
+    {
+      id: "folderAAAAAA",
+      deleted: true,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    localTimeSeconds: now / 1000,
+    remoteTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      "bookmarkFFFF",
+      "folderCCCCCC",
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+    ],
+    "Should leave deleted C; revived F and roots with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [
+        "bookmarkDDDD",
+        "bookmarkEEEE",
+        "bookmarkFFFF",
+        "menu",
+        "toolbar",
+      ],
+      deleted: ["folderCCCCCC"],
+    },
+    "Should upload new and moved items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "E",
+              url: "http://example.com/e",
+            },
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "F",
+              url: "http://example.com/f",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkDDDD",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "D",
+              url: "http://example.com/d",
+            },
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should not decide to keep newly moved items in deleted parents"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["folderCCCCCC"],
+    "Should store local tombstone for C"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_remotely_deleted_also_removes_keyword() {
+  let buf = await openMirror("remotely_deleted_removes_keyword");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+        keyword: "keyworda",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+        keyword: "keywordb",
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        keyword: "keyworda",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        keyword: "keywordb",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Validate the keywords exists
+  let has_keyword_a = await PlacesUtils.keywords.fetch({
+    url: "http://example.com/a",
+  });
+  Assert.equal(has_keyword_a.keyword, "keyworda");
+
+  let has_keyword_b = await PlacesUtils.keywords.fetch({
+    url: "http://example.com/b",
+  });
+  Assert.equal(has_keyword_b.keyword, "keywordb");
+
+  info("Make remote changes: delete A & B");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: [],
+    },
+    {
+      id: "bookmarkAAAA",
+      deleted: true,
+    },
+    {
+      id: "bookmarkBBBB",
+      deleted: true,
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [],
+      deleted: [],
+    },
+    "No local changes done"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+    },
+    "Should've remove A & B from menu"
+  );
+
+  // Validate the keyword no longer exists after removing the bookmark
+  let no_keyword_a = await PlacesUtils.keywords.fetch({
+    url: "http://example.com/a",
+  });
+  Assert.equal(no_keyword_a, null);
+
+  // Both keywords should've been removed after the sync
+  let no_keyword_b = await PlacesUtils.keywords.fetch({
+    url: "http://example.com/b",
+  });
+  Assert.equal(no_keyword_b, null);
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Should not store local tombstones");
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_haschanges.js b/toolkit/components/places/tests/sync/test_bookmark_haschanges.js
new file mode 100644
index 0000000000..32cfd050aa
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_haschanges.js
@@ -0,0 +1,228 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_no_changes() {
+  let buf = await openMirror("nochanges");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "mozBmk______",
+        url: "https://mozilla.org",
+        title: "Mozilla",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["mozBmk______"],
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: [],
+      },
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: [],
+      },
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: [],
+      },
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let controller = new AbortController();
+  const wasMerged = await buf.merge(controller.signal);
+  Assert.ok(!wasMerged);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_changes_remote() {
+  let buf = await openMirror("remote_changes");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "mozBmk______",
+        url: "https://mozilla.org",
+        title: "Mozilla",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["mozBmk______"],
+      },
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "New Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+    { needsMerge: true }
+  );
+
+  let controller = new AbortController();
+  const wasMerged = await buf.merge(controller.signal);
+  Assert.ok(wasMerged);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_changes_local() {
+  let buf = await openMirror("local_changes");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "mozBmk______",
+        url: "https://mozilla.org",
+        title: "Mozilla",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["mozBmk______"],
+      },
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await PlacesUtils.bookmarks.update({
+    guid: "mozBmk______",
+    title: "New Mozilla!",
+  });
+
+  let controller = new AbortController();
+  const wasMerged = await buf.merge(controller.signal);
+  Assert.ok(wasMerged);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_changes_deleted_bookmark() {
+  let buf = await openMirror("delete_bookmark");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "mozBmk______",
+        url: "https://mozilla.org",
+        title: "Mozilla",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["mozBmk______"],
+      },
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let wait = PlacesTestUtils.waitForNotification("bookmark-removed", events =>
+    events.some(event => event.parentGuid == PlacesUtils.bookmarks.tagsGuid)
+  );
+  await PlacesUtils.bookmarks.remove("mozBmk______");
+
+  await wait;
+  // Wait for everything to be finished
+  await new Promise(resolve => Services.tm.dispatchToMainThread(resolve));
+  let controller = new AbortController();
+  const wasMerged = await buf.merge(controller.signal);
+  Assert.ok(wasMerged);
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_kinds.js b/toolkit/components/places/tests/sync/test_bookmark_kinds.js
new file mode 100644
index 0000000000..3372757532
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_kinds.js
@@ -0,0 +1,312 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_queries() {
+  let buf = await openMirror("queries");
+
+  info("Set up places");
+
+  // create a tag and grab the local folder ID.
+  let tag = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.tagsGuid,
+    title: "a-tag",
+  });
+
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // this entry has a tag= query param for a tag that exists.
+        guid: "queryAAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        title: "TAG_QUERY query",
+        url: `place:tag=a-tag&&sort=14&maxResults=10`,
+      },
+      {
+        // this entry has a tag= query param for a tag that doesn't exist.
+        guid: "queryBBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        title: "TAG_QUERY query but invalid folder id",
+        url: `place:tag=b-tag&sort=14&maxResults=10`,
+      },
+      {
+        // this entry has no tag= query param.
+        guid: "queryCCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        title: "TAG_QUERY without a folder at all",
+        url: "place:sort=14&maxResults=10",
+      },
+      {
+        // this entry has only a tag= query.
+        guid: "queryDDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        title: "TAG_QUERY without a folder at all",
+        url: "place:tag=a-tag",
+      },
+    ],
+  });
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "queryEEEEEEE",
+          "queryFFFFFFF",
+          "queryGGGGGGG",
+          "queryHHHHHHH",
+          "queryIIIIIII",
+        ],
+      },
+      {
+        // Legacy tag query.
+        id: "queryEEEEEEE",
+        parentid: "toolbar",
+        type: "query",
+        title: "E",
+        bmkUri: "place:type=7&folder=999",
+        folderName: "taggy",
+      },
+      {
+        // New tag query.
+        id: "queryFFFFFFF",
+        parentid: "toolbar",
+        type: "query",
+        title: "F",
+        bmkUri: "place:tag=a-tag",
+        folderName: "a-tag",
+      },
+      {
+        // Legacy tag query referencing the same tag as the new query.
+        id: "queryGGGGGGG",
+        parentid: "toolbar",
+        type: "query",
+        title: "G",
+        bmkUri: "place:type=7&folder=111&something=else",
+        folderName: "a-tag",
+      },
+      {
+        // Legacy folder lookup query.
+        id: "queryHHHHHHH",
+        parentid: "toolbar",
+        type: "query",
+        title: "H",
+        bmkUri: "place:folder=1",
+      },
+      {
+        // Legacy tag query with invalid tag folder name.
+        id: "queryIIIIIII",
+        parentid: "toolbar",
+        type: "query",
+        title: "I",
+        bmkUri: "place:type=7&folder=222",
+        folderName: "    ",
+      },
+    ])
+  );
+
+  info("Create records to upload");
+  let changes = await buf.apply();
+  deepEqual(
+    Object.keys(changes),
+    [
+      "menu",
+      "toolbar",
+      "queryAAAAAAA",
+      "queryBBBBBBB",
+      "queryCCCCCCC",
+      "queryDDDDDDD",
+      "queryEEEEEEE",
+      "queryGGGGGGG",
+      "queryHHHHHHH",
+      "queryIIIIIII",
+    ],
+    "Should upload roots, new queries, and rewritten queries"
+  );
+  Assert.strictEqual(changes.queryAAAAAAA.cleartext.folderName, tag.title);
+  Assert.strictEqual(changes.queryBBBBBBB.cleartext.folderName, "b-tag");
+  Assert.strictEqual(changes.queryCCCCCCC.cleartext.folderName, undefined);
+  Assert.strictEqual(changes.queryDDDDDDD.cleartext.folderName, tag.title);
+  Assert.strictEqual(changes.queryIIIIIII.tombstone, true);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.toolbarGuid,
+    {
+      guid: PlacesUtils.bookmarks.toolbarGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 1,
+      title: BookmarksToolbarTitle,
+      children: [
+        {
+          guid: "queryEEEEEEE",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "E",
+          url: "place:tag=taggy",
+        },
+        {
+          guid: "queryFFFFFFF",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "F",
+          url: "place:tag=a-tag",
+        },
+        {
+          guid: "queryGGGGGGG",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "G",
+          url: "place:tag=a-tag",
+        },
+        {
+          guid: "queryHHHHHHH",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 3,
+          title: "H",
+          url: "place:folder=1&excludeItems=1",
+        },
+      ],
+    },
+    "Should rewrite legacy remote queries"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_different_but_compatible_bookmark_types() {
+  let buf = await openMirror("partial_queries");
+  try {
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.menuGuid,
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          title: "not yet a query",
+          url: "about:blank",
+        },
+        {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          title: "a query",
+          url: "place:foo",
+        },
+      ],
+    });
+
+    let changes = await buf.apply();
+    // We should have an outgoing record for bookmarkA with type=bookmark
+    // and bookmarkB with type=query.
+    Assert.equal(changes.bookmarkAAAA.cleartext.type, "bookmark");
+    Assert.equal(changes.bookmarkBBBB.cleartext.type, "query");
+
+    // Now pretend that same records are already on the server.
+    await storeRecords(
+      buf,
+      [
+        {
+          id: "menu",
+          parentid: "places",
+          type: "folder",
+          children: ["bookmarkAAAA", "bookmarkBBBB"],
+        },
+        {
+          id: "bookmarkAAAA",
+          parentid: "menu",
+          type: "bookmark",
+          title: "not yet a query",
+          bmkUri: "about:blank",
+        },
+        {
+          id: "bookmarkBBBB",
+          parentid: "menu",
+          type: "query",
+          title: "a query",
+          bmkUri: "place:foo",
+        },
+      ],
+      { needsMerge: false }
+    );
+    await PlacesTestUtils.markBookmarksAsSynced();
+
+    // change the url of bookmarkA to be a "real" query and of bookmarkB to
+    // no longer be a query.
+    await PlacesUtils.bookmarks.update({
+      guid: "bookmarkAAAA",
+      url: "place:type=6&sort=14&maxResults=10",
+    });
+    await PlacesUtils.bookmarks.update({
+      guid: "bookmarkBBBB",
+      url: "about:robots",
+    });
+
+    changes = await buf.apply();
+    // We should have an outgoing record for bookmarkA with type=query and
+    // for bookmarkB with type=bookmark
+    Assert.equal(changes.bookmarkAAAA.cleartext.type, "query");
+    Assert.equal(changes.bookmarkBBBB.cleartext.type, "bookmark");
+  } finally {
+    await buf.finalize();
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesSyncUtils.bookmarks.reset();
+  }
+});
+
+add_task(async function test_incompatible_types() {
+  try {
+    let buf = await openMirror("incompatible_types");
+
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.menuGuid,
+      children: [
+        {
+          guid: "AAAAAAAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          title: "a bookmark",
+          url: "about:blank",
+        },
+      ],
+    });
+
+    await buf.apply();
+
+    // Now pretend that same records are already on the server with incompatible
+    // types.
+    await storeRecords(
+      buf,
+      [
+        {
+          id: "menu",
+          parentid: "places",
+          type: "folder",
+          children: ["AAAAAAAAAAAA"],
+        },
+        {
+          id: "AAAAAAAAAAAA",
+          parentid: "menu",
+          type: "folder",
+          title: "conflicting folder",
+        },
+      ],
+      { needsMerge: true }
+    );
+    await PlacesTestUtils.markBookmarksAsSynced();
+
+    await Assert.rejects(
+      buf.apply(),
+      /Can't merge local Bookmark  and remote Folder /
+    );
+  } finally {
+    await PlacesUtils.bookmarks.eraseEverything();
+    await PlacesSyncUtils.bookmarks.reset();
+  }
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_mirror_meta.js b/toolkit/components/places/tests/sync/test_bookmark_mirror_meta.js
new file mode 100644
index 0000000000..6c475daab6
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_mirror_meta.js
@@ -0,0 +1,193 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_highWaterMark() {
+  let buf = await openMirror("highWaterMark");
+
+  strictEqual(
+    await buf.getCollectionHighWaterMark(),
+    0,
+    "High water mark should be 0 without items"
+  );
+
+  await buf.setCollectionLastModified(123.45);
+  equal(
+    await buf.getCollectionHighWaterMark(),
+    123.45,
+    "High water mark should be last modified time without items"
+  );
+
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: [],
+      modified: 50,
+    },
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: [],
+      modified: 123.95,
+    },
+  ]);
+  equal(
+    await buf.getCollectionHighWaterMark(),
+    123.45,
+    "High water mark should be last modified time if items are older"
+  );
+
+  await storeRecords(buf, [
+    {
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: [],
+      modified: 125.45,
+    },
+  ]);
+  equal(
+    await buf.getCollectionHighWaterMark(),
+    124.45,
+    "High water mark should be modified time - 1s of newest record if exists"
+  );
+
+  await buf.finalize();
+});
+
+add_task(async function test_ensureCurrentSyncId() {
+  let buf = await openMirror("ensureCurrentSyncId");
+
+  await buf.ensureCurrentSyncId("syncIdAAAAAA");
+  equal(
+    await buf.getCollectionHighWaterMark(),
+    0,
+    "High water mark should be 0 after setting sync ID"
+  );
+
+  info("Insert items and set collection last modified");
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+        modified: 125.45,
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        children: [],
+      },
+    ],
+    { needsMerge: false }
+  );
+  await buf.setCollectionLastModified(123.45);
+
+  info("Set matching sync ID");
+  await buf.ensureCurrentSyncId("syncIdAAAAAA");
+  {
+    equal(
+      await buf.getSyncId(),
+      "syncIdAAAAAA",
+      "Should return existing sync ID"
+    );
+    strictEqual(
+      await buf.getCollectionHighWaterMark(),
+      124.45,
+      "Different sync ID should reset high water mark"
+    );
+
+    let itemRows = await buf.db.execute(`
+      SELECT guid, needsMerge FROM items
+      ORDER BY guid`);
+    let itemInfos = itemRows.map(row => ({
+      guid: row.getResultByName("guid"),
+      needsMerge: !!row.getResultByName("needsMerge"),
+    }));
+    deepEqual(
+      itemInfos,
+      [
+        {
+          guid: "folderAAAAAA",
+          needsMerge: false,
+        },
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          needsMerge: false,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          needsMerge: true,
+        },
+        {
+          guid: PlacesUtils.bookmarks.rootGuid,
+          needsMerge: false,
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          needsMerge: true,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          needsMerge: true,
+        },
+      ],
+      "Matching sync ID should not reset items"
+    );
+  }
+
+  info("Set different sync ID");
+  await buf.ensureCurrentSyncId("syncIdBBBBBB");
+  {
+    equal(
+      await buf.getSyncId(),
+      "syncIdBBBBBB",
+      "Should replace existing sync ID"
+    );
+    strictEqual(
+      await buf.getCollectionHighWaterMark(),
+      0,
+      "Different sync ID should reset high water mark"
+    );
+
+    let itemRows = await buf.db.execute(`
+      SELECT guid, needsMerge FROM items
+      ORDER BY guid`);
+    let itemInfos = itemRows.map(row => ({
+      guid: row.getResultByName("guid"),
+      needsMerge: !!row.getResultByName("needsMerge"),
+    }));
+    deepEqual(
+      itemInfos,
+      [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          needsMerge: true,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          needsMerge: true,
+        },
+        {
+          guid: PlacesUtils.bookmarks.rootGuid,
+          needsMerge: false,
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          needsMerge: true,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          needsMerge: true,
+        },
+      ],
+      "Different sync ID should reset items"
+    );
+  }
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_mirror_migration.js b/toolkit/components/places/tests/sync/test_bookmark_mirror_migration.js
new file mode 100644
index 0000000000..9951983f78
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_mirror_migration.js
@@ -0,0 +1,228 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Keep in sync with `SyncedBookmarksMirror.jsm`.
+const CURRENT_MIRROR_SCHEMA_VERSION = 8;
+
+// The oldest schema version that we support. Any databases with schemas older
+// than this will be dropped and recreated.
+const OLDEST_SUPPORTED_MIRROR_SCHEMA_VERSION = 5;
+
+async function getIndexNames(db, table, schema = "mirror") {
+  let rows = await db.execute(`PRAGMA ${schema}.index_list(${table})`);
+  let names = [];
+  for (let row of rows) {
+    // Column 4 is `c` if the index was created via `CREATE INDEX`, `u` if
+    // via `UNIQUE`, and `pk` if via `PRIMARY KEY`.
+    let wasCreated = row.getResultByIndex(3) == "c";
+    if (wasCreated) {
+      // Column 2 is the name of the index.
+      names.push(row.getResultByIndex(1));
+    }
+  }
+  return names.sort();
+}
+
+add_task(async function test_migrate_after_downgrade() {
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let dbFile = await setupFixtureFile("mirror_v5.sqlite");
+  let oldBuf = await SyncedBookmarksMirror.open({
+    path: dbFile.path,
+    recordStepTelemetry() {},
+    recordValidationTelemetry() {},
+  });
+
+  info("Downgrade schema version to oldest supported");
+  await oldBuf.db.setSchemaVersion(
+    OLDEST_SUPPORTED_MIRROR_SCHEMA_VERSION,
+    "mirror"
+  );
+  await oldBuf.finalize();
+
+  let buf = await SyncedBookmarksMirror.open({
+    path: dbFile.path,
+    recordStepTelemetry() {},
+    recordValidationTelemetry() {},
+  });
+
+  // All migrations between `OLDEST_SUPPORTED_MIRROR_SCHEMA_VERSION` should
+  // be idempotent. When we downgrade, we roll back the schema version, but
+  // leave the schema changes in place, since we can't anticipate what a
+  // future version will change.
+  let schemaVersion = await buf.db.getSchemaVersion("mirror");
+  equal(
+    schemaVersion,
+    CURRENT_MIRROR_SCHEMA_VERSION,
+    "Should upgrade downgraded mirror schema"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Migrations between 5 and 7 add three indexes.
+add_task(async function test_migrate_from_5_to_current() {
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let dbFile = await setupFixtureFile("mirror_v5.sqlite");
+  let buf = await SyncedBookmarksMirror.open({
+    path: dbFile.path,
+    recordStepTelemetry() {},
+    recordValidationTelemetry() {},
+  });
+
+  let schemaVersion = await buf.db.getSchemaVersion("mirror");
+  equal(
+    schemaVersion,
+    CURRENT_MIRROR_SCHEMA_VERSION,
+    "Should upgrade mirror schema to current version"
+  );
+
+  let itemsIndexNames = await getIndexNames(buf.db, "items");
+  deepEqual(
+    itemsIndexNames,
+    ["itemKeywords", "itemURLs"],
+    "Should add two indexes on items"
+  );
+
+  let structureIndexNames = await getIndexNames(buf.db, "structure");
+  deepEqual(
+    structureIndexNames,
+    ["structurePositions"],
+    "Should add an index on structure"
+  );
+
+  let changesToUpload = await buf.apply();
+  deepEqual(changesToUpload, {}, "Shouldn't flag any items for reupload");
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAAA",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A",
+          url: "http://example.com/a",
+        },
+        {
+          guid: "bookmarkBBBB",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "B",
+          url: "http://example.com/b",
+          keyword: "hi",
+        },
+      ],
+    },
+    "Should apply mirror tree after migrating"
+  );
+
+  let keywordEntry = await PlacesUtils.keywords.fetch("hi");
+  equal(
+    keywordEntry.url.href,
+    "http://example.com/b",
+    "Should apply keyword from migrated mirror"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Migrations between 1 and 2 discard the entire database.
+add_task(async function test_migrate_from_1_to_2() {
+  let dbFile = await setupFixtureFile("mirror_v1.sqlite");
+  let buf = await SyncedBookmarksMirror.open({
+    path: dbFile.path,
+  });
+  ok(
+    buf.wasCorrupt,
+    "Migrating from unsupported version should mark database as corrupt"
+  );
+  await buf.finalize();
+});
+
+add_task(async function test_database_corrupt() {
+  let corruptFile = await setupFixtureFile("mirror_corrupt.sqlite");
+  let buf = await SyncedBookmarksMirror.open({
+    path: corruptFile.path,
+  });
+  ok(buf.wasCorrupt, "Opening corrupt database should mark it as such");
+  await buf.finalize();
+});
+
+add_task(async function test_migrate_v8() {
+  let buf = await openMirror("test_migrate_v8");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      },
+    ],
+  });
+
+  await buf.db.execute(
+    `UPDATE moz_bookmarks
+     SET syncChangeCounter = 0,
+         syncStatus = ${PlacesUtils.bookmarks.SYNC_STATUS.NEW}`
+  );
+
+  // setup the mirror.
+  await storeRecords(buf, [
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: [],
+    },
+  ]);
+
+  await buf.db.setSchemaVersion(7, "mirror");
+  await buf.finalize();
+
+  // reopen it.
+  buf = await openMirror("test_migrate_v8");
+  Assert.equal(await buf.db.getSchemaVersion("mirror"), 8, "did upgrade");
+
+  let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+    PlacesUtils.bookmarks.menuGuid
+  );
+  let [fieldsA, fieldsB, fieldsMenu] = fields;
+
+  // 'A' was in the mirror - should now be _NORMAL
+  Assert.equal(fieldsA.guid, "bookmarkAAAA");
+  Assert.equal(fieldsA.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NORMAL);
+  // 'B' was not in the mirror so should be untouched.
+  Assert.equal(fieldsB.guid, "bookmarkBBBB");
+  Assert.equal(fieldsB.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NEW);
+  // 'menu' was in the mirror - should now be _NORMAL
+  Assert.equal(fieldsMenu.guid, PlacesUtils.bookmarks.menuGuid);
+  Assert.equal(fieldsMenu.syncStatus, PlacesUtils.bookmarks.SYNC_STATUS.NORMAL);
+  await buf.finalize();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js b/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js
new file mode 100644
index 0000000000..a1dce86b2c
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_observer_recorder.js
@@ -0,0 +1,637 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+async function promiseAllURLFrecencies() {
+  let frecencies = new Map();
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT url, frecency, recalc_frecency
+    FROM moz_places
+    WHERE url_hash BETWEEN hash('http', 'prefix_lo') AND
+                           hash('http', 'prefix_hi')`);
+  for (let row of rows) {
+    frecencies.set(row.getResultByName("url"), {
+      frecency: row.getResultByName("frecency"),
+      recalc: row.getResultByName("recalc_frecency"),
+    });
+  }
+  return frecencies;
+}
+
+function mapFilterIterator(iter, fn) {
+  let results = [];
+  for (let value of iter) {
+    let newValue = fn(value);
+    if (newValue) {
+      results.push(newValue);
+    }
+  }
+  return results;
+}
+
+add_task(async function test_update_frecencies() {
+  let buf = await openMirror("update_frecencies");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Not modified in mirror; shouldn't recalculate frecency.
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        // URL changed to B1 in mirror; should recalculate frecency for B
+        // and B1, using existing frecency to determine order.
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+      {
+        // URL changed to new URL in mirror, should recalculate frecency
+        // for new URL first, before B1.
+        guid: "bookmarkBBB1",
+        title: "B1",
+        url: "http://example.com/b1",
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkBBB1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B1",
+        bmkUri: "http://example.com/b1",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local changes");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // Query; shouldn't recalculate frecency.
+        guid: "queryCCCCCCC",
+        title: "C",
+        url: "place:type=6&sort=14&maxResults=10",
+      },
+    ],
+  });
+
+  info("Calculate frecencies for all local URLs");
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkBBB1"],
+    },
+    {
+      id: "unfiled",
+      parentid: "places",
+      type: "folder",
+      children: [
+        "bookmarkBBB2",
+        "bookmarkDDDD",
+        "bookmarkEEEE",
+        "queryFFFFFFF",
+      ],
+    },
+    {
+      // Existing bookmark changed to existing URL.
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b1",
+    },
+    {
+      // Existing bookmark with new URL; should recalculate frecency first.
+      id: "bookmarkBBB1",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B1",
+      bmkUri: "http://example.com/b11",
+    },
+    {
+      id: "bookmarkBBB2",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "B2",
+      bmkUri: "http://example.com/b",
+    },
+    {
+      // New bookmark with new URL; should recalculate frecency first.
+      id: "bookmarkDDDD",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: null,
+      bmkUri: "http://example.com/d",
+    },
+    {
+      // New bookmark with new URL.
+      id: "bookmarkEEEE",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+    },
+    {
+      // New query; shouldn't count against limit.
+      id: "queryFFFFFFF",
+      parentid: "unfiled",
+      type: "query",
+      title: "F",
+      bmkUri: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
+    },
+  ]);
+
+  info("Apply new items and recalculate 3 frecencies");
+  await buf.apply();
+  await PlacesFrecencyRecalculator.recalculateSomeFrecencies({ chunkSize: 3 });
+
+  {
+    let frecencies = await promiseAllURLFrecencies();
+    let urlsWithFrecency = mapFilterIterator(
+      frecencies.entries(),
+      ([href, { frecency, recalc }]) => (recalc == 0 ? href : null)
+    );
+
+    // A is unchanged, and we should recalculate frecency for three more
+    // random URLs.
+    equal(
+      urlsWithFrecency.length,
+      4,
+      "Should keep unchanged frecency and recalculate 3"
+    );
+    let unexpectedURLs = CommonUtils.difference(
+      urlsWithFrecency,
+      new Set([
+        // A is unchanged.
+        "http://example.com/a",
+
+        // B11, D, and E are new URLs.
+        "http://example.com/b11",
+        "http://example.com/d",
+        "http://example.com/e",
+
+        // B and B1 are existing, changed URLs.
+        "http://example.com/b",
+        "http://example.com/b1",
+      ])
+    );
+    ok(
+      !unexpectedURLs.size,
+      "Should recalculate frecency for new and changed URLs only"
+    );
+  }
+
+  info("Change non-URL property of D");
+  await storeRecords(buf, [
+    {
+      id: "bookmarkDDDD",
+      parentid: "unfiled",
+      type: "bookmark",
+      title: "D (remote)",
+      bmkUri: "http://example.com/d",
+    },
+  ]);
+
+  info("Apply new item and recalculate remaining frecencies");
+  await buf.apply();
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  {
+    let frecencies = await promiseAllURLFrecencies();
+    let urlsWithoutFrecency = mapFilterIterator(
+      frecencies.entries(),
+      ([href, { frecency, recalc }]) => (recalc == 1 ? href : null)
+    );
+    deepEqual(
+      urlsWithoutFrecency,
+      [],
+      "Should finish calculating remaining frecencies"
+    );
+  }
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+async function setupLocalTree(localTimeSeconds) {
+  let dateAdded = new Date(localTimeSeconds * 1000);
+  let lastModified = new Date(localTimeSeconds * 1000);
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        dateAdded,
+        lastModified,
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            title: "B",
+            url: "http://example.com/b",
+            dateAdded,
+            lastModified,
+          },
+          {
+            guid: "bookmarkCCCC",
+            title: "C",
+            url: "http://example.com/c",
+            dateAdded,
+            lastModified,
+          },
+        ],
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: null,
+        url: "http://example.com/d",
+        dateAdded,
+        lastModified,
+      },
+    ],
+  });
+}
+
+// This test ensures we clean up the temp tables between merges, and don't throw
+// constraint errors recording observer notifications.
+add_task(async function test_apply_then_revert() {
+  let buf = await openMirror("apply_then_revert");
+
+  let now = Date.now() / 1000;
+  let localTimeSeconds = now - 180;
+
+  info("Set up initial local tree and mirror");
+  await setupLocalTree(localTimeSeconds);
+  let recordsToUpload = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+  });
+  await storeChangesInMirror(buf, recordsToUpload);
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEE1",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "E",
+    url: "http://example.com/e",
+    dateAdded: new Date(localTimeSeconds * 1000),
+    lastModified: new Date(localTimeSeconds * 1000),
+  });
+
+  info("Make remote changes");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkEEEE", "bookmarkFFFF"],
+      modified: now,
+    },
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: ["folderAAAAAA"],
+      modified: now,
+    },
+    {
+      id: "folderAAAAAA",
+      parentid: "toolbar",
+      type: "folder",
+      title: "A (remote)",
+      children: ["bookmarkCCCC", "bookmarkBBBB"],
+      modified: now,
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "folderAAAAAA",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b-remote",
+      modified: now,
+    },
+    {
+      id: "bookmarkDDDD",
+      deleted: true,
+      modified: now,
+    },
+    {
+      id: "bookmarkEEEE",
+      parentid: "menu",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+      modified: now,
+    },
+    {
+      id: "bookmarkFFFF",
+      parentid: "menu",
+      type: "bookmark",
+      title: "F",
+      bmkUri: "http://example.com/f",
+      modified: now,
+    },
+  ]);
+
+  info("Apply remote changes, first time");
+  let firstTimeRecords = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged after first time"
+  );
+
+  info("Revert local tree");
+  await PlacesSyncUtils.bookmarks.wipe();
+  await setupLocalTree(localTimeSeconds);
+  await PlacesTestUtils.markBookmarksAsSynced();
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEE1",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "E",
+    url: "http://example.com/e",
+    dateAdded: new Date(localTimeSeconds * 1000),
+    lastModified: new Date(localTimeSeconds * 1000),
+  });
+  let localIdForD = await PlacesUtils.promiseItemId("bookmarkDDDD");
+
+  info("Apply remote changes, second time");
+  await buf.db.execute(
+    `
+    UPDATE items SET
+      needsMerge = 1
+    WHERE guid <> :rootGuid`,
+    { rootGuid: PlacesUtils.bookmarks.rootGuid }
+  );
+  let observer = expectBookmarkChangeNotifications();
+  let secondTimeRecords = await buf.apply({
+    localTimeSeconds,
+    remoteTimeSeconds: now,
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid],
+    "Should leave menu with new remote structure unmerged after second time"
+  );
+  deepEqual(
+    secondTimeRecords,
+    firstTimeRecords,
+    "Should stage identical records to upload, first and second time"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkFFFF",
+    "bookmarkEEEE",
+    "folderAAAAAA",
+    "bookmarkCCCC",
+    "bookmarkBBBB",
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+  observer.check([
+    {
+      name: "bookmark-removed",
+      params: {
+        itemId: localIdForD,
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/d",
+        title: "", // null titles get turned into empty strings.
+        guid: "bookmarkDDDD",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-guid-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "",
+        guid: "bookmarkEEEE",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkFFFF"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/f",
+        title: "F",
+        guid: "bookmarkFFFF",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        oldIndex: 2,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkEEEE",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/e",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("folderAAAAAA"),
+        oldIndex: 0,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "folderAAAAAA",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkCCCC",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/c",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        oldIndex: 0,
+        newIndex: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkBBBB",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/b-remote",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("folderAAAAAA"),
+        title: "A (remote)",
+        guid: "folderAAAAAA",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      },
+    },
+    {
+      name: "bookmark-url-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/b-remote",
+        guid: "bookmarkBBBB",
+        parentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        isTagging: false,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "E",
+              url: "http://example.com/e",
+            },
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "F",
+              url: "http://example.com/f",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A (remote)",
+              children: [
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "C",
+                  url: "http://example.com/c",
+                },
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "B",
+                  url: "http://example.com/b-remote",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should apply new structure, second time"
+  );
+
+  await storeChangesInMirror(buf, secondTimeRecords);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_reconcile.js b/toolkit/components/places/tests/sync/test_bookmark_reconcile.js
new file mode 100644
index 0000000000..218e84beb6
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_reconcile.js
@@ -0,0 +1,191 @@
+// Get bookmarks which aren't marked as normally syncing and with no pending
+// changes.
+async function getBookmarksNotMarkedAsSynced() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.executeCached(
+    `
+    SELECT guid, syncStatus, syncChangeCounter FROM moz_bookmarks
+    WHERE syncChangeCounter > 1 OR syncStatus != :syncStatus
+    ORDER BY guid
+    `,
+    { syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL }
+  );
+  return rows.map(row => {
+    return {
+      guid: row.getResultByName("guid"),
+      syncStatus: row.getResultByName("syncStatus"),
+      syncChangeCounter: row.getResultByName("syncChangeCounter"),
+    };
+  });
+}
+
+add_task(async function test_reconcile_metadata() {
+  let buf = await openMirror("test_reconcile_metadata");
+
+  let olderDate = new Date(Date.now() - 100000);
+  info("Set up local tree");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        // this folder is going to reconcile exactly
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+        ],
+      },
+      {
+        // this folder's existing child isn't on the server (so will be
+        // outgoing) and also will take a new child from the server.
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "C",
+        children: [
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+        ],
+      },
+      {
+        // This bookmark is going to take the remote title.
+        guid: "bookmarkFFFF",
+        url: "http://example.com/f",
+        title: "f",
+        dateAdded: olderDate,
+        lastModified: olderDate,
+      },
+    ],
+  });
+  // And a single, local-only bookmark in the toolbar.
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bookmarkTTTT",
+        url: "http://example.com/t",
+        title: "in the toolbar",
+        dateAdded: olderDate,
+        lastModified: olderDate,
+      },
+    ],
+  });
+  // Reset to prepare for our reconciled sync.
+  await PlacesSyncUtils.bookmarks.reset();
+  // setup the mirror.
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "folderCCCCCC", "bookmarkFFFF"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "menu",
+        type: "folder",
+        title: "C",
+        children: ["bookmarkDDDD"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderCCCCCC",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "menu",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+        dateAdded: olderDate,
+        modified: Date.now() / 1000 + 60,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: [],
+        index: 1,
+      },
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: [],
+        index: 3,
+      },
+    ])
+  );
+  info("Applying");
+  let changesToUpload = await buf.apply();
+  // We need to upload a bookmark and the parent as they didn't exist on the
+  // server. Since we always use the local state for roots (bug 1472241), we'll
+  // reupload them too.
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [
+        "bookmarkEEEE",
+        "bookmarkTTTT",
+        "folderCCCCCC",
+        "menu",
+        "mobile",
+        "toolbar",
+        "unfiled",
+      ],
+      deleted: [],
+    },
+    "Should upload the 2 local-only bookmarks and their parents"
+  );
+  // Check it took the remote thing we were expecting.
+  Assert.equal((await PlacesUtils.bookmarks.fetch("bookmarkFFFF")).title, "F");
+  // Most things should be synced and have no change counter.
+  let badGuids = await getBookmarksNotMarkedAsSynced();
+  Assert.deepEqual(badGuids, [
+    {
+      // The bookmark that was only on the server. Still have SYNC_STATUS_NEW
+      // as it's yet to be uploaded.
+      guid: "bookmarkEEEE",
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      syncChangeCounter: 1,
+    },
+    {
+      // This bookmark is local only so is yet to be uploaded.
+      guid: "bookmarkTTTT",
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      syncChangeCounter: 1,
+    },
+  ]);
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js b/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js
new file mode 100644
index 0000000000..fd4f3fdb6e
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_structure_changes.js
@@ -0,0 +1,2870 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_value_structure_conflict() {
+  let buf = await openMirror("value_structure_conflict");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+          },
+        ],
+      },
+      {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        children: [
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "folderDDDDDD"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB", "bookmarkCCCC"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkEEEE"],
+        modified: Date.now() / 1000 - 60,
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+        modified: Date.now() / 1000 - 60,
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local value change");
+  await PlacesUtils.bookmarks.update({
+    guid: "folderAAAAAA",
+    title: "A (local)",
+  });
+
+  info("Make local structure change");
+  await PlacesUtils.bookmarks.update({
+    guid: "bookmarkBBBB",
+    parentGuid: "folderDDDDDD",
+    index: 0,
+  });
+
+  info("Make remote value change");
+  await storeRecords(buf, [
+    {
+      id: "folderDDDDDD",
+      parentid: "menu",
+      type: "folder",
+      title: "D (remote)",
+      children: ["bookmarkEEEE"],
+      modified: Date.now() / 1000 + 60,
+    },
+  ]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: Date.now() / 1000,
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["folderDDDDDD"],
+    "Should leave D with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkBBBB", "folderAAAAAA", "folderDDDDDD"],
+      deleted: [],
+    },
+    "Should upload records for merged and new local items"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "folderAAAAAA",
+    "bookmarkEEEE",
+    "bookmarkBBBB",
+    "folderDDDDDD",
+  ]);
+  observer.check([
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkEEEE",
+        oldParentGuid: "folderDDDDDD",
+        newParentGuid: "folderDDDDDD",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/e",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        oldIndex: 0,
+        newIndex: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkBBBB",
+        oldParentGuid: "folderDDDDDD",
+        newParentGuid: "folderDDDDDD",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/b",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("folderDDDDDD"),
+        title: "D (remote)",
+        guid: "folderDDDDDD",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A (local)",
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: "folderDDDDDD",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "D (remote)",
+          children: [
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "E",
+              url: "http://example.com/e",
+            },
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+      ],
+    },
+    "Should reconcile structure and value changes"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move() {
+  let buf = await openMirror("move");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "devFolder___",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "Dev",
+        children: [
+          {
+            guid: "mdnBmk______",
+            title: "MDN",
+            url: "https://developer.mozilla.org",
+          },
+          {
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            guid: "mozFolder___",
+            title: "Mozilla",
+            children: [
+              {
+                guid: "fxBmk_______",
+                title: "Get Firefox!",
+                url: "http://getfirefox.com/",
+              },
+              {
+                guid: "nightlyBmk__",
+                title: "Nightly",
+                url: "https://nightly.mozilla.org",
+              },
+            ],
+          },
+          {
+            guid: "wmBmk_______",
+            title: "Webmaker",
+            url: "https://webmaker.org",
+          },
+        ],
+      },
+      {
+        guid: "bzBmk_______",
+        title: "Bugzilla",
+        url: "https://bugzilla.mozilla.org",
+      },
+    ],
+  });
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: ["mozFolder___"],
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["devFolder___"],
+      },
+      {
+        // Moving to toolbar.
+        id: "devFolder___",
+        parentid: "toolbar",
+        type: "folder",
+        title: "Dev",
+        children: ["bzBmk_______", "wmBmk_______"],
+      },
+      {
+        // Moving to "Mozilla".
+        id: "mdnBmk______",
+        parentid: "mozFolder___",
+        type: "bookmark",
+        title: "MDN",
+        bmkUri: "https://developer.mozilla.org",
+      },
+      {
+        // Rearranging children and moving to unfiled.
+        id: "mozFolder___",
+        parentid: "unfiled",
+        type: "folder",
+        title: "Mozilla",
+        children: ["nightlyBmk__", "mdnBmk______", "fxBmk_______"],
+      },
+      {
+        id: "fxBmk_______",
+        parentid: "mozFolder___",
+        type: "bookmark",
+        title: "Get Firefox!",
+        bmkUri: "http://getfirefox.com/",
+      },
+      {
+        id: "nightlyBmk__",
+        parentid: "mozFolder___",
+        type: "bookmark",
+        title: "Nightly",
+        bmkUri: "https://nightly.mozilla.org",
+      },
+      {
+        id: "wmBmk_______",
+        parentid: "devFolder___",
+        type: "bookmark",
+        title: "Webmaker",
+        bmkUri: "https://webmaker.org",
+      },
+      {
+        id: "bzBmk_______",
+        parentid: "devFolder___",
+        type: "bookmark",
+        title: "Bugzilla",
+        bmkUri: "https://bugzilla.mozilla.org",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [],
+      deleted: [],
+    },
+    "Should not upload records for remotely moved items"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "devFolder___",
+    "mozFolder___",
+    "bzBmk_______",
+    "wmBmk_______",
+    "nightlyBmk__",
+    "mdnBmk______",
+    "fxBmk_______",
+  ]);
+  observer.check([
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("devFolder___"),
+        oldIndex: 0,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "devFolder___",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("mozFolder___"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "mozFolder___",
+        oldParentGuid: "devFolder___",
+        newParentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bzBmk_______"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bzBmk_______",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: "devFolder___",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "https://bugzilla.mozilla.org/",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("wmBmk_______"),
+        oldIndex: 2,
+        newIndex: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "wmBmk_______",
+        oldParentGuid: "devFolder___",
+        newParentGuid: "devFolder___",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "https://webmaker.org/",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("nightlyBmk__"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "nightlyBmk__",
+        oldParentGuid: "mozFolder___",
+        newParentGuid: "mozFolder___",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "https://nightly.mozilla.org/",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("mdnBmk______"),
+        oldIndex: 0,
+        newIndex: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "mdnBmk______",
+        oldParentGuid: "devFolder___",
+        newParentGuid: "mozFolder___",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "https://developer.mozilla.org/",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("fxBmk_______"),
+        oldIndex: 0,
+        newIndex: 2,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "fxBmk_______",
+        oldParentGuid: "mozFolder___",
+        newParentGuid: "mozFolder___",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://getfirefox.com/",
+        isTagging: false,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "devFolder___",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "Dev",
+              children: [
+                {
+                  guid: "bzBmk_______",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "Bugzilla",
+                  url: "https://bugzilla.mozilla.org/",
+                },
+                {
+                  guid: "wmBmk_______",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "Webmaker",
+                  url: "https://webmaker.org/",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "mozFolder___",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "Mozilla",
+              children: [
+                {
+                  guid: "nightlyBmk__",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "Nightly",
+                  url: "https://nightly.mozilla.org/",
+                },
+                {
+                  guid: "mdnBmk______",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "MDN",
+                  url: "https://developer.mozilla.org/",
+                },
+                {
+                  guid: "fxBmk_______",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 2,
+                  title: "Get Firefox!",
+                  url: "http://getfirefox.com/",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should move and reorder bookmarks to match remote"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_move_into_parent_sibling() {
+  // This test moves a bookmark that exists locally into a new folder that only
+  // exists remotely, and is a later sibling of the local parent. This ensures
+  // we set up the local structure before applying structure changes.
+  let buf = await openMirror("move_into_parent_sibling");
+
+  info("Set up mirror: Menu > A > B");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes: Menu > (A (B > C))");
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["folderAAAAAA", "folderCCCCCC"],
+    },
+    {
+      id: "folderAAAAAA",
+      parentid: "menu",
+      type: "folder",
+      title: "A",
+    },
+    {
+      id: "folderCCCCCC",
+      parentid: "menu",
+      type: "folder",
+      title: "C",
+      children: ["bookmarkBBBB"],
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "folderCCCCCC",
+      type: "bookmark",
+      title: "B",
+      bmkUri: "http://example.com/b",
+    },
+  ]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [],
+      deleted: [],
+    },
+    "Should not upload records for remote-only structure changes"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "folderCCCCCC",
+    "bookmarkBBBB",
+    "folderAAAAAA",
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+  observer.check([
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("folderCCCCCC"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        urlHref: "",
+        title: "C",
+        guid: "folderCCCCCC",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        oldIndex: 0,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkBBBB",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: "folderCCCCCC",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/b",
+        isTagging: false,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "folderAAAAAA",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: "A",
+        },
+        {
+          guid: "folderCCCCCC",
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: "C",
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+          ],
+        },
+      ],
+    },
+    "Should set up local structure correctly"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_complex_move_with_additions() {
+  let mergeTelemetryCounts;
+  let buf = await openMirror("complex_move_with_additions", {
+    recordStepTelemetry(name, took, counts) {
+      if (name == "merge") {
+        mergeTelemetryCounts = counts;
+      }
+    },
+  });
+
+  info("Set up mirror: Menu > A > (B C)");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB", "bookmarkCCCC"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make local change: Menu > A > (B C D)");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkDDDD",
+    parentGuid: "folderAAAAAA",
+    title: "D (local)",
+    url: "http://example.com/d-local",
+  });
+
+  info("Make remote change: ((Menu > C) (Toolbar > A > (B E)))");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkCCCC"],
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "toolbar",
+        type: "folder",
+        title: "A",
+        children: ["bookmarkBBBB", "bookmarkEEEE"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["folderAAAAAA"],
+    "Should leave A with new remote structure unmerged"
+  );
+  deepEqual(
+    mergeTelemetryCounts,
+    [{ name: "items", count: 10 }],
+    "Should record telemetry with structure change counts"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkDDDD", "folderAAAAAA"],
+      deleted: [],
+    },
+    "Should upload new records for (A D)"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkEEEE",
+    "folderAAAAAA",
+    "bookmarkCCCC",
+  ]);
+  observer.check([
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        parentId: localItemIds.get("folderAAAAAA"),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/e",
+        title: "E",
+        guid: "bookmarkEEEE",
+        parentGuid: "folderAAAAAA",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        oldIndex: 1,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkCCCC",
+        oldParentGuid: "folderAAAAAA",
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/c",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("folderAAAAAA"),
+        oldIndex: 0,
+        newIndex: 0,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: "folderAAAAAA",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "",
+        isTagging: false,
+      },
+    },
+  ]);
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              // We can guarantee child order (B E D), since we always walk remote
+              // children first, and the remote folder A record is newer than the
+              // local folder. If the local folder were newer, the order would be
+              // (D B E).
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "B",
+                  url: "http://example.com/b",
+                },
+                {
+                  guid: "bookmarkEEEE",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "E",
+                  url: "http://example.com/e",
+                },
+                {
+                  guid: "bookmarkDDDD",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 2,
+                  title: "D (local)",
+                  url: "http://example.com/d-local",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should take remote order and preserve local children"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_reorder_and_insert() {
+  let buf = await openMirror("reorder_and_insert");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      },
+      {
+        guid: "bookmarkCCCC",
+        url: "http://example.com/c",
+        title: "C",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bookmarkDDDD",
+        url: "http://example.com/d",
+        title: "D",
+      },
+      {
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+      },
+      {
+        guid: "bookmarkFFFF",
+        url: "http://example.com/f",
+        title: "F",
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB", "bookmarkCCCC"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkDDDD", "bookmarkEEEE", "bookmarkFFFF"],
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  let now = Date.now();
+
+  info("Make local changes: Reorder Menu, Toolbar > (G H)");
+  await PlacesUtils.bookmarks.reorder(PlacesUtils.bookmarks.menuGuid, [
+    "bookmarkCCCC",
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+  ]);
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bookmarkGGGG",
+        url: "http://example.com/g",
+        title: "G",
+        dateAdded: new Date(now),
+        lastModified: new Date(now),
+      },
+      {
+        guid: "bookmarkHHHH",
+        url: "http://example.com/h",
+        title: "H",
+        dateAdded: new Date(now),
+        lastModified: new Date(now),
+      },
+    ],
+  });
+
+  info("Make remote changes: Reorder Toolbar, Menu > (I J)");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        // The server has a newer toolbar, so we should use the remote order (F D E)
+        // as the base, then append (G H).
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkFFFF", "bookmarkDDDD", "bookmarkEEEE"],
+        modified: now / 1000 + 5,
+      },
+      {
+        // The server has an older menu, so we should use the local order (C A B)
+        // as the base, then append (I J).
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkAAAA",
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkIIII",
+          "bookmarkJJJJ",
+        ],
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkIIII",
+        parentid: "menu",
+        type: "bookmark",
+        title: "I",
+        bmkUri: "http://example.com/i",
+      },
+      {
+        id: "bookmarkJJJJ",
+        parentid: "menu",
+        type: "bookmark",
+        title: "J",
+        bmkUri: "http://example.com/j",
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    remoteTimeSeconds: now / 1000,
+    localTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.menuGuid, PlacesUtils.bookmarks.toolbarGuid],
+    "Should leave roots with new remote structure unmerged"
+  );
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkGGGG", "bookmarkHHHH", "menu", "toolbar"],
+      deleted: [],
+    },
+    "Should upload records for merged and new local items"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              url: "http://example.com/c",
+              title: "C",
+            },
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              url: "http://example.com/a",
+              title: "A",
+            },
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 2,
+              url: "http://example.com/b",
+              title: "B",
+            },
+            {
+              guid: "bookmarkIIII",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 3,
+              url: "http://example.com/i",
+              title: "I",
+            },
+            {
+              guid: "bookmarkJJJJ",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 4,
+              url: "http://example.com/j",
+              title: "J",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              url: "http://example.com/f",
+              title: "F",
+            },
+            {
+              guid: "bookmarkDDDD",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              url: "http://example.com/d",
+              title: "D",
+            },
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 2,
+              url: "http://example.com/e",
+              title: "E",
+            },
+            {
+              guid: "bookmarkGGGG",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 3,
+              url: "http://example.com/g",
+              title: "G",
+            },
+            {
+              guid: "bookmarkHHHH",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 4,
+              url: "http://example.com/h",
+              title: "H",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should use timestamps to decide base folder order"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_newer_remote_moves() {
+  let now = Date.now();
+  let buf = await openMirror("newer_remote_moves");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+      {
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+            dateAdded: new Date(now - 5000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+      {
+        guid: "folderFFFFFF",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "F",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            guid: "bookmarkGGGG",
+            url: "http://example.com/g",
+            title: "G",
+            dateAdded: new Date(now - 5000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        guid: "folderHHHHHH",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "H",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "folderBBBBBB", "folderDDDDDD"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkEEEE", "folderFFFFFF", "folderHHHHHH"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "toolbar",
+        type: "folder",
+        title: "F",
+        children: ["bookmarkGGGG"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkGGGG",
+        parentid: "folderFFFFFF",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderHHHHHH",
+        parentid: "toolbar",
+        type: "folder",
+        title: "H",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info(
+    "Make local changes: Unfiled > A, Mobile > B; Toolbar > (H F E); D > C; H > G"
+  );
+  let localMoves = [
+    {
+      guid: "bookmarkAAAA",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    },
+    {
+      guid: "folderBBBBBB",
+      parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    },
+    {
+      guid: "bookmarkCCCC",
+      parentGuid: "folderDDDDDD",
+    },
+    {
+      guid: "bookmarkGGGG",
+      parentGuid: "folderHHHHHH",
+    },
+  ];
+  for (let { guid, parentGuid } of localMoves) {
+    await PlacesUtils.bookmarks.update({
+      guid,
+      parentGuid,
+      index: 0,
+      lastModified: new Date(now - 2500),
+    });
+  }
+  await PlacesUtils.bookmarks.reorder(
+    PlacesUtils.bookmarks.toolbarGuid,
+    ["folderHHHHHH", "folderFFFFFF", "bookmarkEEEE"],
+    { lastModified: new Date(now - 2500) }
+  );
+
+  info(
+    "Make remote changes: Mobile > A, Unfiled > B; Toolbar > (F E H); D > G; H > C"
+  );
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderDDDDDD"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        // This is similar to H > C, explained below, except we'll always reupload
+        // the mobile root, because we always prefer the local state for roots.
+        id: "bookmarkAAAA",
+        parentid: "mobile",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: ["folderBBBBBB"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "unfiled",
+        type: "folder",
+        title: "B",
+        children: [],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["folderFFFFFF", "bookmarkEEEE", "folderHHHHHH"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "folderHHHHHH",
+        parentid: "toolbar",
+        type: "folder",
+        title: "H",
+        children: ["bookmarkCCCC"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        // Reparenting an item uploads records for the item and its parent.
+        // The merger would still work if we only marked H as unmerged; we'd
+        // then use the remote state for H, and local state for C. Since C was
+        // changed locally, we'll reupload it, even though it didn't actually
+        // change.
+        id: "bookmarkCCCC",
+        parentid: "folderHHHHHH",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkGGGG"],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        dateAdded: now - 5000,
+        modified: now / 1000,
+        children: ["bookmarkGGGG"],
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "toolbar",
+        type: "folder",
+        title: "F",
+        children: [],
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+      {
+        // Same as C above.
+        id: "bookmarkGGGG",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+        dateAdded: now - 5000,
+        modified: now / 1000,
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    localTimeSeconds: now / 1000,
+    remoteTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.mobileGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave roots with new remote structure unmerged"
+  );
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.mobileGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      // We took the remote structure for the roots, but they're still flagged as
+      // changed locally. Since we always use the local state for roots
+      // (bug 1472241), and can't distinguish between value and structure changes
+      // in Places (see the comment for F below), we'll reupload them.
+      menu: {
+        tombstone: false,
+        counter: 2,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          children: ["folderDDDDDD"],
+          title: BookmarksMenuTitle,
+        },
+      },
+      mobile: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "mobile",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.mobileGuid),
+          children: ["bookmarkAAAA"],
+          title: MobileBookmarksTitle,
+        },
+      },
+      unfiled: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "unfiled",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+          children: ["folderBBBBBB"],
+          title: UnfiledBookmarksTitle,
+        },
+      },
+      toolbar: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "toolbar",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.toolbarGuid),
+          children: ["folderFFFFFF", "bookmarkEEEE", "folderHHHHHH"],
+          title: BookmarksToolbarTitle,
+        },
+      },
+    },
+    "Should only reupload local roots"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderDDDDDD",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "D",
+              children: [
+                {
+                  guid: "bookmarkGGGG",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "G",
+                  url: "http://example.com/g",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "folderFFFFFF",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "F",
+            },
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "E",
+              url: "http://example.com/e",
+            },
+            {
+              guid: "folderHHHHHH",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 2,
+              title: "H",
+              children: [
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "C",
+                  url: "http://example.com/c",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "folderBBBBBB",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "B",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+      ],
+    },
+    "Should use newer remote parents and order"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_newer_local_moves() {
+  let now = Date.now();
+  let buf = await openMirror("newer_local_moves");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+      {
+        guid: "folderBBBBBB",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "B",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+            dateAdded: new Date(now - 5000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        guid: "folderDDDDDD",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "D",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bookmarkEEEE",
+        url: "http://example.com/e",
+        title: "E",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+      {
+        guid: "folderFFFFFF",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "F",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+        children: [
+          {
+            guid: "bookmarkGGGG",
+            url: "http://example.com/g",
+            title: "G",
+            dateAdded: new Date(now - 5000),
+            lastModified: new Date(now - 5000),
+          },
+        ],
+      },
+      {
+        guid: "folderHHHHHH",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "H",
+        dateAdded: new Date(now - 5000),
+        lastModified: new Date(now - 5000),
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "folderBBBBBB", "folderDDDDDD"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "menu",
+        type: "folder",
+        title: "B",
+        children: ["bookmarkCCCC"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderBBBBBB",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkEEEE", "folderFFFFFF", "folderHHHHHH"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "toolbar",
+        type: "folder",
+        title: "F",
+        children: ["bookmarkGGGG"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "bookmarkGGGG",
+        parentid: "folderFFFFFF",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+      {
+        id: "folderHHHHHH",
+        parentid: "toolbar",
+        type: "folder",
+        title: "H",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 5,
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info(
+    "Make local changes: Unfiled > A, Mobile > B; Toolbar > (H F E); D > C; H > G"
+  );
+  let localMoves = [
+    {
+      guid: "bookmarkAAAA",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    },
+    {
+      guid: "folderBBBBBB",
+      parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    },
+    {
+      guid: "bookmarkCCCC",
+      parentGuid: "folderDDDDDD",
+    },
+    {
+      guid: "bookmarkGGGG",
+      parentGuid: "folderHHHHHH",
+    },
+  ];
+  for (let { guid, parentGuid } of localMoves) {
+    await PlacesUtils.bookmarks.update({
+      guid,
+      parentGuid,
+      index: 0,
+      lastModified: new Date(now),
+    });
+  }
+  await PlacesUtils.bookmarks.reorder(
+    PlacesUtils.bookmarks.toolbarGuid,
+    ["folderHHHHHH", "folderFFFFFF", "bookmarkEEEE"],
+    { lastModified: new Date(now) }
+  );
+
+  info(
+    "Make remote changes: Mobile > A, Unfiled > B; Toolbar > (F E H); D > G; H > C"
+  );
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderDDDDDD"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "mobile",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "mobile",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "unfiled",
+        parentid: "places",
+        type: "folder",
+        children: ["folderBBBBBB"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "folderBBBBBB",
+        parentid: "unfiled",
+        type: "folder",
+        title: "B",
+        children: [],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["folderFFFFFF", "bookmarkEEEE", "folderHHHHHH"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "folderHHHHHH",
+        parentid: "toolbar",
+        type: "folder",
+        title: "H",
+        children: ["bookmarkCCCC"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderHHHHHH",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "folderDDDDDD",
+        parentid: "menu",
+        type: "folder",
+        title: "D",
+        children: ["bookmarkGGGG"],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "toolbar",
+        type: "folder",
+        title: "F",
+        children: [],
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+      {
+        id: "bookmarkGGGG",
+        parentid: "folderDDDDDD",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+        dateAdded: now - 5000,
+        modified: now / 1000 - 2.5,
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    localTimeSeconds: now / 1000,
+    remoteTimeSeconds: now / 1000,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      "bookmarkAAAA",
+      "bookmarkCCCC",
+      "bookmarkGGGG",
+      "folderBBBBBB",
+      "folderDDDDDD",
+      "folderFFFFFF",
+      "folderHHHHHH",
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.mobileGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave items with new remote structure unmerged"
+  );
+  let datesAdded = await promiseManyDatesAdded([
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.mobileGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+  ]);
+  deepEqual(
+    changesToUpload,
+    {
+      // Reupload roots with new children.
+      menu: {
+        tombstone: false,
+        counter: 2,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.menuGuid),
+          children: ["folderDDDDDD"],
+          title: BookmarksMenuTitle,
+        },
+      },
+      mobile: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "mobile",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.mobileGuid),
+          children: ["folderBBBBBB"],
+          title: MobileBookmarksTitle,
+        },
+      },
+      unfiled: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "unfiled",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.unfiledGuid),
+          children: ["bookmarkAAAA"],
+          title: UnfiledBookmarksTitle,
+        },
+      },
+      toolbar: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "toolbar",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: datesAdded.get(PlacesUtils.bookmarks.toolbarGuid),
+          children: ["folderHHHHHH", "folderFFFFFF", "bookmarkEEEE"],
+          title: BookmarksToolbarTitle,
+        },
+      },
+      // G moved to H from F, so F and H have new children, and we need
+      // to upload G for the new `parentid`.
+      folderFFFFFF: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderFFFFFF",
+          type: "folder",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: now - 5000,
+          children: [],
+          title: "F",
+        },
+      },
+      folderHHHHHH: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderHHHHHH",
+          type: "folder",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: now - 5000,
+          children: ["bookmarkGGGG"],
+          title: "H",
+        },
+      },
+      bookmarkGGGG: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkGGGG",
+          type: "bookmark",
+          parentid: "folderHHHHHH",
+          hasDupe: true,
+          parentName: "H",
+          dateAdded: now - 5000,
+          bmkUri: "http://example.com/g",
+          title: "G",
+        },
+      },
+      // C moved to D, so we need to reupload D (for `children`) and C
+      // (for `parentid`).
+      folderDDDDDD: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderDDDDDD",
+          type: "folder",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: now - 5000,
+          children: ["bookmarkCCCC"],
+          title: "D",
+        },
+      },
+      bookmarkCCCC: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkCCCC",
+          type: "bookmark",
+          parentid: "folderDDDDDD",
+          hasDupe: true,
+          parentName: "D",
+          dateAdded: now - 5000,
+          bmkUri: "http://example.com/c",
+          title: "C",
+        },
+      },
+      // Reupload A with the new `parentid`. B moved to mobile *and* has
+      // new children` so we should upload it, anyway.
+      bookmarkAAAA: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkAAAA",
+          type: "bookmark",
+          parentid: "unfiled",
+          hasDupe: true,
+          parentName: UnfiledBookmarksTitle,
+          dateAdded: now - 5000,
+          bmkUri: "http://example.com/a",
+          title: "A",
+        },
+      },
+      folderBBBBBB: {
+        tombstone: false,
+        counter: 2,
+        synced: false,
+        cleartext: {
+          id: "folderBBBBBB",
+          type: "folder",
+          parentid: "mobile",
+          hasDupe: true,
+          parentName: MobileBookmarksTitle,
+          dateAdded: now - 5000,
+          children: [],
+          title: "B",
+        },
+      },
+    },
+    "Should reupload new local structure"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderDDDDDD",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "D",
+              children: [
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "C",
+                  url: "http://example.com/c",
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "folderHHHHHH",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "H",
+              children: [
+                {
+                  guid: "bookmarkGGGG",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "G",
+                  url: "http://example.com/g",
+                },
+              ],
+            },
+            {
+              guid: "folderFFFFFF",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 1,
+              title: "F",
+            },
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 2,
+              title: "E",
+              url: "http://example.com/e",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A",
+              url: "http://example.com/a",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+          children: [
+            {
+              guid: "folderBBBBBB",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "B",
+            },
+          ],
+        },
+      ],
+    },
+    "Should use newer local parents and order"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_unchanged_newer_changed_older() {
+  let buf = await openMirror("unchanged_newer_changed_older");
+  let modified = new Date(Date.now() - 5000);
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    dateAdded: new Date(modified.getTime() - 5000),
+  });
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    dateAdded: new Date(modified.getTime() - 5000),
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        dateAdded: new Date(modified.getTime() - 5000),
+        lastModified: modified,
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+        dateAdded: new Date(modified.getTime() - 5000),
+        lastModified: modified,
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "folderCCCCCC",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "C",
+        dateAdded: new Date(modified.getTime() - 5000),
+        lastModified: modified,
+      },
+      {
+        guid: "bookmarkDDDD",
+        url: "http://example.com/d",
+        title: "D",
+        dateAdded: new Date(modified.getTime() - 5000),
+        lastModified: modified,
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "bookmarkBBBB"],
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["folderCCCCCC", "bookmarkDDDD"],
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+      {
+        id: "folderCCCCCC",
+        parentid: "toolbar",
+        type: "folder",
+        title: "C",
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000,
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  // Even though the local menu is newer (local = 5s, remote = 9s; adding E
+  // updated the modified times of A and the menu), it's not *changed* locally,
+  // so we should merge remote children first.
+  info("Add A > E locally with newer time; delete A remotely with older time");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkEEEE",
+    parentGuid: "folderAAAAAA",
+    url: "http://example.com/e",
+    title: "E",
+    index: 0,
+    dateAdded: new Date(modified.getTime() + 5000),
+    lastModified: new Date(modified.getTime() + 5000),
+  });
+  await storeRecords(buf, [
+    {
+      id: "menu",
+      parentid: "places",
+      type: "folder",
+      children: ["bookmarkBBBB"],
+      dateAdded: modified.getTime() - 5000,
+      modified: modified.getTime() / 1000 + 1,
+    },
+    {
+      id: "folderAAAAAA",
+      deleted: true,
+    },
+  ]);
+
+  // Even though the remote toolbar is newer (local = 15s, remote = 10s), it's
+  // not changed remotely, so we should merge local children first.
+  info("Add C > F remotely with newer time; delete C locally with older time");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "folderCCCCCC",
+        parentid: "toolbar",
+        type: "folder",
+        title: "C",
+        children: ["bookmarkFFFF"],
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000 + 5,
+      },
+      {
+        id: "bookmarkFFFF",
+        parentid: "folderCCCCCC",
+        type: "bookmark",
+        title: "F",
+        bmkUri: "http://example.com/f",
+        dateAdded: modified.getTime() - 5000,
+        modified: modified.getTime() / 1000 + 5,
+      },
+    ])
+  );
+  await PlacesUtils.bookmarks.remove("folderCCCCCC");
+  await PlacesUtils.bookmarks.update({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    lastModified: new Date(modified.getTime() - 5000),
+    // Use `SOURCES.SYNC` to avoid bumping the change counter and flagging the
+    // local toolbar as modified.
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+  });
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply({
+    localTimeSeconds: modified.getTime() / 1000 + 10,
+    remoteTimeSeconds: modified.getTime() / 1000 + 10,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkFFFF", "folderCCCCCC", PlacesUtils.bookmarks.menuGuid],
+    "Should leave deleted C; F and menu with new remote structure unmerged"
+  );
+
+  deepEqual(
+    changesToUpload,
+    {
+      menu: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "menu",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: modified.getTime() - 5000,
+          children: ["bookmarkBBBB", "bookmarkEEEE"],
+          title: BookmarksMenuTitle,
+        },
+      },
+      toolbar: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "toolbar",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: modified.getTime() - 5000,
+          children: ["bookmarkDDDD", "bookmarkFFFF"],
+          title: BookmarksToolbarTitle,
+        },
+      },
+      // Upload E and F with new `parentid`.
+      bookmarkEEEE: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkEEEE",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: modified.getTime() + 5000,
+          bmkUri: "http://example.com/e",
+          title: "E",
+        },
+      },
+      bookmarkFFFF: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkFFFF",
+          type: "bookmark",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: modified.getTime() - 5000,
+          bmkUri: "http://example.com/f",
+          title: "F",
+        },
+      },
+      folderCCCCCC: {
+        tombstone: true,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "folderCCCCCC",
+          deleted: true,
+        },
+      },
+    },
+    "Should reupload menu, toolbar, E, F with new structure; tombstone for C"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B",
+              url: "http://example.com/b",
+            },
+            {
+              guid: "bookmarkEEEE",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "E",
+              url: "http://example.com/e",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkDDDD",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "D",
+              url: "http://example.com/d",
+            },
+            {
+              guid: "bookmarkFFFF",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 1,
+              title: "F",
+              url: "http://example.com/f",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should merge children of changed side first, even if they're older"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones.map(({ guid }) => guid),
+    ["folderCCCCCC"],
+    "Should store local tombstone for C"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_bookmark_value_changes.js b/toolkit/components/places/tests/sync/test_bookmark_value_changes.js
new file mode 100644
index 0000000000..6ce7f20562
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_bookmark_value_changes.js
@@ -0,0 +1,2580 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_value_combo() {
+  let buf = await openMirror("value_combo");
+  let now = Date.now();
+
+  info("Set up mirror with existing bookmark to update");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "mozBmk______",
+        url: "https://mozilla.org",
+        title: "Mozilla",
+        tags: ["moz", "dot", "org"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["mozBmk______"],
+      },
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla",
+        bmkUri: "https://mozilla.org",
+        tags: ["moz", "dot", "org"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Insert new local bookmark to upload");
+  let [bzBmk] = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        guid: "bzBmk_______",
+        url: "https://bugzilla.mozilla.org",
+        title: "Bugzilla",
+        tags: ["new", "tag"],
+      },
+    ],
+  });
+
+  info("Insert remote bookmarks and folder to apply");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "mozBmk______",
+        parentid: "menu",
+        type: "bookmark",
+        title: "Mozilla home page",
+        bmkUri: "https://mozilla.org",
+        tags: ["browsers"],
+      },
+      {
+        id: "toolbar",
+        parentid: "places",
+        type: "folder",
+        children: ["fxBmk_______", "tFolder_____"],
+      },
+      {
+        id: "fxBmk_______",
+        parentid: "toolbar",
+        type: "bookmark",
+        title: "Get Firefox",
+        bmkUri: "http://getfirefox.com",
+        tags: ["taggy", "browsers"],
+        dateAdded: now,
+      },
+      {
+        id: "tFolder_____",
+        parentid: "toolbar",
+        type: "folder",
+        title: "Mail",
+        children: ["tbBmk_______"],
+        dateAdded: now,
+      },
+      {
+        id: "tbBmk_______",
+        parentid: "tFolder_____",
+        type: "bookmark",
+        title: "Get Thunderbird",
+        bmkUri: "http://getthunderbird.com",
+        keyword: "tb",
+        dateAdded: now,
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications({
+    skipTags: true,
+    ignoreDates: false,
+  });
+  let localTimeSeconds = Math.floor(now / 1000);
+  let changesToUpload = await buf.apply({
+    localTimeSeconds,
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [PlacesUtils.bookmarks.toolbarGuid],
+    "Should leave toolbar with new remote structure unmerged"
+  );
+
+  let menuInfo = await PlacesUtils.bookmarks.fetch(
+    PlacesUtils.bookmarks.menuGuid
+  );
+  deepEqual(
+    changesToUpload,
+    {
+      bzBmk_______: {
+        tombstone: false,
+        counter: 3,
+        synced: false,
+        cleartext: {
+          id: "bzBmk_______",
+          type: "bookmark",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: bzBmk.dateAdded.getTime(),
+          bmkUri: "https://bugzilla.mozilla.org/",
+          title: "Bugzilla",
+          tags: ["new", "tag"],
+        },
+      },
+      toolbar: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "toolbar",
+          type: "folder",
+          parentid: "places",
+          hasDupe: true,
+          parentName: "",
+          dateAdded: menuInfo.dateAdded.getTime(),
+          title: BookmarksToolbarTitle,
+          children: ["fxBmk_______", "tFolder_____", "bzBmk_______"],
+        },
+      },
+    },
+    "Should upload new local bookmarks and parents"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "fxBmk_______",
+    "tFolder_____",
+    "tbBmk_______",
+    "bzBmk_______",
+    "mozBmk______",
+    PlacesUtils.bookmarks.toolbarGuid,
+  ]);
+
+  observer.check([
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("fxBmk_______"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.toolbarGuid),
+        index: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://getfirefox.com/",
+        title: "Get Firefox",
+        guid: "fxBmk_______",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        dateAdded: now,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("tFolder_____"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.toolbarGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        urlHref: "",
+        title: "Mail",
+        guid: "tFolder_____",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        dateAdded: now,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("tbBmk_______"),
+        parentId: localItemIds.get("tFolder_____"),
+        index: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://getthunderbird.com/",
+        title: "Get Thunderbird",
+        guid: "tbBmk_______",
+        parentGuid: "tFolder_____",
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        dateAdded: now,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bzBmk_______"),
+        oldIndex: 0,
+        newIndex: 2,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bzBmk_______",
+        oldParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        newParentGuid: PlacesUtils.bookmarks.toolbarGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "https://bugzilla.mozilla.org/",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("mozBmk______"),
+        title: "Mozilla home page",
+        guid: "mozBmk______",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+  ]);
+
+  let fxBmk = await PlacesUtils.bookmarks.fetch("fxBmk_______");
+  ok(fxBmk, "New Firefox bookmark should exist");
+  equal(
+    fxBmk.parentGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    "Should add Firefox bookmark to toolbar"
+  );
+  let fxTags = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://getfirefox.com")
+  );
+  deepEqual(
+    fxTags.sort(),
+    ["browsers", "taggy"],
+    "Should tag new Firefox bookmark"
+  );
+
+  let folder = await PlacesUtils.bookmarks.fetch("tFolder_____");
+  ok(folder, "New folder should exist");
+  equal(
+    folder.parentGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    "Should add new folder to toolbar"
+  );
+
+  let tbBmk = await PlacesUtils.bookmarks.fetch("tbBmk_______");
+  ok(tbBmk, "Should insert Thunderbird child bookmark");
+  equal(
+    tbBmk.parentGuid,
+    folder.guid,
+    "Should add Thunderbird bookmark to new folder"
+  );
+  let keywordInfo = await PlacesUtils.keywords.fetch("tb");
+  equal(
+    keywordInfo.url.href,
+    "http://getthunderbird.com/",
+    "Should set keyword for Thunderbird bookmark"
+  );
+
+  let updatedBmk = await PlacesUtils.bookmarks.fetch("mozBmk______");
+  equal(
+    updatedBmk.title,
+    "Mozilla home page",
+    "Should rename Mozilla bookmark"
+  );
+  equal(
+    updatedBmk.parentGuid,
+    PlacesUtils.bookmarks.menuGuid,
+    "Should not move Mozilla bookmark"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_value_only_changes() {
+  let buf = await openMirror("value_only_changes");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "folderAAAAAA",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "A",
+        children: [
+          {
+            guid: "bookmarkBBBB",
+            url: "http://example.com/b",
+            title: "B",
+          },
+          {
+            guid: "bookmarkCCCC",
+            url: "http://example.com/c",
+            title: "C",
+          },
+          {
+            guid: "folderJJJJJJ",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "J",
+            children: [
+              {
+                guid: "bookmarkKKKK",
+                url: "http://example.com/k",
+                title: "K",
+              },
+            ],
+          },
+          {
+            guid: "bookmarkDDDD",
+            url: "http://example.com/d",
+            title: "D",
+          },
+          {
+            guid: "bookmarkEEEE",
+            url: "http://example.com/e",
+            title: "E",
+          },
+        ],
+      },
+      {
+        guid: "folderFFFFFF",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        title: "F",
+        children: [
+          {
+            guid: "bookmarkGGGG",
+            url: "http://example.com/g",
+            title: "G",
+          },
+          {
+            guid: "folderHHHHHH",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            title: "H",
+            children: [
+              {
+                guid: "bookmarkIIII",
+                url: "http://example.com/i",
+                title: "I",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["folderAAAAAA", "folderFFFFFF"],
+      },
+      {
+        id: "folderAAAAAA",
+        parentid: "menu",
+        type: "folder",
+        title: "A",
+        children: [
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "folderJJJJJJ",
+          "bookmarkDDDD",
+          "bookmarkEEEE",
+        ],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "folderJJJJJJ",
+        parentid: "folderAAAAAA",
+        type: "folder",
+        title: "J",
+        children: ["bookmarkKKKK"],
+      },
+      {
+        id: "bookmarkKKKK",
+        parentid: "folderJJJJJJ",
+        type: "bookmark",
+        title: "K",
+        bmkUri: "http://example.com/k",
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "menu",
+        type: "folder",
+        title: "F",
+        children: ["bookmarkGGGG", "folderHHHHHH"],
+      },
+      {
+        id: "bookmarkGGGG",
+        parentid: "folderFFFFFF",
+        type: "bookmark",
+        title: "G",
+        bmkUri: "http://example.com/g",
+      },
+      {
+        id: "folderHHHHHH",
+        parentid: "folderFFFFFF",
+        type: "folder",
+        title: "H",
+        children: ["bookmarkIIII"],
+      },
+      {
+        id: "bookmarkIIII",
+        parentid: "folderHHHHHH",
+        type: "bookmark",
+        title: "I",
+        bmkUri: "http://example.com/i",
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkCCCC",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "C (remote)",
+        bmkUri: "http://example.com/c-remote",
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "folderAAAAAA",
+        type: "bookmark",
+        title: "E (remote)",
+        bmkUri: "http://example.com/e-remote",
+      },
+      {
+        id: "bookmarkIIII",
+        parentid: "folderHHHHHH",
+        type: "bookmark",
+        title: "I (remote)",
+        bmkUri: "http://example.com/i-remote",
+      },
+      {
+        id: "folderFFFFFF",
+        parentid: "menu",
+        type: "folder",
+        title: "F (remote)",
+        children: ["bookmarkGGGG", "folderHHHHHH"],
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: [],
+      deleted: [],
+    },
+    "Should not upload records for remote-only value changes"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "folderAAAAAA",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 0,
+              title: "A",
+              children: [
+                {
+                  guid: "bookmarkBBBB",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "B",
+                  url: "http://example.com/b",
+                },
+                {
+                  guid: "bookmarkCCCC",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 1,
+                  title: "C (remote)",
+                  url: "http://example.com/c-remote",
+                },
+                {
+                  guid: "folderJJJJJJ",
+                  type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                  index: 2,
+                  title: "J",
+                  children: [
+                    {
+                      guid: "bookmarkKKKK",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 0,
+                      title: "K",
+                      url: "http://example.com/k",
+                    },
+                  ],
+                },
+                {
+                  guid: "bookmarkDDDD",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 3,
+                  title: "D",
+                  url: "http://example.com/d",
+                },
+                {
+                  guid: "bookmarkEEEE",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 4,
+                  title: "E (remote)",
+                  url: "http://example.com/e-remote",
+                },
+              ],
+            },
+            {
+              guid: "folderFFFFFF",
+              type: PlacesUtils.bookmarks.TYPE_FOLDER,
+              index: 1,
+              title: "F (remote)",
+              children: [
+                {
+                  guid: "bookmarkGGGG",
+                  type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                  index: 0,
+                  title: "G",
+                  url: "http://example.com/g",
+                },
+                {
+                  guid: "folderHHHHHH",
+                  type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                  index: 1,
+                  title: "H",
+                  children: [
+                    {
+                      guid: "bookmarkIIII",
+                      type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+                      index: 0,
+                      title: "I (remote)",
+                      url: "http://example.com/i-remote",
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should not change structure for value-only changes"
+  );
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_conflicting_keywords() {
+  let buf = await openMirror("conflicting_keywords");
+  let dateAdded = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+        keyword: "one",
+        dateAdded,
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        keyword: "one",
+        dateAdded: dateAdded.getTime(),
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  {
+    let entryByKeyword = await PlacesUtils.keywords.fetch("one");
+    equal(
+      entryByKeyword.url.href,
+      "http://example.com/a",
+      "Should return new keyword entry by URL"
+    );
+    let entryByURL = await PlacesUtils.keywords.fetch({
+      url: "http://example.com/a",
+    });
+    equal(entryByURL.keyword, "one", "Should return new entry by keyword");
+  }
+
+  info("Insert new bookmark with same URL and different keyword");
+  {
+    await storeRecords(
+      buf,
+      shuffle([
+        {
+          id: "toolbar",
+          parentid: "places",
+          type: "folder",
+          children: ["bookmarkAAA1"],
+        },
+        {
+          id: "bookmarkAAA1",
+          parentid: "toolbar",
+          type: "bookmark",
+          title: "A1",
+          bmkUri: "http://example.com/a",
+          keyword: "two",
+          dateAdded: dateAdded.getTime(),
+        },
+      ])
+    );
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["bookmarkAAA1"],
+      "Should leave A1 with conflicting keyword unmerged"
+    );
+    deepEqual(
+      changesToUpload,
+      {
+        bookmarkAAAA: {
+          tombstone: false,
+          counter: 1,
+          synced: false,
+          cleartext: {
+            id: "bookmarkAAAA",
+            type: "bookmark",
+            parentid: "menu",
+            hasDupe: true,
+            parentName: BookmarksMenuTitle,
+            dateAdded: dateAdded.getTime(),
+            bmkUri: "http://example.com/a",
+            title: "A",
+            keyword: "two",
+          },
+        },
+        bookmarkAAA1: {
+          tombstone: false,
+          counter: 1,
+          synced: false,
+          cleartext: {
+            id: "bookmarkAAA1",
+            type: "bookmark",
+            parentid: "toolbar",
+            hasDupe: true,
+            parentName: BookmarksToolbarTitle,
+            dateAdded: dateAdded.getTime(),
+            bmkUri: "http://example.com/a",
+            title: "A1",
+            keyword: "two",
+          },
+        },
+      },
+      "Should reupload bookmarks with different keyword"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+
+    let entryByOldKeyword = await PlacesUtils.keywords.fetch("one");
+    ok(
+      !entryByOldKeyword,
+      "Should remove old entry when inserting bookmark with different keyword"
+    );
+    let entryByNewKeyword = await PlacesUtils.keywords.fetch("two");
+    equal(
+      entryByNewKeyword.url.href,
+      "http://example.com/a",
+      "Should return new keyword entry by URL"
+    );
+    let entryByURL = await PlacesUtils.keywords.fetch({
+      url: "http://example.com/a",
+    });
+    equal(entryByURL.keyword, "two", "Should return new entry by URL");
+  }
+
+  info("Update bookmark with different keyword");
+  {
+    await storeRecords(
+      buf,
+      shuffle([
+        {
+          id: "bookmarkAAAA",
+          parentid: "menu",
+          type: "bookmark",
+          title: "A",
+          bmkUri: "http://example.com/a",
+          keyword: "three",
+          dateAdded: dateAdded.getTime(),
+        },
+      ])
+    );
+    let changesToUpload = await buf.apply();
+    deepEqual(
+      await buf.fetchUnmergedGuids(),
+      ["bookmarkAAAA"],
+      "Should leave A with conflicting keyword unmerged"
+    );
+    deepEqual(
+      changesToUpload,
+      {
+        bookmarkAAAA: {
+          tombstone: false,
+          counter: 1,
+          synced: false,
+          cleartext: {
+            id: "bookmarkAAAA",
+            type: "bookmark",
+            parentid: "menu",
+            hasDupe: true,
+            parentName: BookmarksMenuTitle,
+            dateAdded: dateAdded.getTime(),
+            bmkUri: "http://example.com/a",
+            title: "A",
+            keyword: "three",
+          },
+        },
+        bookmarkAAA1: {
+          tombstone: false,
+          counter: 1,
+          synced: false,
+          cleartext: {
+            id: "bookmarkAAA1",
+            type: "bookmark",
+            parentid: "toolbar",
+            hasDupe: true,
+            parentName: BookmarksToolbarTitle,
+            dateAdded: dateAdded.getTime(),
+            bmkUri: "http://example.com/a",
+            title: "A1",
+            keyword: "three",
+          },
+        },
+      },
+      "Should reupload A and A1 with updated keyword"
+    );
+    await storeChangesInMirror(buf, changesToUpload);
+
+    let entryByOldKeyword = await PlacesUtils.keywords.fetch("two");
+    ok(
+      !entryByOldKeyword,
+      "Should remove old entry when updating bookmark keyword"
+    );
+    let entryByNewKeyword = await PlacesUtils.keywords.fetch("three");
+    equal(
+      entryByNewKeyword.url.href,
+      "http://example.com/a",
+      "Should return updated keyword entry by URL"
+    );
+    let entryByURL = await PlacesUtils.keywords.fetch({
+      url: "http://example.com/a",
+    });
+    equal(entryByURL.keyword, "three", "Should return updated entry by URL");
+  }
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_keywords() {
+  let buf = await openMirror("keywords");
+  let now = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+        keyword: "one",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+        keyword: "two",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkCCCC",
+        title: "C",
+        url: "http://example.com/c",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: "D",
+        url: "http://example.com/d",
+        keyword: "three",
+        dateAdded: now,
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkAAAA",
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkDDDD",
+        ],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        keyword: "one",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        keyword: "two",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "menu",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        keyword: "three",
+        dateAdded: now.getTime(),
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Change keywords remotely");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        keyword: "two",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        dateAdded: now.getTime(),
+      },
+    ])
+  );
+
+  info("Change keywords locally");
+  await PlacesUtils.keywords.insert({
+    keyword: "four",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.keywords.remove("three");
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  deepEqual(
+    changesToUpload,
+    {
+      bookmarkCCCC: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkCCCC",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: now.getTime(),
+          bmkUri: "http://example.com/c",
+          title: "C",
+          keyword: "four",
+        },
+      },
+      bookmarkDDDD: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "bookmarkDDDD",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: now.getTime(),
+          bmkUri: "http://example.com/d",
+          title: "D",
+        },
+      },
+    },
+    "Should upload C with new keyword, D with keyword removed"
+  );
+
+  let entryForOne = await PlacesUtils.keywords.fetch("one");
+  ok(!entryForOne, "Should remove existing keyword from A");
+
+  let entriesForTwo = await fetchAllKeywords("two");
+  deepEqual(
+    entriesForTwo.map(entry => ({
+      keyword: entry.keyword,
+      url: entry.url.href,
+    })),
+    [
+      {
+        keyword: "two",
+        url: "http://example.com/a",
+      },
+    ],
+    "Should move keyword for B to A"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_keywords_complex() {
+  let buf = await openMirror("keywords_complex");
+  let now = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+        keyword: "four",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkCCCC",
+        title: "C",
+        url: "http://example.com/c",
+        keyword: "five",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: "D",
+        url: "http://example.com/d",
+        dateAdded: now,
+      },
+      {
+        guid: "bookmarkEEEE",
+        title: "E",
+        url: "http://example.com/e",
+        keyword: "three",
+        dateAdded: now,
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkDDDD",
+          "bookmarkEEEE",
+        ],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        keyword: "four",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+        keyword: "five",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "menu",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkEEEE",
+        parentid: "menu",
+        type: "bookmark",
+        title: "E",
+        bmkUri: "http://example.com/e",
+        keyword: "three",
+        dateAdded: now.getTime(),
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkAAAA",
+          "bookmarkAAA1",
+          "bookmarkBBB1",
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkDDDD",
+          "bookmarkEEEE",
+        ],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        keyword: "one",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkAAA1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A (copy)",
+        bmkUri: "http://example.com/a",
+        keyword: "two",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkBBB1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        dateAdded: now.getTime(),
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C (remote)",
+        bmkUri: "http://example.com/c-remote",
+        keyword: "six",
+        dateAdded: now.getTime(),
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["bookmarkAAA1", "bookmarkAAAA", "bookmarkBBB1"],
+    "Should leave A1, A, B with conflicting keywords unmerged"
+  );
+
+  let expectedChangesToUpload = {
+    bookmarkBBBB: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkBBBB",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: now.getTime(),
+        bmkUri: "http://example.com/b",
+        title: "B",
+      },
+    },
+    bookmarkBBB1: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkBBB1",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: now.getTime(),
+        bmkUri: "http://example.com/b",
+        title: "B",
+      },
+    },
+    bookmarkAAAA: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkAAAA",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: now.getTime(),
+        bmkUri: "http://example.com/a",
+        title: "A",
+      },
+    },
+    bookmarkAAA1: {
+      tombstone: false,
+      counter: 1,
+      synced: false,
+      cleartext: {
+        id: "bookmarkAAA1",
+        type: "bookmark",
+        parentid: "menu",
+        hasDupe: true,
+        parentName: BookmarksMenuTitle,
+        dateAdded: now.getTime(),
+        bmkUri: "http://example.com/a",
+        title: "A (copy)",
+      },
+    },
+  };
+
+  // We'll take the keyword of either "bookmarkAAAA" or "bookmarkAAA1",
+  // depending on which we see first, and reupload the other.
+  let entriesForOne = await fetchAllKeywords("one");
+  let entriesForTwo = await fetchAllKeywords("two");
+  if (entriesForOne.length) {
+    ok(!entriesForTwo.length, "Should drop conflicting keyword from A1");
+    deepEqual(
+      entriesForOne.map(keyword => keyword.url.href),
+      ["http://example.com/a"],
+      "Should use A keyword for A and A1"
+    );
+    expectedChangesToUpload.bookmarkAAAA.cleartext.keyword = "one";
+    expectedChangesToUpload.bookmarkAAA1.cleartext.keyword = "one";
+  } else {
+    ok(!entriesForOne.length, "Should drop conflicting keyword from A");
+    deepEqual(
+      entriesForTwo.map(keyword => keyword.url.href),
+      ["http://example.com/a"],
+      "Should use A1 keyword for A and A1"
+    );
+    expectedChangesToUpload.bookmarkAAAA.cleartext.keyword = "two";
+    expectedChangesToUpload.bookmarkAAA1.cleartext.keyword = "two";
+  }
+  deepEqual(
+    changesToUpload,
+    expectedChangesToUpload,
+    "Should reupload all local records with corrected keywords"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkAAAA",
+    "bookmarkAAA1",
+    "bookmarkBBB1",
+    "bookmarkBBBB",
+    "bookmarkCCCC",
+    "bookmarkDDDD",
+    "bookmarkEEEE",
+    PlacesUtils.bookmarks.menuGuid,
+  ]);
+  let expectedNotifications = [
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkAAAA"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 0,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/a",
+        title: "A",
+        guid: "bookmarkAAAA",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkAAA1"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 1,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/a",
+        title: "A (copy)",
+        guid: "bookmarkAAA1",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      name: "bookmark-added",
+      params: {
+        itemId: localItemIds.get("bookmarkBBB1"),
+        parentId: localItemIds.get(PlacesUtils.bookmarks.menuGuid),
+        index: 2,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/b",
+        title: "B",
+        guid: "bookmarkBBB1",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      },
+    },
+    {
+      // These `bookmark-moved` notifications aren't necessary: we only moved
+      // (B C D E) to accomodate (A A1 B1), and Places doesn't usually fire move
+      // notifications for repositioned siblings. However, detecting and filtering
+      // these out complicates `noteObserverChanges`, so, for simplicity, we
+      // record and fire the extra notifications.
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        oldIndex: 0,
+        newIndex: 3,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkBBBB",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/b",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        oldIndex: 1,
+        newIndex: 4,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkCCCC",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/c-remote",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkDDDD"),
+        oldIndex: 2,
+        newIndex: 5,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkDDDD",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/d",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-moved",
+      params: {
+        itemId: localItemIds.get("bookmarkEEEE"),
+        oldIndex: 3,
+        newIndex: 6,
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        guid: "bookmarkEEEE",
+        oldParentGuid: PlacesUtils.bookmarks.menuGuid,
+        newParentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        urlHref: "http://example.com/e",
+        isTagging: false,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        title: "C (remote)",
+        guid: "bookmarkCCCC",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+    {
+      name: "bookmark-url-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        urlHref: "http://example.com/c-remote",
+        guid: "bookmarkCCCC",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+        source: PlacesUtils.bookmarks.SOURCES.SYNC,
+        isTagging: false,
+      },
+    },
+  ];
+  observer.check(expectedNotifications);
+
+  let entriesForFour = await fetchAllKeywords("four");
+  ok(!entriesForFour.length, "Should remove all keywords for B");
+
+  let entriesForOldC = await fetchAllKeywords({
+    url: "http://example.com/c",
+  });
+  ok(!entriesForOldC.length, "Should remove all keywords from old C URL");
+  let entriesForNewC = await fetchAllKeywords({
+    url: "http://example.com/c-remote",
+  });
+  deepEqual(
+    entriesForNewC.map(entry => entry.keyword),
+    ["six"],
+    "Should add new keyword to new C URL"
+  );
+
+  let entriesForD = await fetchAllKeywords("http://example.com/d");
+  ok(!entriesForD.length, "Should not add keywords to D");
+
+  let entriesForThree = await fetchAllKeywords("three");
+  deepEqual(
+    entriesForThree.map(keyword => keyword.url.href),
+    ["http://example.com/e"],
+    "Should not change keywords for E"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_tags_complex() {
+  let buf = await openMirror("tags_complex");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAA1",
+        title: "A1",
+        url: "http://example.com/a",
+        tags: ["one", "two"],
+      },
+      {
+        guid: "bookmarkAAA2",
+        title: "A2",
+        url: "http://example.com/a",
+        tags: ["one", "two"],
+      },
+      {
+        guid: "bookmarkBBB1",
+        title: "B1",
+        url: "http://example.com/b",
+        tags: ["one"],
+      },
+      {
+        guid: "bookmarkBBB2",
+        title: "B2",
+        url: "http://example.com/b",
+        tags: ["one"],
+      },
+      {
+        guid: "bookmarkCCC1",
+        title: "C1",
+        url: "http://example.com/c",
+        tags: ["two", "three"],
+      },
+      {
+        guid: "bookmarkCCC2",
+        title: "C2",
+        url: "http://example.com/c",
+        tags: ["two", "three"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkAAA1",
+          "bookmarkAAA2",
+          "bookmarkBBB1",
+          "bookmarkBBB2",
+          "bookmarkCCC1",
+          "bookmarkCCC2",
+        ],
+      },
+      {
+        id: "bookmarkAAA1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two"],
+      },
+      {
+        id: "bookmarkAAA2",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two"],
+      },
+      {
+        id: "bookmarkBBB1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B1",
+        bmkUri: "http://example.com/b",
+        tags: ["one"],
+      },
+      {
+        id: "bookmarkBBB2",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B2",
+        bmkUri: "http://example.com/b",
+        tags: ["one"],
+      },
+      {
+        id: "bookmarkCCC1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C1",
+        bmkUri: "http://example.com/c",
+        tags: ["two", "three"],
+      },
+      {
+        id: "bookmarkCCC2",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C2",
+        bmkUri: "http://example.com/c",
+        tags: ["two", "three"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Add tags for B locally");
+  PlacesUtils.tagging.tagURI(Services.io.newURI("http://example.com/b"), [
+    "four",
+    "five",
+  ]);
+
+  info("Remove tag from C locally");
+  PlacesUtils.tagging.untagURI(Services.io.newURI("http://example.com/c"), [
+    "two",
+  ]);
+
+  info("Update tags for A remotely");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkAAA1",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A1",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two", "four", "six"],
+      },
+      {
+        id: "bookmarkAAA2",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A2",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two", "four", "six"],
+      },
+    ])
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+
+  let datesAdded = await promiseManyDatesAdded([
+    "bookmarkBBB1",
+    "bookmarkBBB2",
+    "bookmarkCCC1",
+    "bookmarkCCC2",
+  ]);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+  deepEqual(
+    changesToUpload,
+    {
+      bookmarkBBB1: {
+        counter: 2,
+        synced: false,
+        tombstone: false,
+        cleartext: {
+          id: "bookmarkBBB1",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: datesAdded.get("bookmarkBBB1"),
+          bmkUri: "http://example.com/b",
+          title: "B1",
+          tags: ["five", "four", "one"],
+        },
+      },
+      bookmarkBBB2: {
+        counter: 2,
+        synced: false,
+        tombstone: false,
+        cleartext: {
+          id: "bookmarkBBB2",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: datesAdded.get("bookmarkBBB2"),
+          bmkUri: "http://example.com/b",
+          title: "B2",
+          tags: ["five", "four", "one"],
+        },
+      },
+      bookmarkCCC1: {
+        counter: 1,
+        synced: false,
+        tombstone: false,
+        cleartext: {
+          id: "bookmarkCCC1",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: datesAdded.get("bookmarkCCC1"),
+          bmkUri: "http://example.com/c",
+          title: "C1",
+          tags: ["three"],
+        },
+      },
+      bookmarkCCC2: {
+        counter: 1,
+        synced: false,
+        tombstone: false,
+        cleartext: {
+          id: "bookmarkCCC2",
+          type: "bookmark",
+          parentid: "menu",
+          hasDupe: true,
+          parentName: BookmarksMenuTitle,
+          dateAdded: datesAdded.get("bookmarkCCC2"),
+          bmkUri: "http://example.com/c",
+          title: "C2",
+          tags: ["three"],
+        },
+      },
+    },
+    "Should upload local records with new tags"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      guid: PlacesUtils.bookmarks.menuGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: BookmarksMenuTitle,
+      children: [
+        {
+          guid: "bookmarkAAA1",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 0,
+          title: "A1",
+          url: "http://example.com/a",
+          tags: ["four", "one", "six", "two"],
+        },
+        {
+          guid: "bookmarkAAA2",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 1,
+          title: "A2",
+          url: "http://example.com/a",
+          tags: ["four", "one", "six", "two"],
+        },
+        {
+          guid: "bookmarkBBB1",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 2,
+          title: "B1",
+          url: "http://example.com/b",
+          tags: ["five", "four", "one"],
+        },
+        {
+          guid: "bookmarkBBB2",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 3,
+          title: "B2",
+          url: "http://example.com/b",
+          tags: ["five", "four", "one"],
+        },
+        {
+          guid: "bookmarkCCC1",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 4,
+          title: "C1",
+          url: "http://example.com/c",
+          tags: ["three"],
+        },
+        {
+          guid: "bookmarkCCC2",
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          index: 5,
+          title: "C2",
+          url: "http://example.com/c",
+          tags: ["three"],
+        },
+      ],
+    },
+    "Should update local items with new tags"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_tags() {
+  let buf = await openMirror("tags");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+        tags: ["one", "two", "three", "four"],
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+        tags: ["five", "six"],
+      },
+      {
+        guid: "bookmarkCCCC",
+        title: "C",
+        url: "http://example.com/c",
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: "D",
+        url: "http://example.com/d",
+        tags: ["seven", "eight", "nine"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: [
+          "bookmarkAAAA",
+          "bookmarkBBBB",
+          "bookmarkCCCC",
+          "bookmarkDDDD",
+        ],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two", "three", "four"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        tags: ["five", "six"],
+      },
+      {
+        id: "bookmarkCCCC",
+        parentid: "menu",
+        type: "bookmark",
+        title: "C",
+        bmkUri: "http://example.com/c",
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "menu",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        tags: ["seven", "eight", "nine"],
+      },
+    ]),
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Change tags remotely");
+  await storeRecords(
+    buf,
+    shuffle([
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+        tags: ["one", "two", "ten"],
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        bmkUri: "http://example.com/b",
+        tags: [],
+      },
+    ])
+  );
+
+  info("Change tags locally");
+  PlacesUtils.tagging.tagURI(Services.io.newURI("http://example.com/c"), [
+    "eleven",
+    "twelve",
+  ]);
+
+  let wait = PlacesTestUtils.waitForNotification("bookmark-removed", events =>
+    events.some(event => event.parentGuid == PlacesUtils.bookmarks.tagsGuid)
+  );
+
+  PlacesUtils.tagging.untagURI(
+    Services.io.newURI("http://example.com/d"),
+    null
+  );
+
+  await wait;
+
+  await new Promise(resolve => Services.tm.dispatchToMainThread(resolve));
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkCCCC", "bookmarkDDDD"],
+      deleted: [],
+    },
+    "Should upload local records with new tags"
+  );
+
+  deepEqual(
+    changesToUpload.bookmarkCCCC.cleartext.tags.sort(),
+    ["eleven", "twelve"],
+    "Should upload record with new tags for C"
+  );
+  ok(
+    !changesToUpload.bookmarkDDDD.cleartext.tags,
+    "Should upload record for D with tags removed"
+  );
+
+  let tagsForA = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://example.com/a")
+  );
+  deepEqual(tagsForA.sort(), ["one", "ten", "two"], "Should change tags for A");
+
+  let tagsForB = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://example.com/b")
+  );
+  deepEqual(tagsForB, [], "Should remove all tags from B");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_rewrite_tag_queries() {
+  let buf = await openMirror("rewrite_tag_queries");
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkDDDD",
+        title: "D",
+        url: "http://example.com/d",
+        tags: ["kitty"],
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkDDDD"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkDDDD",
+        parentid: "menu",
+        type: "bookmark",
+        title: "D",
+        bmkUri: "http://example.com/d",
+        tags: ["kitty"],
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Add tag queries for new and existing tags");
+  await storeRecords(buf, [
+    {
+      id: "toolbar",
+      parentid: "places",
+      type: "folder",
+      children: ["queryBBBBBBB", "queryCCCCCCC", "bookmarkEEEE"],
+    },
+    {
+      id: "queryBBBBBBB",
+      parentid: "toolbar",
+      type: "query",
+      title: "Tagged stuff",
+      bmkUri: "place:type=7&folder=999",
+      folderName: "taggy",
+    },
+    {
+      id: "queryCCCCCCC",
+      parentid: "toolbar",
+      type: "query",
+      title: "Cats",
+      bmkUri: "place:type=7&folder=888",
+      folderName: "kitty",
+    },
+    {
+      id: "bookmarkEEEE",
+      parentid: "toolbar",
+      type: "bookmark",
+      title: "E",
+      bmkUri: "http://example.com/e",
+      tags: ["taggy"],
+    },
+  ]);
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    ["queryBBBBBBB", "queryCCCCCCC"],
+    "Should leave rewritten queries unmerged"
+  );
+
+  deepEqual(
+    changesToUpload,
+    {
+      queryBBBBBBB: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "queryBBBBBBB",
+          type: "query",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: undefined,
+          bmkUri: "place:tag=taggy",
+          title: "Tagged stuff",
+          folderName: "taggy",
+        },
+      },
+      queryCCCCCCC: {
+        tombstone: false,
+        counter: 1,
+        synced: false,
+        cleartext: {
+          id: "queryCCCCCCC",
+          type: "query",
+          parentid: "toolbar",
+          hasDupe: true,
+          parentName: BookmarksToolbarTitle,
+          dateAdded: undefined,
+          bmkUri: "place:tag=kitty",
+          title: "Cats",
+          folderName: "kitty",
+        },
+      },
+    },
+    "Should reupload (E C) with rewritten URLs"
+  );
+
+  let bmWithTaggy = await PlacesUtils.bookmarks.fetch({ tags: ["taggy"] });
+  equal(
+    bmWithTaggy.url.href,
+    "http://example.com/e",
+    "Should insert bookmark with new tag"
+  );
+
+  let bmWithKitty = await PlacesUtils.bookmarks.fetch({ tags: ["kitty"] });
+  equal(
+    bmWithKitty.url.href,
+    "http://example.com/d",
+    "Should retain existing tag"
+  );
+
+  let { root: toolbarContainer } = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.toolbarGuid,
+    false,
+    true
+  );
+  equal(
+    toolbarContainer.childCount,
+    3,
+    "Should add queries and bookmark to toolbar"
+  );
+
+  let containerForB = PlacesUtils.asContainer(toolbarContainer.getChild(0));
+  containerForB.containerOpen = true;
+  for (let i = 0; i < containerForB.childCount; ++i) {
+    let child = containerForB.getChild(i);
+    equal(
+      child.uri,
+      "http://example.com/e",
+      `Rewritten tag query B should have tagged child node at ${i}`
+    );
+  }
+  containerForB.containerOpen = false;
+
+  let containerForC = PlacesUtils.asContainer(toolbarContainer.getChild(1));
+  containerForC.containerOpen = true;
+  for (let i = 0; i < containerForC.childCount; ++i) {
+    let child = containerForC.getChild(i);
+    equal(
+      child.uri,
+      "http://example.com/d",
+      `Rewritten tag query C should have tagged child node at ${i}`
+    );
+  }
+  containerForC.containerOpen = false;
+
+  toolbarContainer.containerOpen = false;
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_date_added() {
+  let buf = await openMirror("date_added");
+
+  let aDateAdded = new Date(Date.now() - 1 * 24 * 60 * 60 * 1000);
+  let bDateAdded = new Date();
+
+  info("Set up mirror");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        dateAdded: aDateAdded,
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        dateAdded: bDateAdded,
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await storeRecords(
+    buf,
+    [
+      {
+        id: "menu",
+        parentid: "places",
+        type: "folder",
+        children: ["bookmarkAAAA", "bookmarkBBBB"],
+      },
+      {
+        id: "bookmarkAAAA",
+        parentid: "menu",
+        type: "bookmark",
+        title: "A",
+        dateAdded: aDateAdded.getTime(),
+        bmkUri: "http://example.com/a",
+      },
+      {
+        id: "bookmarkBBBB",
+        parentid: "menu",
+        type: "bookmark",
+        title: "B",
+        dateAdded: bDateAdded.getTime(),
+        bmkUri: "http://example.com/b",
+      },
+    ],
+    { needsMerge: false }
+  );
+  await PlacesTestUtils.markBookmarksAsSynced();
+
+  info("Make remote changes");
+  let bNewDateAdded = new Date(bDateAdded.getTime() - 1 * 60 * 60 * 1000);
+  await storeRecords(buf, [
+    {
+      id: "bookmarkAAAA",
+      parentid: "menu",
+      type: "bookmark",
+      title: "A (remote)",
+      dateAdded: Date.now(),
+      bmkUri: "http://example.com/a",
+    },
+    {
+      id: "bookmarkBBBB",
+      parentid: "menu",
+      type: "bookmark",
+      title: "B (remote)",
+      dateAdded: bNewDateAdded.getTime(),
+      bmkUri: "http://example.com/b",
+    },
+  ]);
+
+  info("Apply remote");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  let idsToUpload = inspectChangeRecords(changesToUpload);
+  deepEqual(
+    idsToUpload,
+    {
+      updated: ["bookmarkAAAA"],
+      deleted: [],
+    },
+    "Should flag A for weak reupload"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+  ]);
+  observer.check([
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkAAAA"),
+        title: "A (remote)",
+        guid: "bookmarkAAAA",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        title: "B (remote)",
+        guid: "bookmarkBBBB",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+  ]);
+
+  let changeCounter = changesToUpload.bookmarkAAAA.counter;
+  strictEqual(changeCounter, 0, "Should not bump change counter for A");
+
+  let aInfo = await PlacesUtils.bookmarks.fetch("bookmarkAAAA");
+  equal(aInfo.title, "A (remote)", "Should change local title for A");
+  deepEqual(
+    aInfo.dateAdded,
+    aDateAdded,
+    "Should not change date added for A to newer remote date"
+  );
+
+  let bInfo = await PlacesUtils.bookmarks.fetch("bookmarkBBBB");
+  equal(bInfo.title, "B (remote)", "Should change local title for B");
+  deepEqual(
+    bInfo.dateAdded,
+    bNewDateAdded,
+    "Should take older date added for B"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+// Bug 1472435.
+add_task(async function test_duplicate_url_rows() {
+  let buf = await openMirror("test_duplicate_url_rows");
+
+  let placesToInsert = [
+    {
+      guid: "placeAAAAAAA",
+      href: "http://example.com",
+    },
+    {
+      guid: "placeBBBBBBB",
+      href: "http://example.com",
+    },
+    {
+      guid: "placeCCCCCCC",
+      href: "http://example.com/c",
+    },
+  ];
+
+  let itemsToInsert = [
+    {
+      guid: "bookmarkAAAA",
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      placeGuid: "placeAAAAAAA",
+      localTitle: "A",
+      remoteTitle: "A (remote)",
+    },
+    {
+      guid: "bookmarkBBBB",
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      placeGuid: "placeBBBBBBB",
+      localTitle: "B",
+      remoteTitle: "B (remote)",
+    },
+    {
+      guid: "bookmarkCCCC",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      placeGuid: "placeCCCCCCC",
+      localTitle: "C",
+      remoteTitle: "C (remote)",
+    },
+  ];
+
+  info("Manually insert local and remote items with duplicate URLs");
+  await buf.db.executeTransaction(async function () {
+    for (let { guid, href } of placesToInsert) {
+      let url = new URL(href);
+      await buf.db.executeCached(
+        `
+        INSERT INTO moz_places(url, url_hash, rev_host, hidden, frecency, guid)
+        VALUES(:url, hash(:url), :revHost, 0, -1, :guid)`,
+        { url: url.href, revHost: PlacesUtils.getReversedHost(url), guid }
+      );
+
+      await buf.db.executeCached(
+        `
+        INSERT INTO urls(guid, url, hash, revHost)
+        VALUES(:guid, :url, hash(:url), :revHost)`,
+        { guid, url: url.href, revHost: PlacesUtils.getReversedHost(url) }
+      );
+    }
+
+    for (let {
+      guid,
+      parentGuid,
+      placeGuid,
+      localTitle,
+      remoteTitle,
+    } of itemsToInsert) {
+      await buf.db.executeCached(
+        `
+        INSERT INTO moz_bookmarks(guid, parent, fk, position, type, title,
+                                  syncStatus, syncChangeCounter)
+        VALUES(:guid, (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid),
+               (SELECT id FROM moz_places WHERE guid = :placeGuid),
+               (SELECT count(*) FROM moz_bookmarks b
+                JOIN moz_bookmarks p ON p.id = b.parent
+                WHERE p.guid = :parentGuid), :type, :localTitle,
+                :syncStatus, 1)`,
+        {
+          guid,
+          parentGuid,
+          placeGuid,
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          localTitle,
+          syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+        }
+      );
+
+      await buf.db.executeCached(
+        `
+        INSERT INTO items(guid, parentGuid, needsMerge, kind, title, urlId)
+        VALUES(:guid, :parentGuid, 1, :kind, :remoteTitle,
+               (SELECT id FROM urls WHERE guid = :placeGuid))`,
+        {
+          guid,
+          parentGuid,
+          placeGuid,
+          kind: Ci.mozISyncedBookmarksMerger.KIND_BOOKMARK,
+          remoteTitle,
+        }
+      );
+
+      await buf.db.executeCached(
+        `
+        INSERT INTO structure(guid, parentGuid, position)
+        VALUES(:guid, :parentGuid,
+               IFNULL((SELECT count(*) FROM structure
+                       WHERE parentGuid = :parentGuid), 0))`,
+        { guid, parentGuid }
+      );
+    }
+  });
+
+  info("Apply mirror");
+  let observer = expectBookmarkChangeNotifications();
+  let changesToUpload = await buf.apply({
+    notifyInStableOrder: true,
+  });
+  deepEqual(
+    await buf.fetchUnmergedGuids(),
+    [
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.mobileGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+    ],
+    "Should leave roots unmerged"
+  );
+  deepEqual(
+    Object.keys(changesToUpload).sort(),
+    ["menu", "mobile", "toolbar", "unfiled"],
+    "Should upload roots"
+  );
+
+  await assertLocalTree(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+      index: 0,
+      title: "",
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 0,
+          title: BookmarksMenuTitle,
+          children: [
+            {
+              guid: "bookmarkAAAA",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "A (remote)",
+              url: "http://example.com/",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 1,
+          title: BookmarksToolbarTitle,
+          children: [
+            {
+              guid: "bookmarkBBBB",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "B (remote)",
+              url: "http://example.com/",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 3,
+          title: UnfiledBookmarksTitle,
+          children: [
+            {
+              guid: "bookmarkCCCC",
+              type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+              index: 0,
+              title: "C (remote)",
+              url: "http://example.com/c",
+            },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          index: 4,
+          title: MobileBookmarksTitle,
+        },
+      ],
+    },
+    "Should update titles for items with duplicate URLs"
+  );
+
+  let localItemIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+    "bookmarkCCCC",
+  ]);
+  observer.check([
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkAAAA"),
+        title: "A (remote)",
+        guid: "bookmarkAAAA",
+        parentGuid: PlacesUtils.bookmarks.menuGuid,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkBBBB"),
+        title: "B (remote)",
+        guid: "bookmarkBBBB",
+        parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      },
+    },
+    {
+      name: "bookmark-title-changed",
+      params: {
+        itemId: localItemIds.get("bookmarkCCCC"),
+        title: "C (remote)",
+        guid: "bookmarkCCCC",
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      },
+    },
+  ]);
+
+  info("Remove duplicate URLs from Places to avoid tripping debug asserts");
+  await buf.db.executeTransaction(async function () {
+    for (let { guid } of placesToInsert) {
+      await buf.db.executeCached(
+        `
+        DELETE FROM moz_places WHERE guid = :guid`,
+        { guid }
+      );
+    }
+  });
+
+  await storeChangesInMirror(buf, changesToUpload);
+  deepEqual(await buf.fetchUnmergedGuids(), [], "Should merge all items");
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_duplicate_local_tags() {
+  let buf = await openMirror("duplicate_local_tags");
+  let now = new Date();
+
+  info("Insert A");
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkAAAA",
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "A",
+    url: "http://example.com/a",
+    dateAdded: now,
+  });
+
+  // Each tag folder should have unique tag entries, but the tagging service
+  // doesn't enforce this. We should still sync the correct set of tags,
+  // though, even if there are duplicates for the same URL.
+  info("Manually insert local tags for A");
+  for (let [tag, dupes] of [
+    ["one", 2],
+    ["two", 1],
+    ["three", 2],
+  ]) {
+    let tagFolderInfo = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.tagsGuid,
+      title: tag,
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    });
+    for (let i = 0; i < dupes; ++i) {
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: tagFolderInfo.guid,
+        url: "http://example.com/a",
+      });
+    }
+  }
+
+  let tagsForA = PlacesUtils.tagging.getTagsForURI(
+    Services.io.newURI("http://example.com/a")
+  );
+  deepEqual(
+    tagsForA.sort(),
+    ["one", "one", "three", "three", "two"],
+    "Tagging service should return duplicate tags"
+  );
+
+  info("Apply remote");
+  let changesToUpload = await buf.apply();
+  deepEqual(
+    changesToUpload.bookmarkAAAA.cleartext,
+    {
+      id: "bookmarkAAAA",
+      type: "bookmark",
+      parentid: "menu",
+      hasDupe: true,
+      parentName: BookmarksMenuTitle,
+      dateAdded: now.getTime(),
+      bmkUri: "http://example.com/a",
+      title: "A",
+      tags: ["one", "three", "two"],
+    },
+    "Should upload A with tags"
+  );
+
+  await buf.finalize();
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
diff --git a/toolkit/components/places/tests/sync/test_sync_utils.js b/toolkit/components/places/tests/sync/test_sync_utils.js
new file mode 100644
index 0000000000..8b5a893e39
--- /dev/null
+++ b/toolkit/components/places/tests/sync/test_sync_utils.js
@@ -0,0 +1,3074 @@
+ChromeUtils.defineESModuleGetters(this, {
+  Preferences: "resource://gre/modules/Preferences.sys.mjs",
+});
+
+var makeGuid = PlacesUtils.history.makeGuid;
+
+function shuffle(array) {
+  let results = [];
+  for (let i = 0; i < array.length; ++i) {
+    let randomIndex = Math.floor(Math.random() * (i + 1));
+    results[i] = results[randomIndex];
+    results[randomIndex] = array[i];
+  }
+  return results;
+}
+
+async function assertTagForURLs(tag, urls, message) {
+  let taggedURLs = new Set();
+  await PlacesUtils.bookmarks.fetch({ tags: [tag] }, b =>
+    taggedURLs.add(b.url.href)
+  );
+  deepEqual(
+    Array.from(taggedURLs).sort(compareAscending),
+    urls.sort(compareAscending),
+    message
+  );
+}
+
+function assertURLHasTags(url, tags, message) {
+  let actualTags = PlacesUtils.tagging.getTagsForURI(uri(url));
+  deepEqual(actualTags.sort(compareAscending), tags, message);
+}
+
+var populateTree = async function populate(parentGuid, ...items) {
+  let guids = {};
+
+  for (let index = 0; index < items.length; index++) {
+    let item = items[index];
+    let guid = makeGuid();
+
+    switch (item.kind) {
+      case "bookmark":
+      case "query":
+        await PlacesUtils.bookmarks.insert({
+          type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+          url: item.url,
+          title: item.title,
+          parentGuid,
+          guid,
+          index,
+        });
+        break;
+
+      case "separator":
+        await PlacesUtils.bookmarks.insert({
+          type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+          parentGuid,
+          guid,
+        });
+        break;
+
+      case "folder":
+        await PlacesUtils.bookmarks.insert({
+          type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          title: item.title,
+          parentGuid,
+          guid,
+        });
+        if (item.children) {
+          Object.assign(guids, await populate(guid, ...item.children));
+        }
+        break;
+
+      default:
+        throw new Error(`Unsupported item type: ${item.type}`);
+    }
+
+    guids[item.title] = guid;
+  }
+
+  return guids;
+};
+
+var moveSyncedBookmarksToUnsyncedParent = async function () {
+  info("Insert synced bookmarks");
+  let syncedGuids = await populateTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      kind: "folder",
+      title: "folder",
+      children: [
+        {
+          kind: "bookmark",
+          title: "childBmk",
+          url: "https://example.org",
+        },
+      ],
+    },
+    {
+      kind: "bookmark",
+      title: "topBmk",
+      url: "https://example.com",
+    }
+  );
+  // Pretend we've synced each bookmark at least once.
+  await PlacesTestUtils.setBookmarkSyncFields(
+    ...Object.values(syncedGuids).map(guid => ({
+      guid,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+    }))
+  );
+
+  info("Make new folder");
+  let unsyncedFolder = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "unsyncedFolder",
+  });
+
+  info("Move synced bookmarks into unsynced new folder");
+  for (let guid of Object.values(syncedGuids)) {
+    await PlacesUtils.bookmarks.update({
+      guid,
+      parentGuid: unsyncedFolder.guid,
+      index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+    });
+  }
+
+  return { syncedGuids, unsyncedFolder };
+};
+
+var setChangesSynced = async function (changes) {
+  for (let recordId in changes) {
+    changes[recordId].synced = true;
+  }
+  await PlacesSyncUtils.bookmarks.pushChanges(changes);
+};
+
+var ignoreChangedRoots = async function () {
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  let expectedRoots = ["menu", "mobile", "toolbar", "unfiled"];
+  if (!ObjectUtils.deepEqual(Object.keys(changes).sort(), expectedRoots)) {
+    // Make sure the previous test cleaned up.
+    throw new Error(
+      `Unexpected changes at start of test: ${JSON.stringify(changes)}`
+    );
+  }
+  await setChangesSynced(changes);
+};
+
+add_task(async function test_fetchURLFrecency() {
+  // Add visits to the following URLs and then check if frecency for those URLs is not -1.
+  let arrayOfURLsToVisit = [
+    "https://www.mozilla.org/en-US/",
+    "http://getfirefox.com",
+    "http://getthunderbird.com",
+  ];
+  for (let url of arrayOfURLsToVisit) {
+    await PlacesTestUtils.addVisits(url);
+  }
+  for (let url of arrayOfURLsToVisit) {
+    let frecency = await PlacesSyncUtils.history.fetchURLFrecency(url);
+    equal(typeof frecency, "number", "The frecency should be of type: number");
+    notEqual(
+      frecency,
+      -1,
+      "The frecency of this url should be different than -1"
+    );
+  }
+  // Do not add visits to the following URLs, and then check if frecency for those URLs is -1.
+  let arrayOfURLsNotVisited = ["https://bugzilla.org", "https://example.org"];
+  for (let url of arrayOfURLsNotVisited) {
+    let frecency = await PlacesSyncUtils.history.fetchURLFrecency(url);
+    equal(frecency, -1, "The frecency of this url should be -1");
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_determineNonSyncableGuids() {
+  // Add visits to the following URLs with different transition types.
+  let arrayOfVisits = [
+    { uri: "https://www.mozilla.org/en-US/", transition: TRANSITION_TYPED },
+    { uri: "http://getfirefox.com/", transition: TRANSITION_LINK },
+    { uri: "http://getthunderbird.com/", transition: TRANSITION_FRAMED_LINK },
+    { uri: "http://downloads.com/", transition: TRANSITION_DOWNLOAD },
+  ];
+  for (let visit of arrayOfVisits) {
+    await PlacesTestUtils.addVisits(visit);
+  }
+
+  // Fetch the guid for each visit.
+  let guids = [];
+  let dictURLGuid = {};
+  for (let visit of arrayOfVisits) {
+    let guid = await PlacesSyncUtils.history.fetchGuidForURL(visit.uri);
+    guids.push(guid);
+    dictURLGuid[visit.uri] = guid;
+  }
+
+  // Filter the visits.
+  let filteredGuids = await PlacesSyncUtils.history.determineNonSyncableGuids(
+    guids
+  );
+
+  let filtered = [TRANSITION_FRAMED_LINK, TRANSITION_DOWNLOAD];
+  // Check if the filtered visits are of type TRANSITION_FRAMED_LINK.
+  for (let visit of arrayOfVisits) {
+    if (filtered.includes(visit.transition)) {
+      ok(
+        filteredGuids.includes(dictURLGuid[visit.uri]),
+        "This url should be one of the filtered guids."
+      );
+    } else {
+      ok(
+        !filteredGuids.includes(dictURLGuid[visit.uri]),
+        "This url should not be one of the filtered guids."
+      );
+    }
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_changeGuid() {
+  // Add some visits of the following URLs.
+  let arrayOfURLsToVisit = [
+    "https://www.mozilla.org/en-US/",
+    "http://getfirefox.com/",
+    "http://getthunderbird.com/",
+  ];
+  for (let url of arrayOfURLsToVisit) {
+    await PlacesTestUtils.addVisits(url);
+  }
+
+  for (let url of arrayOfURLsToVisit) {
+    let originalGuid = await PlacesSyncUtils.history.fetchGuidForURL(url);
+    let newGuid = makeGuid();
+
+    // Change the original GUID for the new GUID.
+    await PlacesSyncUtils.history.changeGuid(url, newGuid);
+
+    // Fetch the GUID for this URL.
+    let newGuidFetched = await PlacesSyncUtils.history.fetchGuidForURL(url);
+
+    // Check that the URL has the new GUID as its GUID and not the original one.
+    equal(
+      newGuid,
+      newGuidFetched,
+      "These should be equal since we changed the guid for the visit."
+    );
+    notEqual(
+      originalGuid,
+      newGuidFetched,
+      "These should be different since we changed the guid for the visit."
+    );
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_fetchVisitsForURL() {
+  // Get the date for this moment and a date for a minute ago.
+  let now = new Date();
+  let aMinuteAgo = new Date(now.getTime() - 1 * 60000);
+
+  // Add some visits of the following URLs, specifying the transition and the visit date.
+  let arrayOfVisits = [
+    {
+      uri: "https://www.mozilla.org/en-US/",
+      transition: TRANSITION_TYPED,
+      visitDate: aMinuteAgo,
+    },
+    {
+      uri: "http://getfirefox.com/",
+      transition: TRANSITION_LINK,
+      visitDate: aMinuteAgo,
+    },
+    {
+      uri: "http://getthunderbird.com/",
+      transition: TRANSITION_LINK,
+      visitDate: aMinuteAgo,
+    },
+  ];
+  for (let elem of arrayOfVisits) {
+    await PlacesTestUtils.addVisits(elem);
+  }
+
+  for (let elem of arrayOfVisits) {
+    // Fetch all the visits for this URL.
+    let visits = await PlacesSyncUtils.history.fetchVisitsForURL(elem.uri);
+    // Since the visit we added will be the last one in the collection of visits, we get the index of it.
+    let iLast = visits.length - 1;
+
+    // The date is saved in _micro_seconds, here we change it to milliseconds.
+    let dateInMilliseconds = visits[iLast].date * 0.001;
+
+    // Check that the info we provided for this URL is the same one retrieved.
+    equal(
+      dateInMilliseconds,
+      elem.visitDate.getTime(),
+      "The date we provided should be the same we retrieved."
+    );
+    equal(
+      visits[iLast].type,
+      elem.transition,
+      "The transition type we provided should be the same we retrieved."
+    );
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_fetchGuidForURL() {
+  // Add some visits of the following URLs.
+  let arrayOfURLsToVisit = [
+    "https://www.mozilla.org/en-US/",
+    "http://getfirefox.com/",
+    "http://getthunderbird.com/",
+  ];
+  for (let url of arrayOfURLsToVisit) {
+    await PlacesTestUtils.addVisits(url);
+  }
+
+  // This tries to test fetchGuidForURL in two ways:
+  // 1- By fetching the GUID, and then using that GUID to retrieve the info of the visit.
+  //    It then compares the URL with the URL that is on the visits info.
+  // 2- By creating a new GUID, changing the GUID for the visit, fetching the GUID and comparing them.
+  for (let url of arrayOfURLsToVisit) {
+    let guid = await PlacesSyncUtils.history.fetchGuidForURL(url);
+    let info = await PlacesSyncUtils.history.fetchURLInfoForGuid(guid);
+
+    let newGuid = makeGuid();
+    await PlacesSyncUtils.history.changeGuid(url, newGuid);
+    let newGuid2 = await PlacesSyncUtils.history.fetchGuidForURL(url);
+
+    equal(
+      url,
+      info.url,
+      "The url provided and the url retrieved should be the same."
+    );
+    equal(
+      newGuid,
+      newGuid2,
+      "The changed guid and the retrieved guid should be the same."
+    );
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_fetchURLInfoForGuid() {
+  // Add some visits of the following URLs. specifying the title.
+  let visits = [
+    { uri: "https://www.mozilla.org/en-US/", title: "mozilla" },
+    { uri: "http://getfirefox.com/", title: "firefox" },
+    { uri: "http://getthunderbird.com/", title: "thunderbird" },
+    { uri: "http://quantum.mozilla.com/", title: null },
+  ];
+  for (let visit of visits) {
+    await PlacesTestUtils.addVisits(visit);
+  }
+
+  for (let visit of visits) {
+    let guid = await PlacesSyncUtils.history.fetchGuidForURL(visit.uri);
+    let info = await PlacesSyncUtils.history.fetchURLInfoForGuid(guid);
+
+    // Compare the info returned by fetchURLInfoForGuid,
+    // URL and title should match while frecency must be different than -1.
+    equal(
+      info.url,
+      visit.uri,
+      "The url provided should be the same as the url retrieved."
+    );
+    equal(
+      info.title,
+      visit.title || "",
+      "The title provided should be the same as the title retrieved."
+    );
+    notEqual(
+      info.frecency,
+      -1,
+      "The frecency of the visit should be different than -1."
+    );
+  }
+
+  // Create a "fake" GUID and check that the result of fetchURLInfoForGuid is null.
+  let guid = makeGuid();
+  let info = await PlacesSyncUtils.history.fetchURLInfoForGuid(guid);
+
+  equal(
+    info,
+    null,
+    "The information object of a non-existent guid should be null."
+  );
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_getAllURLs() {
+  // Add some visits of the following URLs.
+  let arrayOfURLsToVisit = [
+    "https://www.mozilla.org/en-US/",
+    "http://getfirefox.com/",
+    "http://getthunderbird.com/",
+  ];
+  for (let url of arrayOfURLsToVisit) {
+    await PlacesTestUtils.addVisits(url);
+  }
+
+  // Get all URLs.
+  let allURLs = await PlacesSyncUtils.history.getAllURLs({
+    since: new Date(Date.now() - 2592000000),
+    limit: 5000,
+  });
+
+  // The amount of URLs must be the same in both collections.
+  equal(
+    allURLs.length,
+    arrayOfURLsToVisit.length,
+    "The amount of urls retrived should match the amount of urls provided."
+  );
+
+  // Check that the correct URLs were retrived.
+  for (let url of arrayOfURLsToVisit) {
+    ok(
+      allURLs.includes(url),
+      "The urls retrieved should match the ones used in this test."
+    );
+  }
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_getAllURLs_skips_downloads() {
+  // Add some visits of the following URLs.
+  let arrayOfURLsToVisit = [
+    "https://www.mozilla.org/en-US/",
+    { uri: "http://downloads.com/", transition: TRANSITION_DOWNLOAD },
+  ];
+  for (let url of arrayOfURLsToVisit) {
+    await PlacesTestUtils.addVisits(url);
+  }
+
+  // Get all URLs.
+  let allURLs = await PlacesSyncUtils.history.getAllURLs({
+    since: new Date(Date.now() - 2592000000),
+    limit: 5000,
+  });
+
+  // Should be only the non-download
+  equal(allURLs.length, 1, "Should only get one URL back.");
+
+  // Check that the correct URLs were retrived.
+  equal(allURLs[0], arrayOfURLsToVisit[0], "Should get back our non-download.");
+
+  // Remove the visits added during this test.
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_order() {
+  info("Insert some bookmarks");
+  let guids = await populateTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      kind: "bookmark",
+      title: "childBmk",
+      url: "http://getfirefox.com",
+    },
+    {
+      kind: "bookmark",
+      title: "siblingBmk",
+      url: "http://getthunderbird.com",
+    },
+    {
+      kind: "folder",
+      title: "siblingFolder",
+    },
+    {
+      kind: "separator",
+      title: "siblingSep",
+    }
+  );
+
+  info("Reorder inserted bookmarks");
+  {
+    let order = [
+      guids.siblingFolder,
+      guids.siblingSep,
+      guids.childBmk,
+      guids.siblingBmk,
+    ];
+    await PlacesSyncUtils.bookmarks.order(
+      PlacesUtils.bookmarks.menuGuid,
+      order
+    );
+    let childRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+      PlacesUtils.bookmarks.menuGuid
+    );
+    deepEqual(
+      childRecordIds,
+      order,
+      "New bookmarks should be reordered according to array"
+    );
+  }
+
+  info("Same order with unspecified children");
+  {
+    await PlacesSyncUtils.bookmarks.order(PlacesUtils.bookmarks.menuGuid, [
+      guids.siblingSep,
+      guids.siblingBmk,
+    ]);
+    let childRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+      PlacesUtils.bookmarks.menuGuid
+    );
+    deepEqual(
+      childRecordIds,
+      [guids.siblingFolder, guids.siblingSep, guids.childBmk, guids.siblingBmk],
+      "Current order should be respected if possible"
+    );
+  }
+
+  info("New order with unspecified children");
+  {
+    await PlacesSyncUtils.bookmarks.order(PlacesUtils.bookmarks.menuGuid, [
+      guids.siblingBmk,
+      guids.siblingSep,
+    ]);
+    let childRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+      PlacesUtils.bookmarks.menuGuid
+    );
+    deepEqual(
+      childRecordIds,
+      [guids.siblingBmk, guids.siblingSep, guids.siblingFolder, guids.childBmk],
+      "Unordered children should be moved to end if current order can't be respected"
+    );
+  }
+
+  info("Reorder with nonexistent children");
+  {
+    await PlacesSyncUtils.bookmarks.order(PlacesUtils.bookmarks.menuGuid, [
+      guids.childBmk,
+      makeGuid(),
+      guids.siblingBmk,
+      guids.siblingSep,
+      makeGuid(),
+      guids.siblingFolder,
+      makeGuid(),
+    ]);
+    let childRecordIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+      PlacesUtils.bookmarks.menuGuid
+    );
+    deepEqual(
+      childRecordIds,
+      [guids.childBmk, guids.siblingBmk, guids.siblingSep, guids.siblingFolder],
+      "Nonexistent children should be ignored"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_order_roots() {
+  let oldOrder = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+    PlacesUtils.bookmarks.rootGuid
+  );
+  await PlacesSyncUtils.bookmarks.order(
+    PlacesUtils.bookmarks.rootGuid,
+    shuffle(oldOrder)
+  );
+  let newOrder = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+    PlacesUtils.bookmarks.rootGuid
+  );
+  deepEqual(oldOrder, newOrder, "Should ignore attempts to reorder roots");
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_tags() {
+  await ignoreChangedRoots();
+
+  info("Insert untagged items with same URL");
+  let firstItem = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    url: "https://example.org",
+  });
+  let secondItem = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    url: "https://example.org",
+  });
+  let untaggedItem = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    url: "https://bugzilla.org",
+  });
+  let taggedItem = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    url: "https://mozilla.org",
+  });
+
+  info("Create tag");
+  PlacesUtils.tagging.tagURI(uri("https://example.org"), ["taggy"]);
+
+  let tagBm = await PlacesUtils.bookmarks.fetch({
+    parentGuid: PlacesUtils.bookmarks.tagsGuid,
+    index: 0,
+  });
+  let tagFolderGuid = tagBm.guid;
+  let tagFolderId = await PlacesUtils.promiseItemId(tagFolderGuid);
+
+  info("Tagged bookmarks should be in changeset");
+  {
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId].sort(),
+      "Should include tagged bookmarks in changeset"
+    );
+    await setChangesSynced(changes);
+  }
+
+  info("Change tag case");
+  {
+    PlacesUtils.tagging.tagURI(uri("https://mozilla.org"), ["TaGgY"]);
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId, taggedItem.recordId].sort(),
+      "Should include tagged bookmarks after changing case"
+    );
+    await assertTagForURLs(
+      "TaGgY",
+      ["https://example.org/", "https://mozilla.org/"],
+      "Should add tag for new URL"
+    );
+    await setChangesSynced(changes);
+  }
+
+  // These tests change a tag item directly, without going through the tagging
+  // service. This behavior isn't supported, but the tagging service registers
+  // an observer to handle these cases, so we make sure we handle them
+  // correctly.
+
+  info("Rename tag folder using Bookmarks.setItemTitle");
+  {
+    PlacesUtils.bookmarks.setItemTitle(tagFolderId, "sneaky");
+    deepEqual(
+      (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+      ["sneaky"],
+      "Tagging service should update cache with new title"
+    );
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId].sort(),
+      "Should include tagged bookmarks after renaming tag folder"
+    );
+    await setChangesSynced(changes);
+  }
+
+  info("Rename tag folder using Bookmarks.update");
+  {
+    await PlacesUtils.bookmarks.update({
+      guid: tagFolderGuid,
+      title: "tricky",
+    });
+    deepEqual(
+      (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+      ["tricky"],
+      "Tagging service should update cache after updating tag folder"
+    );
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId].sort(),
+      "Should include tagged bookmarks after updating tag folder"
+    );
+    await setChangesSynced(changes);
+  }
+
+  info("Change tag entry URL using Bookmarks.update");
+  {
+    let bm = await PlacesUtils.bookmarks.fetch({
+      parentGuid: tagFolderGuid,
+      index: 0,
+    });
+    bm.url = "https://bugzilla.org/";
+    await PlacesUtils.bookmarks.update(bm);
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId, untaggedItem.recordId].sort(),
+      "Should include tagged bookmarks after changing tag entry URI"
+    );
+    await assertTagForURLs(
+      "tricky",
+      ["https://bugzilla.org/", "https://mozilla.org/"],
+      "Should remove tag entry for old URI"
+    );
+    await setChangesSynced(changes);
+
+    bm.url = "https://example.org/";
+    await PlacesUtils.bookmarks.update(bm);
+    changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [firstItem.recordId, secondItem.recordId, untaggedItem.recordId].sort(),
+      "Should include tagged bookmarks after changing tag entry URL"
+    );
+    await assertTagForURLs(
+      "tricky",
+      ["https://example.org/", "https://mozilla.org/"],
+      "Should remove tag entry for old URL"
+    );
+    await setChangesSynced(changes);
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_conflicting_keywords() {
+  await ignoreChangedRoots();
+
+  info("Insert bookmark with new keyword");
+  let tbBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "unfiled",
+    url: "http://getthunderbird.com",
+    keyword: "tbird",
+  });
+  {
+    let entryByKeyword = await PlacesUtils.keywords.fetch("tbird");
+    equal(
+      entryByKeyword.url.href,
+      "http://getthunderbird.com/",
+      "Should return new keyword entry by URL"
+    );
+    let entryByURL = await PlacesUtils.keywords.fetch({
+      url: "http://getthunderbird.com",
+    });
+    equal(entryByURL.keyword, "tbird", "Should return new entry by keyword");
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      changes,
+      {},
+      "Should not bump change counter for new keyword entry"
+    );
+  }
+
+  info("Insert bookmark with same URL and different keyword");
+  let dupeTbBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    recordId: makeGuid(),
+    parentRecordId: "toolbar",
+    url: "http://getthunderbird.com",
+    keyword: "tb",
+  });
+  {
+    let oldKeywordByURL = await PlacesUtils.keywords.fetch("tbird");
+    ok(
+      !oldKeywordByURL,
+      "Should remove old entry when inserting bookmark with different keyword"
+    );
+    let entryByKeyword = await PlacesUtils.keywords.fetch("tb");
+    equal(
+      entryByKeyword.url.href,
+      "http://getthunderbird.com/",
+      "Should return different keyword entry by URL"
+    );
+    let entryByURL = await PlacesUtils.keywords.fetch({
+      url: "http://getthunderbird.com",
+    });
+    equal(entryByURL.keyword, "tb", "Should return different entry by keyword");
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [tbBmk.recordId, dupeTbBmk.recordId].sort(),
+      "Should bump change counter for bookmarks with different keyword"
+    );
+    await setChangesSynced(changes);
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_insert() {
+  info("Insert bookmark");
+  {
+    let item = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "bookmark",
+      recordId: makeGuid(),
+      parentRecordId: "menu",
+      url: "https://example.org",
+    });
+    let { type } = await PlacesUtils.bookmarks.fetch({ guid: item.recordId });
+    equal(
+      type,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      "Bookmark should have correct type"
+    );
+  }
+
+  info("Insert query");
+  {
+    let item = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "query",
+      recordId: makeGuid(),
+      parentRecordId: "menu",
+      url: "place:terms=term&folder=TOOLBAR&queryType=1",
+      folder: "Saved search",
+    });
+    let { type } = await PlacesUtils.bookmarks.fetch({ guid: item.recordId });
+    equal(
+      type,
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      "Queries should be stored as bookmarks"
+    );
+  }
+
+  info("Insert folder");
+  {
+    let item = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "folder",
+      recordId: makeGuid(),
+      parentRecordId: "menu",
+      title: "New folder",
+    });
+    let { type } = await PlacesUtils.bookmarks.fetch({ guid: item.recordId });
+    equal(
+      type,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      "Folder should have correct type"
+    );
+  }
+
+  info("Insert separator");
+  {
+    let item = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "separator",
+      recordId: makeGuid(),
+      parentRecordId: "menu",
+    });
+    let { type } = await PlacesUtils.bookmarks.fetch({ guid: item.recordId });
+    equal(
+      type,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      "Separator should have correct type"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_insert_tags() {
+  await Promise.all(
+    [
+      {
+        kind: "bookmark",
+        url: "https://example.com",
+        recordId: makeGuid(),
+        parentRecordId: "menu",
+        tags: ["foo", "bar"],
+      },
+      {
+        kind: "bookmark",
+        url: "https://example.org",
+        recordId: makeGuid(),
+        parentRecordId: "toolbar",
+        tags: ["foo", "baz"],
+      },
+      {
+        kind: "query",
+        url: "place:queryType=1&sort=12&maxResults=10",
+        recordId: makeGuid(),
+        parentRecordId: "toolbar",
+        folder: "bar",
+        tags: ["baz", "qux"],
+        title: "bar",
+      },
+    ].map(info => PlacesSyncUtils.test.bookmarks.insert(info))
+  );
+
+  await assertTagForURLs(
+    "foo",
+    ["https://example.com/", "https://example.org/"],
+    "2 URLs with new tag"
+  );
+  await assertTagForURLs(
+    "bar",
+    ["https://example.com/"],
+    "1 URL with existing tag"
+  );
+  await assertTagForURLs(
+    "baz",
+    ["https://example.org/", "place:queryType=1&sort=12&maxResults=10"],
+    "Should support tagging URLs and tag queries"
+  );
+  await assertTagForURLs(
+    "qux",
+    ["place:queryType=1&sort=12&maxResults=10"],
+    "Should support tagging tag queries"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_insert_tags_whitespace() {
+  info("Untrimmed and blank tags");
+  let taggedBlanks = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    url: "https://example.org",
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    tags: [" untrimmed ", " ", "taggy"],
+  });
+  deepEqual(
+    taggedBlanks.tags,
+    ["untrimmed", "taggy"],
+    "Should not return empty tags"
+  );
+  assertURLHasTags(
+    "https://example.org/",
+    ["taggy", "untrimmed"],
+    "Should set trimmed tags and ignore dupes"
+  );
+
+  info("Dupe tags");
+  let taggedDupes = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    url: "https://example.net",
+    recordId: makeGuid(),
+    parentRecordId: "toolbar",
+    tags: [" taggy", "taggy ", " taggy ", "taggy"],
+  });
+  deepEqual(
+    taggedDupes.tags,
+    ["taggy", "taggy", "taggy", "taggy"],
+    "Should return trimmed and dupe tags"
+  );
+  assertURLHasTags(
+    "https://example.net/",
+    ["taggy"],
+    "Should ignore dupes when setting tags"
+  );
+
+  await assertTagForURLs(
+    "taggy",
+    ["https://example.net/", "https://example.org/"],
+    "Should exclude falsy tags"
+  );
+
+  PlacesUtils.tagging.untagURI(uri("https://example.org"), [
+    "untrimmed",
+    "taggy",
+  ]);
+  PlacesUtils.tagging.untagURI(uri("https://example.net"), ["taggy"]);
+  deepEqual(
+    (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+    [],
+    "Should clean up all tags"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_insert_keyword() {
+  info("Insert item with new keyword");
+  {
+    await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "bookmark",
+      parentRecordId: "menu",
+      url: "https://example.com",
+      keyword: "moz",
+      recordId: makeGuid(),
+    });
+    let entry = await PlacesUtils.keywords.fetch("moz");
+    equal(
+      entry.url.href,
+      "https://example.com/",
+      "Should add keyword for item"
+    );
+  }
+
+  info("Insert item with existing keyword");
+  {
+    await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "bookmark",
+      parentRecordId: "menu",
+      url: "https://mozilla.org",
+      keyword: "moz",
+      recordId: makeGuid(),
+    });
+    let entry = await PlacesUtils.keywords.fetch("moz");
+    equal(
+      entry.url.href,
+      "https://mozilla.org/",
+      "Should reassign keyword to new item"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_insert_tag_query() {
+  info("Use the public tagging API to ensure we added the tag correctly");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    url: "https://mozilla.org",
+    title: "Mozilla",
+  });
+  PlacesUtils.tagging.tagURI(uri("https://mozilla.org"), ["taggy"]);
+  assertURLHasTags(
+    "https://mozilla.org/",
+    ["taggy"],
+    "Should set tags using the tagging API"
+  );
+
+  info("Insert tag query for non existing tag");
+  {
+    let query = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "query",
+      recordId: makeGuid(),
+      parentRecordId: "toolbar",
+      url: "place:type=7&folder=90",
+      folder: "nonexisting",
+      title: "Tagged stuff",
+    });
+    let params = new URLSearchParams(query.url.pathname);
+    ok(!params.has("type"), "Should not preserve query type");
+    ok(!params.has("folder"), "Should not preserve folder");
+    equal(params.get("tag"), "nonexisting", "Should add tag");
+    deepEqual(
+      (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+      ["taggy"],
+      "The nonexisting tag should not be added"
+    );
+  }
+
+  info("Insert tag query for existing tag");
+  {
+    let url = "place:type=7&folder=90&maxResults=15";
+    let query = await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "query",
+      recordId: makeGuid(),
+      parentRecordId: "menu",
+      url,
+      folder: "taggy",
+      title: "Sorted and tagged",
+    });
+    let params = new URLSearchParams(query.url.pathname);
+    ok(!params.get("type"), "Should not preserve query type");
+    ok(!params.has("folder"), "Should not preserve folder");
+    equal(params.get("maxResults"), "15", "Should preserve additional params");
+    equal(params.get("tag"), "taggy", "Should add tag");
+    deepEqual(
+      (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+      ["taggy"],
+      "Should not duplicate existing tags"
+    );
+  }
+
+  info("Removing the tag should clean up the tag folder");
+  PlacesUtils.tagging.untagURI(uri("https://mozilla.org"), null);
+  deepEqual(
+    (await PlacesUtils.bookmarks.fetchTags()).map(t => t.name),
+    [],
+    "Should remove tag folder once last item is untagged"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_fetch() {
+  let folder = await PlacesSyncUtils.test.bookmarks.insert({
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    kind: "folder",
+  });
+  let bmk = await PlacesSyncUtils.test.bookmarks.insert({
+    recordId: makeGuid(),
+    parentRecordId: "menu",
+    kind: "bookmark",
+    url: "https://example.com",
+    tags: ["taggy"],
+  });
+  let folderBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    recordId: makeGuid(),
+    parentRecordId: folder.recordId,
+    kind: "bookmark",
+    url: "https://example.org",
+    keyword: "kw",
+  });
+  let folderSep = await PlacesSyncUtils.test.bookmarks.insert({
+    recordId: makeGuid(),
+    parentRecordId: folder.recordId,
+    kind: "separator",
+  });
+  let tagQuery = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "query",
+    recordId: makeGuid(),
+    parentRecordId: "toolbar",
+    url: "place:tag=taggy",
+    folder: "taggy",
+    title: "Tagged stuff",
+  });
+
+  info("Fetch empty folder");
+  {
+    let item = await PlacesSyncUtils.bookmarks.fetch(folder.recordId);
+    deepEqual(
+      item,
+      {
+        recordId: folder.recordId,
+        kind: "folder",
+        parentRecordId: "menu",
+        childRecordIds: [folderBmk.recordId, folderSep.recordId],
+        parentTitle: "menu",
+        dateAdded: item.dateAdded,
+        title: "",
+      },
+      "Should include children, title, and parent title in folder"
+    );
+  }
+
+  info("Fetch bookmark with tags");
+  {
+    let item = await PlacesSyncUtils.bookmarks.fetch(bmk.recordId);
+    deepEqual(
+      Object.keys(item).sort(),
+      [
+        "recordId",
+        "kind",
+        "parentRecordId",
+        "url",
+        "tags",
+        "parentTitle",
+        "title",
+        "dateAdded",
+      ].sort(),
+      "Should include bookmark-specific properties"
+    );
+    equal(item.recordId, bmk.recordId, "Sync ID should match");
+    equal(item.url.href, "https://example.com/", "Should return URL");
+    equal(item.parentRecordId, "menu", "Should return parent sync ID");
+    deepEqual(item.tags, ["taggy"], "Should return tags");
+    equal(item.parentTitle, "menu", "Should return parent title");
+    strictEqual(item.title, "", "Should return empty title");
+  }
+
+  info("Fetch bookmark with keyword; without parent title");
+  {
+    let item = await PlacesSyncUtils.bookmarks.fetch(folderBmk.recordId);
+    deepEqual(
+      Object.keys(item).sort(),
+      [
+        "recordId",
+        "kind",
+        "parentRecordId",
+        "url",
+        "keyword",
+        "tags",
+        "parentTitle",
+        "title",
+        "dateAdded",
+      ].sort(),
+      "Should omit blank bookmark-specific properties"
+    );
+    deepEqual(item.tags, [], "Tags should be empty");
+    equal(item.keyword, "kw", "Should return keyword");
+    strictEqual(
+      item.parentTitle,
+      "",
+      "Should include parent title even if empty"
+    );
+    strictEqual(item.title, "", "Should include bookmark title even if empty");
+  }
+
+  info("Fetch separator");
+  {
+    let item = await PlacesSyncUtils.bookmarks.fetch(folderSep.recordId);
+    strictEqual(item.index, 1, "Should return separator position");
+  }
+
+  info("Fetch tag query");
+  {
+    let item = await PlacesSyncUtils.bookmarks.fetch(tagQuery.recordId);
+    deepEqual(
+      Object.keys(item).sort(),
+      [
+        "recordId",
+        "kind",
+        "parentRecordId",
+        "url",
+        "title",
+        "folder",
+        "parentTitle",
+        "dateAdded",
+      ].sort(),
+      "Should include query-specific properties"
+    );
+    equal(
+      item.url.href,
+      `place:tag=taggy`,
+      "Should not rewrite outgoing tag queries"
+    );
+    equal(item.folder, "taggy", "Should return tag name for tag queries");
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_new_parent() {
+  await ignoreChangedRoots();
+
+  let { syncedGuids, unsyncedFolder } =
+    await moveSyncedBookmarksToUnsyncedParent();
+
+  info("Unsynced parent and synced items should be tracked");
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(changes).sort(),
+    [
+      syncedGuids.folder,
+      syncedGuids.topBmk,
+      syncedGuids.childBmk,
+      unsyncedFolder.guid,
+      "menu",
+    ].sort(),
+    "Should return change records for moved items and new parent"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_deleted_folder() {
+  await ignoreChangedRoots();
+
+  let { syncedGuids, unsyncedFolder } =
+    await moveSyncedBookmarksToUnsyncedParent();
+
+  info("Remove unsynced new folder");
+  await PlacesUtils.bookmarks.remove(unsyncedFolder.guid);
+
+  info("Deleted synced items should be tracked; unsynced folder should not");
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(changes).sort(),
+    [
+      syncedGuids.folder,
+      syncedGuids.topBmk,
+      syncedGuids.childBmk,
+      "menu",
+    ].sort(),
+    "Should return change records for all deleted items"
+  );
+  for (let guid of Object.values(syncedGuids)) {
+    strictEqual(
+      changes[guid].tombstone,
+      true,
+      `Tombstone flag should be set for deleted item ${guid}`
+    );
+    equal(
+      changes[guid].counter,
+      1,
+      `Change counter should be 1 for deleted item ${guid}`
+    );
+    equal(
+      changes[guid].status,
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      `Sync status should be normal for deleted item ${guid}`
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_import_html() {
+  await ignoreChangedRoots();
+
+  info("Add unsynced bookmark");
+  let unsyncedBmk = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "https://example.com",
+  });
+
+  {
+    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      unsyncedBmk.guid
+    );
+    ok(
+      fields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+      ),
+      "Unsynced bookmark statuses should match"
+    );
+  }
+
+  info("Import new bookmarks from HTML");
+  let { path } = do_get_file("./sync_utils_bookmarks.html");
+  await BookmarkHTMLUtils.importFromFile(path);
+
+  // Bookmarks.html doesn't store IDs, so we need to look these up.
+  let mozBmk = await PlacesUtils.bookmarks.fetch({
+    url: "https://www.mozilla.org/",
+  });
+  let fxBmk = await PlacesUtils.bookmarks.fetch({
+    url: "https://www.mozilla.org/en-US/firefox/",
+  });
+  // All Bookmarks.html bookmarks are stored under the menu. For toolbar
+  // bookmarks, this means they're imported into a "Bookmarks Toolbar"
+  // subfolder under the menu, instead of the real toolbar root.
+  let toolbarSubfolder = (
+    await PlacesUtils.bookmarks.search({
+      title: "Bookmarks Toolbar",
+    })
+  ).find(item => item.guid != PlacesUtils.bookmarks.toolbarGuid);
+  let importedFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    mozBmk.guid,
+    fxBmk.guid,
+    toolbarSubfolder.guid
+  );
+  ok(
+    importedFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Sync statuses should match for HTML imports"
+  );
+
+  info("Fetch new HTML imports");
+  let newChanges = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(newChanges).sort(),
+    [
+      mozBmk.guid,
+      fxBmk.guid,
+      toolbarSubfolder.guid,
+      "menu",
+      unsyncedBmk.guid,
+    ].sort(),
+    "Should return new IDs imported from HTML file"
+  );
+  let newFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    unsyncedBmk.guid,
+    mozBmk.guid,
+    fxBmk.guid,
+    toolbarSubfolder.guid
+  );
+  ok(
+    newFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Pulling new HTML imports should not mark them as syncing"
+  );
+
+  info("Mark new HTML imports as syncing");
+  await PlacesSyncUtils.bookmarks.markChangesAsSyncing(newChanges);
+  let normalFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    unsyncedBmk.guid,
+    mozBmk.guid,
+    fxBmk.guid,
+    toolbarSubfolder.guid
+  );
+  ok(
+    normalFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    ),
+    "Marking new HTML imports as syncing should update their statuses"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_import_json() {
+  await ignoreChangedRoots();
+
+  info("Add synced folder");
+  let syncedFolder = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "syncedFolder",
+  });
+  await PlacesTestUtils.setBookmarkSyncFields({
+    guid: syncedFolder.guid,
+    syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+  });
+
+  info("Import new bookmarks from JSON");
+  let { path } = do_get_file("./sync_utils_bookmarks.json");
+  await BookmarkJSONUtils.importFromFile(path);
+  {
+    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      syncedFolder.guid,
+      "NnvGl3CRA4hC",
+      "APzP8MupzA8l"
+    );
+    deepEqual(
+      fields.map(field => field.syncStatus),
+      [
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+        PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      ],
+      "Sync statuses should match for JSON imports"
+    );
+  }
+
+  info("Fetch new JSON imports");
+  let newChanges = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(newChanges).sort(),
+    [
+      "NnvGl3CRA4hC",
+      "APzP8MupzA8l",
+      "menu",
+      "toolbar",
+      syncedFolder.guid,
+    ].sort(),
+    "Should return items imported from JSON backup"
+  );
+  let existingFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    syncedFolder.guid,
+    "NnvGl3CRA4hC",
+    "APzP8MupzA8l"
+  );
+  deepEqual(
+    existingFields.map(field => field.syncStatus),
+    [
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+      PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+    ],
+    "Pulling new JSON imports should not mark them as syncing"
+  );
+
+  info("Mark new JSON imports as syncing");
+  await PlacesSyncUtils.bookmarks.markChangesAsSyncing(newChanges);
+  let normalFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    syncedFolder.guid,
+    "NnvGl3CRA4hC",
+    "APzP8MupzA8l"
+  );
+  ok(
+    normalFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    ),
+    "Marking new JSON imports as syncing should update their statuses"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_restore_json_tracked() {
+  await ignoreChangedRoots();
+
+  let unsyncedBmk = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "https://example.com",
+  });
+  info(`Unsynced bookmark GUID: ${unsyncedBmk.guid}`);
+  let syncedFolder = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "syncedFolder",
+  });
+  info(`Synced folder GUID: ${syncedFolder.guid}`);
+  await PlacesTestUtils.setBookmarkSyncFields({
+    guid: syncedFolder.guid,
+    syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+  });
+  {
+    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      unsyncedBmk.guid,
+      syncedFolder.guid
+    );
+    deepEqual(
+      fields.map(field => field.syncStatus),
+      [
+        PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      ],
+      "Sync statuses should match before restoring from JSON"
+    );
+  }
+
+  info("Restore from JSON, replacing existing items");
+  let { path } = do_get_file("./sync_utils_bookmarks.json");
+  await BookmarkJSONUtils.importFromFile(path, { replace: true });
+  {
+    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      "NnvGl3CRA4hC",
+      "APzP8MupzA8l"
+    );
+    ok(
+      fields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+      ),
+      "All bookmarks should be NEW after restoring from JSON"
+    );
+  }
+
+  info("Fetch new items restored from JSON");
+  {
+    let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+    deepEqual(
+      Object.keys(changes).sort(),
+      [
+        "menu",
+        "toolbar",
+        "unfiled",
+        "mobile",
+        "NnvGl3CRA4hC",
+        "APzP8MupzA8l",
+      ].sort(),
+      "Should restore items from JSON backup"
+    );
+
+    let existingFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+      PlacesUtils.bookmarks.mobileGuid,
+      "NnvGl3CRA4hC",
+      "APzP8MupzA8l"
+    );
+    ok(
+      existingFields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+      ),
+      "Items restored from JSON backup should not be marked as syncing"
+    );
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones,
+      [],
+      "Tombstones should not exist after restoring from JSON backup"
+    );
+
+    await PlacesSyncUtils.bookmarks.markChangesAsSyncing(changes);
+    let normalFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+      "NnvGl3CRA4hC",
+      "APzP8MupzA8l"
+    );
+    ok(
+      normalFields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+      ),
+      "Roots and NEW items restored from JSON backup should be marked as NORMAL"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pullChanges_tombstones() {
+  await ignoreChangedRoots();
+
+  info("Insert new bookmarks");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      },
+    ],
+  });
+
+  info("Manually insert conflicting tombstone for new bookmark");
+  await PlacesUtils.withConnectionWrapper(
+    "test_pullChanges_tombstones",
+    async function (db) {
+      await db.executeCached(
+        `
+        INSERT INTO moz_bookmarks_deleted(guid)
+        VALUES(:guid)`,
+        { guid: "bookmarkAAAA" }
+      );
+    }
+  );
+
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(changes).sort(),
+    ["bookmarkAAAA", "bookmarkBBBB", "menu"],
+    "Should handle undeleted items when returning changes"
+  );
+  strictEqual(
+    changes.bookmarkAAAA.tombstone,
+    false,
+    "Should replace tombstone for A with undeleted item"
+  );
+  strictEqual(
+    changes.bookmarkBBBB.tombstone,
+    false,
+    "Should not report B as deleted"
+  );
+
+  await setChangesSynced(changes);
+
+  let newChanges = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    newChanges,
+    {},
+    "Should not return changes after marking undeleted items as synced"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_pushChanges() {
+  await ignoreChangedRoots();
+
+  info("Populate test bookmarks");
+  let guids = await populateTree(
+    PlacesUtils.bookmarks.menuGuid,
+    {
+      kind: "bookmark",
+      title: "unknownBmk",
+      url: "https://example.org",
+    },
+    {
+      kind: "bookmark",
+      title: "syncedBmk",
+      url: "https://example.com",
+    },
+    {
+      kind: "bookmark",
+      title: "newBmk",
+      url: "https://example.info",
+    },
+    {
+      kind: "bookmark",
+      title: "deletedBmk",
+      url: "https://example.edu",
+    },
+    {
+      kind: "bookmark",
+      title: "unchangedBmk",
+      url: "https://example.systems",
+    }
+  );
+
+  info("Update sync statuses");
+  await PlacesTestUtils.setBookmarkSyncFields(
+    {
+      guid: guids.syncedBmk,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+    },
+    {
+      guid: guids.unknownBmk,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN,
+    },
+    {
+      guid: guids.deletedBmk,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+    },
+    {
+      guid: guids.unchangedBmk,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      syncChangeCounter: 0,
+    }
+  );
+
+  info("Change synced bookmark; should bump change counter");
+  await PlacesUtils.bookmarks.update({
+    guid: guids.syncedBmk,
+    url: "https://example.ninja",
+  });
+
+  info("Remove synced bookmark");
+  {
+    await PlacesUtils.bookmarks.remove(guids.deletedBmk);
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    ok(
+      tombstones.some(({ guid }) => guid == guids.deletedBmk),
+      "Should write tombstone for deleted synced bookmark"
+    );
+  }
+
+  info("Pull changes");
+  let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges;
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  {
+    let actualChanges = Object.entries(changes).map(([recordId, change]) => ({
+      recordId,
+      syncChangeCounter: change.counter,
+    }));
+    let expectedChanges = [
+      {
+        recordId: guids.unknownBmk,
+        syncChangeCounter: 1,
+      },
+      {
+        // Parent of changed bookmarks.
+        recordId: "menu",
+        syncChangeCounter: 6,
+      },
+      {
+        recordId: guids.syncedBmk,
+        syncChangeCounter: 2,
+      },
+      {
+        recordId: guids.newBmk,
+        syncChangeCounter: 1,
+      },
+      {
+        recordId: guids.deletedBmk,
+        syncChangeCounter: 1,
+      },
+    ];
+    deepEqual(
+      sortBy(actualChanges, "recordId"),
+      sortBy(expectedChanges, "recordId"),
+      "Should return deleted, new, and unknown bookmarks"
+    );
+  }
+
+  info("Modify changed bookmark to bump its counter");
+  await PlacesUtils.bookmarks.update({
+    guid: guids.newBmk,
+    url: "https://example.club",
+  });
+
+  info("Mark some bookmarks as synced");
+  for (let title of ["unknownBmk", "newBmk", "deletedBmk"]) {
+    let guid = guids[title];
+    strictEqual(
+      changes[guid].synced,
+      false,
+      "All bookmarks should not be marked as synced yet"
+    );
+    changes[guid].synced = true;
+  }
+
+  await PlacesSyncUtils.bookmarks.pushChanges(changes);
+  equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 4);
+
+  {
+    let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      guids.newBmk,
+      guids.unknownBmk
+    );
+    ok(
+      fields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+      ),
+      "Should update sync statuses for synced bookmarks"
+    );
+  }
+
+  {
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    ok(
+      !tombstones.some(({ guid }) => guid == guids.deletedBmk),
+      "Should remove tombstone after syncing"
+    );
+
+    let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      guids.unknownBmk,
+      guids.syncedBmk,
+      guids.newBmk
+    );
+    {
+      let info = syncFields.find(field => field.guid == guids.unknownBmk);
+      equal(
+        info.syncStatus,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        "Syncing an UNKNOWN bookmark should set its sync status to NORMAL"
+      );
+      strictEqual(
+        info.syncChangeCounter,
+        0,
+        "Syncing an UNKNOWN bookmark should reduce its change counter"
+      );
+    }
+    {
+      let info = syncFields.find(field => field.guid == guids.syncedBmk);
+      equal(
+        info.syncStatus,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        "Syncing a NORMAL bookmark should not update its sync status"
+      );
+      equal(
+        info.syncChangeCounter,
+        2,
+        "Should not reduce counter for NORMAL bookmark not marked as synced"
+      );
+    }
+    {
+      let info = syncFields.find(field => field.guid == guids.newBmk);
+      equal(
+        info.syncStatus,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        "Syncing a NEW bookmark should update its sync status"
+      );
+      strictEqual(
+        info.syncChangeCounter,
+        1,
+        "Updating new bookmark after pulling changes should bump change counter"
+      );
+    }
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_changes_between_pull_and_push() {
+  await ignoreChangedRoots();
+
+  info("Populate test bookmarks");
+  let guids = await populateTree(PlacesUtils.bookmarks.menuGuid, {
+    kind: "bookmark",
+    title: "bmk",
+    url: "https://example.info",
+  });
+
+  info("Update sync statuses");
+  await PlacesTestUtils.setBookmarkSyncFields({
+    guid: guids.bmk,
+    syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+    syncChangeCounter: 1,
+  });
+
+  info("Pull changes");
+  let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges;
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  Assert.equal(changes[guids.bmk].counter, 1);
+  Assert.equal(changes[guids.bmk].tombstone, false);
+
+  // delete the bookmark.
+  await PlacesUtils.bookmarks.remove(guids.bmk);
+
+  info("Push changes");
+  await PlacesSyncUtils.bookmarks.pushChanges(changes);
+  equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2);
+
+  // we should have a tombstone.
+  let ts = await PlacesTestUtils.fetchSyncTombstones();
+  Assert.equal(ts.length, 1);
+  Assert.equal(ts[0].guid, guids.bmk);
+
+  // there should be no record for the item we deleted.
+  Assert.strictEqual(await PlacesUtils.bookmarks.fetch(guids.bmk), null);
+
+  // and re-fetching changes should list it as a tombstone.
+  changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  Assert.equal(changes[guids.bmk].counter, 1);
+  Assert.equal(changes[guids.bmk].tombstone, true);
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_separator() {
+  await ignoreChangedRoots();
+
+  await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: "menu",
+    recordId: makeGuid(),
+    url: "https://example.com",
+  });
+  let childBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: "menu",
+    recordId: makeGuid(),
+    url: "https://foo.bar",
+  });
+  let separatorRecordId = makeGuid();
+  let separator = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "separator",
+    parentRecordId: "menu",
+    recordId: separatorRecordId,
+  });
+  await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: "menu",
+    recordId: makeGuid(),
+    url: "https://bar.foo",
+  });
+
+  let child2Guid = await PlacesSyncUtils.bookmarks.recordIdToGuid(
+    childBmk.recordId
+  );
+  let parentGuid = await await PlacesSyncUtils.bookmarks.recordIdToGuid("menu");
+  let separatorGuid =
+    PlacesSyncUtils.bookmarks.recordIdToGuid(separatorRecordId);
+
+  info("Move a bookmark around the separator");
+  await PlacesUtils.bookmarks.update({
+    guid: child2Guid,
+    parentGuid,
+    index: 2,
+  });
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(Object.keys(changes).sort(), [separator.recordId, "menu"].sort());
+
+  await setChangesSynced(changes);
+
+  info("Move a separator around directly");
+  await PlacesUtils.bookmarks.update({
+    guid: separatorGuid,
+    parentGuid,
+    index: 0,
+  });
+
+  changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(Object.keys(changes).sort(), [separator.recordId, "menu"].sort());
+
+  await setChangesSynced(changes);
+
+  info("Move a separator around directly using update");
+  await PlacesUtils.bookmarks.update({ guid: separatorGuid, index: 2 });
+  changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(Object.keys(changes).sort(), [separator.recordId, "menu"].sort());
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_remove() {
+  await ignoreChangedRoots();
+
+  info("Insert subtree for removal");
+  let parentFolder = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "folder",
+    parentRecordId: "menu",
+    recordId: makeGuid(),
+  });
+  let childBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.com",
+  });
+  let childFolder = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "folder",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+  });
+  let grandChildBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: childFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.edu",
+  });
+
+  info("Remove entire subtree");
+  await PlacesSyncUtils.bookmarks.remove([
+    parentFolder.recordId,
+    childFolder.recordId,
+    childBmk.recordId,
+    grandChildBmk.recordId,
+  ]);
+
+  /**
+   * Even though we've removed the entire subtree, we still track the menu
+   * because we 1) removed `parentFolder`, 2) reparented `childFolder` to
+   * `menu`, and 3) removed `childFolder`.
+   *
+   * This depends on the order of the folders passed to `remove`. If we
+   * removed `childFolder` *before* `parentFolder`, we wouldn't reparent
+   * anything to `menu`.
+   *
+   * `deleteSyncedFolder` could check if it's reparenting an item that will
+   * eventually be removed, and avoid bumping the new parent's change counter.
+   * Unfortunately, that introduces inconsistencies if `deleteSyncedFolder` is
+   * interrupted by shutdown. If the server changes before the next sync,
+   * we'll never upload records for the reparented item or the new parent.
+   *
+   * Another alternative: we can try to remove folders in level order, instead
+   * of the order passed to `remove`. But that means we need a recursive query
+   * to determine the order. This is already enough of an edge case that
+   * occasionally reuploading the closest living ancestor is the simplest
+   * solution.
+   */
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(changes),
+    ["menu"],
+    "Should track closest living ancestor of removed subtree"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_remove_partial() {
+  await ignoreChangedRoots();
+
+  info("Insert subtree for partial removal");
+  let parentFolder = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "folder",
+    parentRecordId: PlacesUtils.bookmarks.menuGuid,
+    recordId: makeGuid(),
+  });
+  let prevSiblingBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.net",
+  });
+  let childBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.com",
+  });
+  let nextSiblingBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.org",
+  });
+  let childFolder = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "folder",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+  });
+  let grandChildBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://example.edu",
+  });
+  let grandChildSiblingBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: parentFolder.recordId,
+    recordId: makeGuid(),
+    url: "https://mozilla.org",
+  });
+  let grandChildFolder = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "folder",
+    parentRecordId: childFolder.recordId,
+    recordId: makeGuid(),
+  });
+  let greatGrandChildPrevSiblingBmk =
+    await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "bookmark",
+      parentRecordId: grandChildFolder.recordId,
+      recordId: makeGuid(),
+      url: "http://getfirefox.com",
+    });
+  let greatGrandChildNextSiblingBmk =
+    await PlacesSyncUtils.test.bookmarks.insert({
+      kind: "bookmark",
+      parentRecordId: grandChildFolder.recordId,
+      recordId: makeGuid(),
+      url: "http://getthunderbird.com",
+    });
+  let menuBmk = await PlacesSyncUtils.test.bookmarks.insert({
+    kind: "bookmark",
+    parentRecordId: "menu",
+    recordId: makeGuid(),
+    url: "https://example.info",
+  });
+
+  info("Remove subset of folders and items in subtree");
+  let changes = await PlacesSyncUtils.bookmarks.remove([
+    parentFolder.recordId,
+    childBmk.recordId,
+    grandChildFolder.recordId,
+    grandChildBmk.recordId,
+    childFolder.recordId,
+  ]);
+  deepEqual(
+    Object.keys(changes).sort(),
+    [
+      // Closest living ancestor.
+      "menu",
+      // Reparented bookmarks.
+      prevSiblingBmk.recordId,
+      nextSiblingBmk.recordId,
+      grandChildSiblingBmk.recordId,
+      greatGrandChildPrevSiblingBmk.recordId,
+      greatGrandChildNextSiblingBmk.recordId,
+    ].sort(),
+    "Should track reparented bookmarks and their closest living ancestor"
+  );
+
+  /**
+   * Reparented bookmarks should maintain their order relative to their
+   * siblings: `prevSiblingBmk` (0) should precede `nextSiblingBmk` (2) in the
+   * menu, and `greatGrandChildPrevSiblingBmk` (0) should precede
+   * `greatGrandChildNextSiblingBmk` (1).
+   */
+  let menuChildren = await PlacesSyncUtils.bookmarks.fetchChildRecordIds(
+    PlacesUtils.bookmarks.menuGuid
+  );
+  deepEqual(
+    menuChildren,
+    [
+      // Existing bookmark.
+      menuBmk.recordId,
+      // 1) Moved out of `parentFolder` to `menu`.
+      prevSiblingBmk.recordId,
+      nextSiblingBmk.recordId,
+      // 3) Moved out of `childFolder` to `menu`. After this step, `childFolder`
+      // is deleted.
+      grandChildSiblingBmk.recordId,
+      // 2) Moved out of `grandChildFolder` to `childFolder`, because we remove
+      // `grandChildFolder` *before* `childFolder`. After this step,
+      // `grandChildFolder` is deleted and `childFolder`'s children are
+      // `[grandChildSiblingBmk, greatGrandChildPrevSiblingBmk,
+      // greatGrandChildNextSiblingBmk]`.
+      greatGrandChildPrevSiblingBmk.recordId,
+      greatGrandChildNextSiblingBmk.recordId,
+    ],
+    "Should move descendants to closest living ancestor"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_migrateOldTrackerEntries() {
+  let timerPrecision = Preferences.get("privacy.reduceTimerPrecision");
+  Preferences.set("privacy.reduceTimerPrecision", false);
+
+  registerCleanupFunction(function () {
+    Preferences.set("privacy.reduceTimerPrecision", timerPrecision);
+  });
+
+  let unknownBmk = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://getfirefox.com",
+    title: "Get Firefox!",
+  });
+  let newBmk = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://getthunderbird.com",
+    title: "Get Thunderbird!",
+  });
+  let normalBmk = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "https://mozilla.org",
+    title: "Mozilla",
+  });
+
+  await PlacesTestUtils.setBookmarkSyncFields(
+    {
+      guid: unknownBmk.guid,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN,
+      syncChangeCounter: 0,
+    },
+    {
+      guid: normalBmk.guid,
+      syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+    }
+  );
+  PlacesUtils.tagging.tagURI(uri("http://getfirefox.com"), ["taggy"]);
+
+  let tombstoneRecordId = makeGuid();
+  await PlacesSyncUtils.bookmarks.migrateOldTrackerEntries([
+    {
+      recordId: normalBmk.guid,
+      modified: Date.now(),
+    },
+    {
+      recordId: tombstoneRecordId,
+      modified: 1479162463976,
+    },
+  ]);
+
+  let changes = await PlacesSyncUtils.bookmarks.pullChanges();
+  deepEqual(
+    Object.keys(changes).sort(),
+    [normalBmk.guid, tombstoneRecordId].sort(),
+    "Should return change records for migrated bookmark and tombstone"
+  );
+
+  let fields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    unknownBmk.guid,
+    newBmk.guid,
+    normalBmk.guid
+  );
+  for (let field of fields) {
+    if (field.guid == normalBmk.guid) {
+      ok(
+        field.lastModified > normalBmk.lastModified,
+        `Should bump last modified date for migrated bookmark ${field.guid}`
+      );
+      equal(
+        field.syncChangeCounter,
+        1,
+        `Should bump change counter for migrated bookmark ${field.guid}`
+      );
+    } else {
+      strictEqual(
+        field.syncChangeCounter,
+        0,
+        `Should not bump change counter for ${field.guid}`
+      );
+    }
+    equal(
+      field.syncStatus,
+      PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+      `Should set sync status for ${field.guid} to NORMAL`
+    );
+  }
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones,
+    [
+      {
+        guid: tombstoneRecordId,
+        dateRemoved: new Date(1479162463976),
+      },
+    ],
+    "Should write tombstone for nonexistent migrated item"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_ensureMobileQuery() {
+  info("Ensure we correctly set the showMobileBookmarks preference");
+  const mobilePref = "browser.bookmarks.showMobileBookmarks";
+  Services.prefs.clearUserPref(mobilePref);
+
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkAAAA",
+    parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    url: "http://example.com/a",
+    title: "A",
+  });
+
+  await PlacesUtils.bookmarks.insert({
+    guid: "bookmarkBBBB",
+    parentGuid: PlacesUtils.bookmarks.mobileGuid,
+    url: "http://example.com/b",
+    title: "B",
+  });
+
+  await PlacesSyncUtils.bookmarks.ensureMobileQuery();
+
+  Assert.ok(
+    Services.prefs.getBoolPref(mobilePref),
+    "Pref should be true where there are bookmarks in the folder."
+  );
+
+  await PlacesUtils.bookmarks.remove("bookmarkAAAA");
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+
+  await PlacesSyncUtils.bookmarks.ensureMobileQuery();
+
+  Assert.ok(
+    !Services.prefs.getBoolPref(mobilePref),
+    "Pref should be false where there are no bookmarks in the folder."
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_remove_stale_tombstones() {
+  info("Insert and delete synced bookmark");
+  {
+    await PlacesUtils.bookmarks.insert({
+      guid: "bookmarkAAAA",
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: "http://example.com/a",
+      title: "A",
+      source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    });
+    await PlacesUtils.bookmarks.remove("bookmarkAAAA");
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones.map(({ guid }) => guid),
+      ["bookmarkAAAA"],
+      "Should store tombstone for deleted synced bookmark"
+    );
+  }
+
+  info("Reinsert deleted bookmark");
+  {
+    // Different parent, URL, and title, but same GUID.
+    await PlacesUtils.bookmarks.insert({
+      guid: "bookmarkAAAA",
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: "http://example.com/a-restored",
+      title: "A (Restored)",
+    });
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones,
+      [],
+      "Should remove tombstone for reinserted bookmark"
+    );
+  }
+
+  info("Insert tree and erase everything");
+  {
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.menuGuid,
+      source: PlacesUtils.bookmarks.SOURCES.SYNC,
+      children: [
+        {
+          guid: "bookmarkBBBB",
+          url: "http://example.com/b",
+          title: "B",
+        },
+        {
+          guid: "bookmarkCCCC",
+          url: "http://example.com/c",
+          title: "C",
+        },
+      ],
+    });
+    await PlacesUtils.bookmarks.eraseEverything();
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones.map(({ guid }) => guid).sort(),
+      ["bookmarkBBBB", "bookmarkCCCC"],
+      "Should store tombstones after erasing everything"
+    );
+  }
+
+  info("Reinsert tree");
+  {
+    await PlacesUtils.bookmarks.insertTree({
+      guid: PlacesUtils.bookmarks.mobileGuid,
+      children: [
+        {
+          guid: "bookmarkBBBB",
+          url: "http://example.com/b",
+          title: "B",
+        },
+        {
+          guid: "bookmarkCCCC",
+          url: "http://example.com/c",
+          title: "C",
+        },
+      ],
+    });
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones.map(({ guid }) => guid).sort(),
+      [],
+      "Should remove tombstones after reinserting tree"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_resetSyncId() {
+  let syncId = await PlacesSyncUtils.bookmarks.getSyncId();
+  strictEqual(syncId, "", "Should start with empty bookmarks sync ID");
+
+  // Add a tree with a NORMAL bookmark (A), tombstone (B), NEW bookmark (C),
+  // and UNKNOWN bookmark (D).
+  info("Set up local tree before resetting bookmarks sync ID");
+  await ignoreChangedRoots();
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "bookmarkCCCC",
+    title: "C",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    guid: "bookmarkDDDD",
+    title: "D",
+    url: "http://example.com/d",
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  info("Assign new bookmarks sync ID for first time");
+  let newSyncId = await PlacesSyncUtils.bookmarks.resetSyncId();
+  syncId = await PlacesSyncUtils.bookmarks.getSyncId();
+  equal(
+    newSyncId,
+    syncId,
+    "Should assign new bookmarks sync ID for first time"
+  );
+
+  let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    "bookmarkAAAA",
+    "bookmarkCCCC",
+    "bookmarkDDDD"
+  );
+  ok(
+    syncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Should change all sync statuses to NEW after resetting bookmarks sync ID"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(
+    tombstones,
+    [],
+    "Should remove all tombstones after resetting bookmarks sync ID"
+  );
+
+  info("Set bookmarks last sync time");
+  let lastSync = Date.now() / 1000;
+  await PlacesSyncUtils.bookmarks.setLastSync(lastSync);
+  equal(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    lastSync,
+    "Should record bookmarks last sync time"
+  );
+
+  newSyncId = await PlacesSyncUtils.bookmarks.resetSyncId();
+  notEqual(
+    newSyncId,
+    syncId,
+    "Should set new bookmarks sync ID if one already exists"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    0,
+    "Should reset bookmarks last sync time after resetting sync ID"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_wipe() {
+  info("Add Sync metadata before wipe");
+  let newSyncId = await PlacesSyncUtils.bookmarks.resetSyncId();
+  await PlacesSyncUtils.bookmarks.setLastSync(Date.now() / 1000);
+
+  let existingSyncId = await PlacesSyncUtils.bookmarks.getSyncId();
+  equal(
+    existingSyncId,
+    newSyncId,
+    "Ensure bookmarks sync ID was recorded before wipe"
+  );
+
+  info("Set up local tree before wipe");
+  await ignoreChangedRoots();
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "bookmarkCCCC",
+    title: "C",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    guid: "bookmarkDDDD",
+    title: "D",
+    url: "http://example.com/d",
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  info("Wipe bookmarks");
+  await PlacesSyncUtils.bookmarks.wipe();
+
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getSyncId(),
+    "",
+    "Should reset bookmarks sync ID after wipe"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    0,
+    "Should reset bookmarks last sync after wipe"
+  );
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Wiping bookmarks locally should not wipe server"
+  );
+
+  let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+  deepEqual(tombstones, [], "Should drop tombstones after wipe");
+
+  deepEqual(
+    await PlacesSyncUtils.bookmarks.fetchChildRecordIds("menu"),
+    [],
+    "Should wipe menu children"
+  );
+  deepEqual(
+    await PlacesSyncUtils.bookmarks.fetchChildRecordIds("toolbar"),
+    [],
+    "Should wipe toolbar children"
+  );
+
+  let rootSyncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    PlacesUtils.bookmarks.unfiledGuid
+  );
+  ok(
+    rootSyncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Should reset all sync statuses to NEW after wipe"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_meta_eraseEverything() {
+  info("Add Sync metadata before erase");
+  let newSyncId = await PlacesSyncUtils.bookmarks.resetSyncId();
+  let lastSync = Date.now() / 1000;
+  await PlacesSyncUtils.bookmarks.setLastSync(lastSync);
+
+  info("Set up local tree before reset");
+  await ignoreChangedRoots();
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "bookmarkCCCC",
+    title: "C",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    guid: "bookmarkDDDD",
+    title: "D",
+    url: "http://example.com/d",
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  info("Erase all bookmarks");
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getSyncId(),
+    newSyncId,
+    "Should not reset bookmarks sync ID after erase"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    lastSync,
+    "Should not reset bookmarks last sync after erase"
+  );
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Erasing everything should not wipe server"
+  );
+
+  deepEqual(
+    (await PlacesTestUtils.fetchSyncTombstones()).map(info => info.guid),
+    ["bookmarkAAAA", "bookmarkBBBB"],
+    "Should keep tombstones after erasing everything"
+  );
+
+  let rootSyncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    PlacesUtils.bookmarks.menuGuid,
+    PlacesUtils.bookmarks.toolbarGuid,
+    PlacesUtils.bookmarks.unfiledGuid,
+    PlacesUtils.bookmarks.mobileGuid
+  );
+  ok(
+    rootSyncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NORMAL
+    ),
+    "Should not reset sync statuses after erasing everything"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_reset() {
+  info("Add Sync metadata before reset");
+  await PlacesSyncUtils.bookmarks.resetSyncId();
+  await PlacesSyncUtils.bookmarks.setLastSync(Date.now() / 1000);
+
+  info("Set up local tree before reset");
+  await ignoreChangedRoots();
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "bookmarkCCCC",
+    title: "C",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    guid: "bookmarkDDDD",
+    title: "D",
+    url: "http://example.com/d",
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  info("Reset Sync metadata for bookmarks");
+  await PlacesSyncUtils.bookmarks.reset();
+
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getSyncId(),
+    "",
+    "Should reset bookmarks sync ID after reset"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    0,
+    "Should reset bookmarks last sync after reset"
+  );
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Resetting Sync metadata should not wipe server"
+  );
+
+  deepEqual(
+    await PlacesTestUtils.fetchSyncTombstones(),
+    [],
+    "Should drop tombstones after reset"
+  );
+
+  let itemSyncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    "bookmarkAAAA",
+    "bookmarkCCCC"
+  );
+  ok(
+    itemSyncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Should reset sync statuses for existing items to NEW after reset"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_meta_restore() {
+  info("Add Sync metadata before manual restore");
+  await PlacesSyncUtils.bookmarks.resetSyncId();
+  await PlacesSyncUtils.bookmarks.setLastSync(Date.now() / 1000);
+
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Should not wipe server before manual restore"
+  );
+
+  info("Manually restore");
+  let { path } = do_get_file("./sync_utils_bookmarks.json");
+  await BookmarkJSONUtils.importFromFile(path, { replace: true });
+
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getSyncId(),
+    "",
+    "Should reset bookmarks sync ID after manual restore"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    0,
+    "Should reset bookmarks last sync after manual restore"
+  );
+  ok(
+    await PlacesSyncUtils.bookmarks.shouldWipeRemote(),
+    "Should wipe server after manual restore"
+  );
+
+  let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    PlacesUtils.bookmarks.menuGuid,
+    "NnvGl3CRA4hC",
+    PlacesUtils.bookmarks.toolbarGuid,
+    "APzP8MupzA8l"
+  );
+  ok(
+    syncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.NEW
+    ),
+    "Should reset all sync stauses to NEW after manual restore"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_meta_restore_on_startup() {
+  info("Add Sync metadata before simulated automatic restore");
+  await PlacesSyncUtils.bookmarks.resetSyncId();
+  await PlacesSyncUtils.bookmarks.setLastSync(Date.now() / 1000);
+
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Should not wipe server before automatic restore"
+  );
+
+  info("Simulate automatic restore on startup");
+  let { path } = do_get_file("./sync_utils_bookmarks.json");
+  await BookmarkJSONUtils.importFromFile(path, {
+    replace: true,
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getSyncId(),
+    "",
+    "Should reset bookmarks sync ID after automatic restore"
+  );
+  strictEqual(
+    await PlacesSyncUtils.bookmarks.getLastSync(),
+    0,
+    "Should reset bookmarks last sync after automatic restore"
+  );
+  ok(
+    !(await PlacesSyncUtils.bookmarks.shouldWipeRemote()),
+    "Should not wipe server after manual restore"
+  );
+
+  let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+    PlacesUtils.bookmarks.menuGuid,
+    "NnvGl3CRA4hC",
+    PlacesUtils.bookmarks.toolbarGuid,
+    "APzP8MupzA8l"
+  );
+  ok(
+    syncFields.every(
+      field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN
+    ),
+    "Should reset all sync stauses to UNKNOWN after automatic restore"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_bookmarks_ensureCurrentSyncId() {
+  info("Set up local tree");
+  await ignoreChangedRoots();
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    source: PlacesUtils.bookmarks.SOURCES.SYNC,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        title: "A",
+        url: "http://example.com/a",
+      },
+      {
+        guid: "bookmarkBBBB",
+        title: "B",
+        url: "http://example.com/b",
+      },
+    ],
+  });
+  await PlacesUtils.bookmarks.remove("bookmarkBBBB");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    guid: "bookmarkCCCC",
+    title: "C",
+    url: "http://example.com/c",
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    guid: "bookmarkDDDD",
+    title: "D",
+    url: "http://example.com/d",
+    source: PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+  });
+
+  let existingSyncId = await PlacesSyncUtils.bookmarks.getSyncId();
+  strictEqual(existingSyncId, "", "Should start without bookmarks sync ID");
+
+  info("Assign new bookmarks sync ID");
+  {
+    await PlacesSyncUtils.bookmarks.ensureCurrentSyncId("syncIdAAAAAA");
+
+    let newSyncId = await PlacesSyncUtils.bookmarks.getSyncId();
+    equal(
+      newSyncId,
+      "syncIdAAAAAA",
+      "Should assign bookmarks sync ID if one doesn't exist"
+    );
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones.map(({ guid }) => guid),
+      ["bookmarkBBBB"],
+      "Should keep tombstones after assigning new bookmarks sync ID"
+    );
+
+    let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+      "bookmarkAAAA",
+      "bookmarkCCCC",
+      "bookmarkDDDD"
+    );
+    deepEqual(
+      syncFields.map(field => field.syncStatus),
+      [
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+        PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN,
+      ],
+      "Should not reset sync statuses after assigning new bookmarks sync ID"
+    );
+  }
+
+  info("Ensure existing bookmarks sync ID matches");
+  {
+    let lastSync = Date.now() / 1000;
+    await PlacesSyncUtils.bookmarks.setLastSync(lastSync);
+    await PlacesSyncUtils.bookmarks.ensureCurrentSyncId("syncIdAAAAAA");
+
+    equal(
+      await PlacesSyncUtils.bookmarks.getSyncId(),
+      "syncIdAAAAAA",
+      "Should keep existing bookmarks sync ID on match"
+    );
+    equal(
+      await PlacesSyncUtils.bookmarks.getLastSync(),
+      lastSync,
+      "Should keep existing bookmarks last sync time on sync ID match"
+    );
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones.map(({ guid }) => guid),
+      ["bookmarkBBBB"],
+      "Should keep tombstones if bookmarks sync IDs match"
+    );
+
+    let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+      "bookmarkAAAA",
+      "bookmarkCCCC",
+      "bookmarkDDDD"
+    );
+    deepEqual(
+      syncFields.map(field => field.syncStatus),
+      [
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NORMAL,
+        PlacesUtils.bookmarks.SYNC_STATUS.NEW,
+        PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN,
+      ],
+      "Should not reset sync statuses if bookmarks sync IDs match"
+    );
+  }
+
+  info("Replace existing bookmarks sync ID with new ID");
+  {
+    await PlacesSyncUtils.bookmarks.ensureCurrentSyncId("syncIdBBBBBB");
+
+    equal(
+      await PlacesSyncUtils.bookmarks.getSyncId(),
+      "syncIdBBBBBB",
+      "Should replace existing bookmarks sync ID on mismatch"
+    );
+    strictEqual(
+      await PlacesSyncUtils.bookmarks.getLastSync(),
+      0,
+      "Should reset bookmarks last sync time on sync ID mismatch"
+    );
+
+    let tombstones = await PlacesTestUtils.fetchSyncTombstones();
+    deepEqual(
+      tombstones,
+      [],
+      "Should drop tombstones after bookmarks sync ID mismatch"
+    );
+
+    let syncFields = await PlacesTestUtils.fetchBookmarkSyncFields(
+      PlacesUtils.bookmarks.menuGuid,
+      PlacesUtils.bookmarks.toolbarGuid,
+      PlacesUtils.bookmarks.unfiledGuid,
+      "bookmarkAAAA",
+      "bookmarkCCCC",
+      "bookmarkDDDD"
+    );
+    ok(
+      syncFields.every(
+        field => field.syncStatus == PlacesUtils.bookmarks.SYNC_STATUS.UNKNOWN
+      ),
+      "Should reset all sync statuses to UNKNOWN after bookmarks sync ID mismatch"
+    );
+  }
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesSyncUtils.bookmarks.reset();
+});
+
+add_task(async function test_history_resetSyncId() {
+  let syncId = await PlacesSyncUtils.history.getSyncId();
+  strictEqual(syncId, "", "Should start with empty history sync ID");
+
+  info("Assign new history sync ID for first time");
+  let newSyncId = await PlacesSyncUtils.history.resetSyncId();
+  syncId = await PlacesSyncUtils.history.getSyncId();
+  equal(newSyncId, syncId, "Should assign new history sync ID for first time");
+
+  info("Set history last sync time");
+  let lastSync = Date.now() / 1000;
+  await PlacesSyncUtils.history.setLastSync(lastSync);
+  equal(
+    await PlacesSyncUtils.history.getLastSync(),
+    lastSync,
+    "Should record history last sync time"
+  );
+
+  newSyncId = await PlacesSyncUtils.history.resetSyncId();
+  notEqual(
+    newSyncId,
+    syncId,
+    "Should set new history sync ID if one already exists"
+  );
+  strictEqual(
+    await PlacesSyncUtils.history.getLastSync(),
+    0,
+    "Should reset history last sync time after resetting sync ID"
+  );
+
+  await PlacesSyncUtils.history.reset();
+});
+
+add_task(async function test_history_ensureCurrentSyncId() {
+  info("Assign new history sync ID");
+  await PlacesSyncUtils.history.ensureCurrentSyncId("syncIdAAAAAA");
+  equal(
+    await PlacesSyncUtils.history.getSyncId(),
+    "syncIdAAAAAA",
+    "Should assign history sync ID if one doesn't exist"
+  );
+
+  info("Ensure existing history sync ID matches");
+  let lastSync = Date.now() / 1000;
+  await PlacesSyncUtils.history.setLastSync(lastSync);
+  await PlacesSyncUtils.history.ensureCurrentSyncId("syncIdAAAAAA");
+
+  equal(
+    await PlacesSyncUtils.history.getSyncId(),
+    "syncIdAAAAAA",
+    "Should keep existing history sync ID on match"
+  );
+  equal(
+    await PlacesSyncUtils.history.getLastSync(),
+    lastSync,
+    "Should keep existing history last sync time on sync ID match"
+  );
+
+  info("Replace existing history sync ID with new ID");
+  await PlacesSyncUtils.history.ensureCurrentSyncId("syncIdBBBBBB");
+
+  equal(
+    await PlacesSyncUtils.history.getSyncId(),
+    "syncIdBBBBBB",
+    "Should replace existing history sync ID on mismatch"
+  );
+  strictEqual(
+    await PlacesSyncUtils.history.getLastSync(),
+    0,
+    "Should reset history last sync time on sync ID mismatch"
+  );
+
+  await PlacesSyncUtils.history.reset();
+});
diff --git a/toolkit/components/places/tests/sync/xpcshell.ini b/toolkit/components/places/tests/sync/xpcshell.ini
new file mode 100644
index 0000000000..0f0953a83c
--- /dev/null
+++ b/toolkit/components/places/tests/sync/xpcshell.ini
@@ -0,0 +1,23 @@
+[DEFAULT]
+head = head_sync.js
+support-files =
+  sync_utils_bookmarks.html
+  sync_utils_bookmarks.json
+  mirror_corrupt.sqlite
+  mirror_v1.sqlite
+  mirror_v5.sqlite
+
+[test_bookmark_abort_merging.js]
+[test_bookmark_chunking.js]
+[test_bookmark_corruption.js]
+[test_bookmark_deduping.js]
+[test_bookmark_deletion.js]
+[test_bookmark_haschanges.js]
+[test_bookmark_kinds.js]
+[test_bookmark_mirror_meta.js]
+[test_bookmark_mirror_migration.js]
+[test_bookmark_observer_recorder.js]
+[test_bookmark_reconcile.js]
+[test_bookmark_structure_changes.js]
+[test_bookmark_value_changes.js]
+[test_sync_utils.js]
diff --git a/toolkit/components/places/tests/unit/bookmarks.corrupt.html b/toolkit/components/places/tests/unit/bookmarks.corrupt.html
new file mode 100644
index 0000000000..3cf43367fb
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks.corrupt.html
@@ -0,0 +1,36 @@
+
+
+
+Bookmarks
+
      Bookmarks
      
+
+
      
+    
      Mozilla Firefox
      
+    
      
+        
      Help and Tutorials
+        
      Customize Firefox
+        
      Get Involved
+        
      About Us
+        
      About Us
+    
      
+    
      test
      
+
      folder test comment
+    
      
+        
      test post keyword
+
      item description
+    
      
+    
      Unsorted Bookmarks
      
+    
      
+        
      Example.tld
+    
      
+    
      Bookmarks Toolbar Folder
      
+
      Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar
+    
      
+        
      Getting Started
+        
      Latest Headlines
+        
      Getting Started
+
      Livemark test comment
+    
      
+
      
diff --git a/toolkit/components/places/tests/unit/bookmarks.json b/toolkit/components/places/tests/unit/bookmarks.json
new file mode 100644
index 0000000000..27ed9ce5ca
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks.json
@@ -0,0 +1,307 @@
+{
+  "guid": "root________",
+  "title": "",
+  "id": 1,
+  "dateAdded": 1361551978957783,
+  "lastModified": 1361551978957783,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "id": 2,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551979382837,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FF",
+          "title": "Mozilla Firefox",
+          "id": 6,
+          "parent": 2,
+          "dateAdded": 1361551979350273,
+          "lastModified": 1361551979376699,
+          "type": "text/x-moz-place-container",
+          "children": [
+            {
+              "guid": "OCyeUO5uu9FG",
+              "title": "Help and Tutorials",
+              "id": 7,
+              "parent": 6,
+              "dateAdded": 1361551979356436,
+              "lastModified": 1361551979362718,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/help/",
+              "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FH",
+              "index": 1,
+              "title": "Customize Firefox",
+              "id": 8,
+              "parent": 6,
+              "dateAdded": 1361551979365662,
+              "lastModified": 1361551979368077,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/customize/",
+              "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FJ",
+              "index": 3,
+              "title": "About Us",
+              "id": 10,
+              "parent": 6,
+              "dateAdded": 1361551979376699,
+              "lastModified": 1361551979379060,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/about/",
+              "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FI",
+              "index": 2,
+              "title": "Get Involved",
+              "id": 9,
+              "parent": 6,
+              "dateAdded": 1361551979371071,
+              "lastModified": 1361551979373745,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/community/",
+              "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "QFM-QnE2ZpMz",
+              "title": "Test null postData",
+              "index": 4,
+              "dateAdded": 1481639510868000,
+              "lastModified": 1489563704300000,
+              "id": 17,
+              "charset": "UTF-8",
+              "annos": [
+                {
+                  "name": "bookmarkProperties/description",
+                  "flags": 0,
+                  "expires": 4,
+                  "value": "The best"
+                }
+              ],
+              "type": "text/x-moz-place",
+              "uri": "http://example.com/search?q=%s&suggid=",
+              "postData": null
+            }
+          ]
+        },
+        {
+          "guid": "OCyeUO5uu9FK",
+          "index": 1,
+          "title": "",
+          "id": 11,
+          "parent": 2,
+          "dateAdded": 1361551979380988,
+          "lastModified": 1361551979380988,
+          "type": "text/x-moz-place-separator"
+        },
+        {
+          "guid": "OCyeUO5uu9FL",
+          "index": 2,
+          "title": "test",
+          "id": 12,
+          "parent": 2,
+          "dateAdded": 1177541020000000,
+          "lastModified": 1177541050000000,
+          "annos": [
+            {
+              "name": "bookmarkProperties/description",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "folder test comment"
+            }
+          ],
+          "type": "text/x-moz-place-container",
+          "children": [
+            {
+              "guid": "OCyeUO5uu9GX",
+              "title": "test post keyword",
+              "id": 13,
+              "parent": 12,
+              "dateAdded": 1177375336000000,
+              "lastModified": 1177375423000000,
+              "annos": [
+                {
+                  "name": "bookmarkProperties/description",
+                  "flags": 0,
+                  "expires": 4,
+                  "mimeType": null,
+                  "type": 3,
+                  "value": "item description"
+                },
+                {
+                  "name": "bookmarkProperties/loadInSidebar",
+                  "flags": 0,
+                  "expires": 4,
+                  "mimeType": null,
+                  "type": 1,
+                  "value": 1
+                }
+              ],
+              "type": "text/x-moz-place",
+              "uri": "http://test/post",
+              "keyword": "test",
+              "charset": "ISO-8859-1",
+              "postData": "hidden1%3Dbar&text1%3D%25s"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "index": 1,
+      "title": "Bookmarks Toolbar",
+      "id": 3,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1177541050000000,
+      "annos": [
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "mimeType": null,
+          "type": 3,
+          "value": "Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FB",
+          "title": "Getting Started",
+          "id": 15,
+          "parent": 3,
+          "dateAdded": 1361551979409695,
+          "lastModified": 1361551979412080,
+          "type": "text/x-moz-place",
+          "uri": "http://en-us.www.mozilla.com/en-US/firefox/central/",
+          "icon": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+        },
+        {
+          "guid": "OCyeUO5uu9FR",
+          "index": 1,
+          "title": "Latest Headlines",
+          "id": 16,
+          "parent": 3,
+          "dateAdded": 1361551979451584,
+          "lastModified": 1361551979457086,
+          "livemark": 1,
+          "annos": [
+            {
+              "name": "livemark/feedURI",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "http://en-us.fxfeeds.mozilla.com/en-US/firefox/headlines.xml"
+            },
+            {
+              "name": "livemark/siteURI",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "http://en-us.fxfeeds.mozilla.com/en-US/firefox/livebookmarks/"
+            }
+          ],
+          "type": "text/x-moz-place-container",
+          "children": []
+        }
+      ]
+    },
+    {
+      "index": 2,
+      "title": "Tags",
+      "id": 4,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551978957783,
+      "type": "text/x-moz-place-container",
+      "root": "tagsFolder",
+      "children": []
+    },
+    {
+      "index": 3,
+      "title": "Unsorted Bookmarks",
+      "id": 5,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1177541050000000,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FW",
+          "title": "Example.tld",
+          "id": 14,
+          "parent": 5,
+          "dateAdded": 1361551979401846,
+          "lastModified": 1361551979402952,
+          "type": "text/x-moz-place",
+          "uri": "http://example.tld/"
+        },
+        {
+          "guid": "Cfkety492Afk",
+          "title": "test tagged bookmark",
+          "id": 15,
+          "parent": 5,
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "type": "text/x-moz-place",
+          "uri": "http://example.tld/tagged",
+          "tags": "foo"
+        },
+        {
+          "guid": "lOZGoFR1eXbl",
+          "title": "Bookmarks Toolbar Shortcut",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 16,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=TOOLBAR"
+        },
+        {
+          "guid": "7yJWnBVhjRtP",
+          "title": "Folder Shortcut",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 17,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6"
+        },
+        {
+          "guid": "vm5QXWuWc12l",
+          "title": "Folder Shortcut 2",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 18,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6123443"
+        },
+        {
+          "guid": "Icg1XlIozA1D",
+          "title": "Folder Shortcut 3",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 18,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6&folder=BOOKMARKS_MENU"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/bookmarks.preplaces.html b/toolkit/components/places/tests/unit/bookmarks.preplaces.html
new file mode 100644
index 0000000000..0ddf7725b4
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks.preplaces.html
@@ -0,0 +1,36 @@
+
+
+
+Bookmarks
+
      Bookmarks
      
+
+
      
+    
      Mozilla Firefox
      
+    
      
+        
      Help and Tutorials
+        
      Customize Firefox
+        
      Get Involved
+        
      About Us
+    
      
+    
      
+    
      test
      
+
      folder test comment
+    
      
+        
      test post keyword
+
      item description
+    
      
+    
      Unsorted Bookmarks
      
+    
      
+        
      Example.tld
+    
      
+    
      Bookmarks Toolbar Folder
      
+
      Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar
+    
      
+        
      Getting Started
+        
      Latest Headlines
+        
      Latest Headlines No Site
+
      Livemark test comment
+    
      
+
      
diff --git a/toolkit/components/places/tests/unit/bookmarks_corrupt.json b/toolkit/components/places/tests/unit/bookmarks_corrupt.json
new file mode 100644
index 0000000000..93f21d3ece
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks_corrupt.json
@@ -0,0 +1,72 @@
+{
+  "guid": "root________",
+  "title": "",
+  "id": 1,
+  "dateAdded": 1361551978957783,
+  "lastModified": 1361551978957783,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "id": 2,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551979382837,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FG",
+          "title": "Help and Tutorials",
+          "id": 7,
+          "dateAdded": 1361551979356436,
+          "lastModified": 1361551979362718,
+          "type": "x/invalid",
+          "uri": "http://en-us.www.mozilla.com/en-US/firefox/help/"
+        },
+        {
+          "guid": "OCyeUO5uu9FH",
+          "index": 1,
+          "title": "Customize Firefox",
+          "id": 8,
+          "dateAdded": 1361551979365662,
+          "lastModified": 1361551979368077,
+          "type": "text/x-moz-place",
+          "uri": "http://en-us.www.mozilla.com/en-US/firefox/customize/"
+        },
+        {
+          "guid": "OCyeUO5uu9FG",
+          "title": "Bad URL",
+          "id": 9,
+          "dateAdded": 1361551979356436,
+          "lastModified": 1361551979362718,
+          "type": "text/x-moz-place",
+          "uri": "http:///"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "id": 2,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551979382837,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FG",
+          "title": "Bad URL",
+          "id": 9,
+          "dateAdded": 1361551979356436,
+          "lastModified": 1361551979362718,
+          "type": "text/x-moz-place",
+          "uri": "http:///"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/bookmarks_html_localized.html b/toolkit/components/places/tests/unit/bookmarks_html_localized.html
new file mode 100644
index 0000000000..bc3bacc54d
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks_html_localized.html
@@ -0,0 +1,21 @@
+
+
+
+
+
+Bookmarks
+
+
+
+
      Bookmarks
      
+
+
      
+    
      bookmarks-html-localized-folder
      
+    
      
+        
      bookmarks-html-localized-bookmark
+    
      
+
      
+
+
diff --git a/toolkit/components/places/tests/unit/bookmarks_html_singleframe.html b/toolkit/components/places/tests/unit/bookmarks_html_singleframe.html
new file mode 100644
index 0000000000..9fe662f320
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks_html_singleframe.html
@@ -0,0 +1,10 @@
+
+	
+	
+	Bookmarks
+	
      Bookmarks
      
+    
      Subtitle
      
+	
      
+		
      Mozilla
+    
      
+
diff --git a/toolkit/components/places/tests/unit/bookmarks_iconuri.json b/toolkit/components/places/tests/unit/bookmarks_iconuri.json
new file mode 100644
index 0000000000..4059c1d53f
--- /dev/null
+++ b/toolkit/components/places/tests/unit/bookmarks_iconuri.json
@@ -0,0 +1,307 @@
+{
+  "guid": "root________",
+  "title": "",
+  "id": 1,
+  "dateAdded": 1361551978957783,
+  "lastModified": 1361551978957783,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "id": 2,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551979382837,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FF",
+          "title": "Mozilla Firefox",
+          "id": 6,
+          "parent": 2,
+          "dateAdded": 1361551979350273,
+          "lastModified": 1361551979376699,
+          "type": "text/x-moz-place-container",
+          "children": [
+            {
+              "guid": "OCyeUO5uu9FG",
+              "title": "Help and Tutorials",
+              "id": 7,
+              "parent": 6,
+              "dateAdded": 1361551979356436,
+              "lastModified": 1361551979362718,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/help/",
+              "iconUri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FH",
+              "index": 1,
+              "title": "Customize Firefox",
+              "id": 8,
+              "parent": 6,
+              "dateAdded": 1361551979365662,
+              "lastModified": 1361551979368077,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/customize/",
+              "iconUri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FJ",
+              "index": 3,
+              "title": "About Us",
+              "id": 10,
+              "parent": 6,
+              "dateAdded": 1361551979376699,
+              "lastModified": 1361551979379060,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/about/",
+              "iconUri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "OCyeUO5uu9FI",
+              "index": 2,
+              "title": "Get Involved",
+              "id": 9,
+              "parent": 6,
+              "dateAdded": 1361551979371071,
+              "lastModified": 1361551979373745,
+              "type": "text/x-moz-place",
+              "uri": "http://en-us.www.mozilla.com/en-US/firefox/community/",
+              "iconUri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+            },
+            {
+              "guid": "QFM-QnE2ZpMz",
+              "title": "Test null postData",
+              "index": 4,
+              "dateAdded": 1481639510868000,
+              "lastModified": 1489563704300000,
+              "id": 17,
+              "charset": "UTF-8",
+              "annos": [
+                {
+                  "name": "bookmarkProperties/description",
+                  "flags": 0,
+                  "expires": 4,
+                  "value": "The best"
+                }
+              ],
+              "type": "text/x-moz-place",
+              "uri": "http://example.com/search?q=%s&suggid=",
+              "postData": null
+            }
+          ]
+        },
+        {
+          "guid": "OCyeUO5uu9FK",
+          "index": 1,
+          "title": "",
+          "id": 11,
+          "parent": 2,
+          "dateAdded": 1361551979380988,
+          "lastModified": 1361551979380988,
+          "type": "text/x-moz-place-separator"
+        },
+        {
+          "guid": "OCyeUO5uu9FL",
+          "index": 2,
+          "title": "test",
+          "id": 12,
+          "parent": 2,
+          "dateAdded": 1177541020000000,
+          "lastModified": 1177541050000000,
+          "annos": [
+            {
+              "name": "bookmarkProperties/description",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "folder test comment"
+            }
+          ],
+          "type": "text/x-moz-place-container",
+          "children": [
+            {
+              "guid": "OCyeUO5uu9GX",
+              "title": "test post keyword",
+              "id": 13,
+              "parent": 12,
+              "dateAdded": 1177375336000000,
+              "lastModified": 1177375423000000,
+              "annos": [
+                {
+                  "name": "bookmarkProperties/description",
+                  "flags": 0,
+                  "expires": 4,
+                  "mimeType": null,
+                  "type": 3,
+                  "value": "item description"
+                },
+                {
+                  "name": "bookmarkProperties/loadInSidebar",
+                  "flags": 0,
+                  "expires": 4,
+                  "mimeType": null,
+                  "type": 1,
+                  "value": 1
+                }
+              ],
+              "type": "text/x-moz-place",
+              "uri": "http://test/post",
+              "keyword": "test",
+              "charset": "ISO-8859-1",
+              "postData": "hidden1%3Dbar&text1%3D%25s"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "index": 1,
+      "title": "Bookmarks Toolbar",
+      "id": 3,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1177541050000000,
+      "annos": [
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "mimeType": null,
+          "type": 3,
+          "value": "Add bookmarks to this folder to see them displayed on the Bookmarks Toolbar"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FB",
+          "title": "Getting Started",
+          "id": 15,
+          "parent": 3,
+          "dateAdded": 1361551979409695,
+          "lastModified": 1361551979412080,
+          "type": "text/x-moz-place",
+          "uri": "http://en-us.www.mozilla.com/en-US/firefox/central/",
+          "iconUri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg=="
+        },
+        {
+          "guid": "OCyeUO5uu9FR",
+          "index": 1,
+          "title": "Latest Headlines",
+          "id": 16,
+          "parent": 3,
+          "dateAdded": 1361551979451584,
+          "lastModified": 1361551979457086,
+          "livemark": 1,
+          "annos": [
+            {
+              "name": "livemark/feedURI",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "http://en-us.fxfeeds.mozilla.com/en-US/firefox/headlines.xml"
+            },
+            {
+              "name": "livemark/siteURI",
+              "flags": 0,
+              "expires": 4,
+              "mimeType": null,
+              "type": 3,
+              "value": "http://en-us.fxfeeds.mozilla.com/en-US/firefox/livebookmarks/"
+            }
+          ],
+          "type": "text/x-moz-place-container",
+          "children": []
+        }
+      ]
+    },
+    {
+      "index": 2,
+      "title": "Tags",
+      "id": 4,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1361551978957783,
+      "type": "text/x-moz-place-container",
+      "root": "tagsFolder",
+      "children": []
+    },
+    {
+      "index": 3,
+      "title": "Unsorted Bookmarks",
+      "id": 5,
+      "parent": 1,
+      "dateAdded": 1361551978957783,
+      "lastModified": 1177541050000000,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder",
+      "children": [
+        {
+          "guid": "OCyeUO5uu9FW",
+          "title": "Example.tld",
+          "id": 14,
+          "parent": 5,
+          "dateAdded": 1361551979401846,
+          "lastModified": 1361551979402952,
+          "type": "text/x-moz-place",
+          "uri": "http://example.tld/"
+        },
+        {
+          "guid": "Cfkety492Afk",
+          "title": "test tagged bookmark",
+          "id": 15,
+          "parent": 5,
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "type": "text/x-moz-place",
+          "uri": "http://example.tld/tagged",
+          "tags": "foo"
+        },
+        {
+          "guid": "lOZGoFR1eXbl",
+          "title": "Bookmarks Toolbar Shortcut",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 16,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=TOOLBAR"
+        },
+        {
+          "guid": "7yJWnBVhjRtP",
+          "title": "Folder Shortcut",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 17,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6"
+        },
+        {
+          "guid": "vm5QXWuWc12l",
+          "title": "Folder Shortcut 2",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 18,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6123443"
+        },
+        {
+          "guid": "Icg1XlIozA1D",
+          "title": "Folder Shortcut 3",
+          "dateAdded": 1507025843703345,
+          "lastModified": 1507025844703124,
+          "id": 18,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=6&folder=BOOKMARKS_MENU"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/head_bookmarks.js b/toolkit/components/places/tests/unit/head_bookmarks.js
new file mode 100644
index 0000000000..5f6250aa5d
--- /dev/null
+++ b/toolkit/components/places/tests/unit/head_bookmarks.js
@@ -0,0 +1,30 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Import common head.
+{
+  /* import-globals-from ../head_common.js */
+  let commonFile = do_get_file("../head_common.js", false);
+  let uri = Services.io.newFileURI(commonFile);
+  Services.scriptloader.loadSubScript(uri.spec, this);
+}
+
+// Put any other stuff relative to this test folder below.
+const { AddonTestUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/AddonTestUtils.sys.mjs"
+);
+
+AddonTestUtils.init(this, false);
+AddonTestUtils.overrideCertDB();
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "42"
+);
+
+add_task(async function setup() {
+  await AddonTestUtils.promiseStartupManager();
+});
diff --git a/toolkit/components/places/tests/unit/missingBuiltIn.sqlite b/toolkit/components/places/tests/unit/missingBuiltIn.sqlite
new file mode 100644
index 0000000000..4dbbb1ac75
Binary files /dev/null and b/toolkit/components/places/tests/unit/missingBuiltIn.sqlite differ
diff --git a/toolkit/components/places/tests/unit/mobile_bookmarks_folder_import.json b/toolkit/components/places/tests/unit/mobile_bookmarks_folder_import.json
new file mode 100644
index 0000000000..930b7a8382
--- /dev/null
+++ b/toolkit/components/places/tests/unit/mobile_bookmarks_folder_import.json
@@ -0,0 +1,135 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1475084731479000,
+  "lastModified": 1475084731479000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731768000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "X6lUyOspVYwi",
+          "title": "Test Pilot",
+          "index": 0,
+          "dateAdded": 1475084731768000,
+          "lastModified": 1475084731768000,
+          "id": 3,
+          "type": "text/x-moz-place",
+          "uri": "https://testpilot.firefox.com/"
+        },
+        {
+          "guid": "XF4yRP6bTuil",
+          "title": "Mobile bookmarks query",
+          "index": 1,
+          "dateAdded": 1475084731768000,
+          "lastModified": 1475084731768000,
+          "id": 11,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=101"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 4,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "buy7711R3ZgE",
+          "title": "MDN",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 5,
+          "type": "text/x-moz-place",
+          "uri": "https://developer.mozilla.org"
+        }
+      ]
+    },
+    {
+      "guid": "3qmd_imziEBE",
+      "title": "Mobile Bookmarks",
+      "index": 5,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731770000,
+      "id": 101,
+      "annos": [
+        {
+          "name": "mobile/bookmarksRoot",
+          "flags": 0,
+          "expires": 4,
+          "value": 1
+        },
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "mimeType": null,
+          "type": 3,
+          "value": "A description of the mobile folder that should be ignored on import"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "children": [
+        {
+          "guid": "_o8e1_zxTJFg",
+          "title": "Get Firefox!",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 7,
+          "type": "text/x-moz-place",
+          "uri": "http://getfirefox.com/"
+        },
+        {
+          "guid": "QCtSqkVYUbXB",
+          "title": "Get Thunderbird!",
+          "index": 1,
+          "dateAdded": 1475084731770000,
+          "lastModified": 1475084731770000,
+          "id": 8,
+          "type": "text/x-moz-place",
+          "uri": "http://getthunderbird.com/"
+        }
+      ]
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 9,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder",
+      "children": [
+        {
+          "guid": "KIa9iKZab2Z5",
+          "title": "Add-ons",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 10,
+          "type": "text/x-moz-place",
+          "uri": "https://addons.mozilla.org"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/mobile_bookmarks_folder_merge.json b/toolkit/components/places/tests/unit/mobile_bookmarks_folder_merge.json
new file mode 100644
index 0000000000..8d376bf69c
--- /dev/null
+++ b/toolkit/components/places/tests/unit/mobile_bookmarks_folder_merge.json
@@ -0,0 +1,101 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1475084731479000,
+  "lastModified": 1475084731479000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731768000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "Utodo9b0oVws",
+          "title": "Firefox Accounts",
+          "index": 0,
+          "dateAdded": 1475084731955000,
+          "lastModified": 1475084731955000,
+          "id": 3,
+          "type": "text/x-moz-place",
+          "uri": "https://accounts.firefox.com/"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 4,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder"
+    },
+    {
+      "guid": "3qmd_imziEBE",
+      "title": "Mobile Bookmarks",
+      "index": 5,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731770000,
+      "id": 5,
+      "annos": [
+        {
+          "name": "mobile/bookmarksRoot",
+          "flags": 0,
+          "expires": 4,
+          "value": 1
+        },
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "mimeType": null,
+          "type": 3,
+          "value": "A description of the mobile folder that should be ignored on import"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "children": [
+        {
+          "guid": "a17yW6-nTxEJ",
+          "title": "Mozilla",
+          "index": 0,
+          "dateAdded": 1475084731959000,
+          "lastModified": 1475084731959000,
+          "id": 6,
+          "type": "text/x-moz-place",
+          "uri": "https://mozilla.org/"
+        },
+        {
+          "guid": "xV10h9Wi3FBM",
+          "title": "Bugzilla",
+          "index": 1,
+          "dateAdded": 1475084731961000,
+          "lastModified": 1475084731961000,
+          "id": 7,
+          "type": "text/x-moz-place",
+          "uri": "https://bugzilla.mozilla.org/"
+        }
+      ]
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 8,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder"
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/mobile_bookmarks_multiple_folders.json b/toolkit/components/places/tests/unit/mobile_bookmarks_multiple_folders.json
new file mode 100644
index 0000000000..3c5cb63194
--- /dev/null
+++ b/toolkit/components/places/tests/unit/mobile_bookmarks_multiple_folders.json
@@ -0,0 +1,159 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1475084731479000,
+  "lastModified": 1475084731479000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731768000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "buy7711R3ZgE",
+          "title": "MDN",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 3,
+          "type": "text/x-moz-place",
+          "uri": "https://developer.mozilla.org"
+        },
+        {
+          "guid": "F_LBgd1fS_uQ",
+          "title": "Mobile bookmarks query for first folder",
+          "index": 1,
+          "dateAdded": 1475084731768000,
+          "lastModified": 1475084731768000,
+          "id": 11,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=101"
+        },
+        {
+          "guid": "oIpmQXMWsXvY",
+          "title": "Mobile bookmarks query for second folder",
+          "index": 2,
+          "dateAdded": 1475084731768000,
+          "lastModified": 1475084731768000,
+          "id": 12,
+          "type": "text/x-moz-place",
+          "uri": "place:folder=102"
+        }
+      ]
+    },
+    {
+      "guid": "3qmd_imziEBE",
+      "title": "Mobile Bookmarks",
+      "index": 5,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731770000,
+      "id": 101,
+      "annos": [
+        {
+          "name": "mobile/bookmarksRoot",
+          "flags": 0,
+          "expires": 4,
+          "value": 1
+        },
+        {
+          "name": "bookmarkProperties/description",
+          "flags": 0,
+          "expires": 4,
+          "mimeType": null,
+          "type": 3,
+          "value": "A description of the mobile folder that should be ignored on import"
+        }
+      ],
+      "type": "text/x-moz-place-container",
+      "children": [
+        {
+          "guid": "a17yW6-nTxEJ",
+          "title": "Mozilla",
+          "index": 0,
+          "dateAdded": 1475084731959000,
+          "lastModified": 1475084731959000,
+          "id": 5,
+          "type": "text/x-moz-place",
+          "uri": "https://mozilla.org/"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 6,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder",
+      "children": [
+        {
+          "guid": "Utodo9b0oVws",
+          "title": "Firefox Accounts",
+          "index": 0,
+          "dateAdded": 1475084731955000,
+          "lastModified": 1475084731955000,
+          "id": 7,
+          "type": "text/x-moz-place",
+          "uri": "https://accounts.firefox.com/"
+        }
+      ]
+    },
+    {
+      "guid": "o4YjJpgsufU-",
+      "title": "Mobile Bookmarks",
+      "index": 7,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731770000,
+      "id": 102,
+      "annos": [
+        { "name": "mobile/bookmarksRoot", "flags": 0, "expires": 4, "value": 1 }
+      ],
+      "type": "text/x-moz-place-container",
+      "children": [
+        {
+          "guid": "sSZ86WT9WbN3",
+          "title": "DXR",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 9,
+          "type": "text/x-moz-place",
+          "uri": "https://dxr.mozilla.org"
+        }
+      ]
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 10,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder",
+      "children": [
+        {
+          "guid": "xV10h9Wi3FBM",
+          "title": "Bugzilla",
+          "index": 1,
+          "dateAdded": 1475084731961000,
+          "lastModified": 1475084731961000,
+          "id": 11,
+          "type": "text/x-moz-place",
+          "uri": "https://bugzilla.mozilla.org/"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/mobile_bookmarks_root_import.json b/toolkit/components/places/tests/unit/mobile_bookmarks_root_import.json
new file mode 100644
index 0000000000..33908e1fea
--- /dev/null
+++ b/toolkit/components/places/tests/unit/mobile_bookmarks_root_import.json
@@ -0,0 +1,89 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1475084731479000,
+  "lastModified": 1475084731479000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731768000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "X6lUyOspVYwi",
+          "title": "Test Pilot",
+          "index": 0,
+          "dateAdded": 1475084731768000,
+          "lastModified": 1475084731768000,
+          "id": 3,
+          "type": "text/x-moz-place",
+          "uri": "https://testpilot.firefox.com/"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 4,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder"
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731742000,
+      "id": 5,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder"
+    },
+    {
+      "guid": "mobile______",
+      "title": "Mobile Bookmarks",
+      "index": 4,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731770000,
+      "id": 6,
+      "annos": [
+        { "name": "mobile/bookmarksRoot", "flags": 0, "expires": 4, "value": 1 }
+      ],
+      "type": "text/x-moz-place-container",
+      "root": "mobileFolder",
+      "children": [
+        {
+          "guid": "_o8e1_zxTJFg",
+          "title": "Get Firefox!",
+          "index": 0,
+          "dateAdded": 1475084731769000,
+          "lastModified": 1475084731769000,
+          "id": 7,
+          "type": "text/x-moz-place",
+          "uri": "http://getfirefox.com/"
+        },
+        {
+          "guid": "QCtSqkVYUbXB",
+          "title": "Get Thunderbird!",
+          "index": 1,
+          "dateAdded": 1475084731770000,
+          "lastModified": 1475084731770000,
+          "id": 8,
+          "type": "text/x-moz-place",
+          "uri": "http://getthunderbird.com/"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/mobile_bookmarks_root_merge.json b/toolkit/components/places/tests/unit/mobile_bookmarks_root_merge.json
new file mode 100644
index 0000000000..97af52c44a
--- /dev/null
+++ b/toolkit/components/places/tests/unit/mobile_bookmarks_root_merge.json
@@ -0,0 +1,89 @@
+{
+  "guid": "root________",
+  "title": "",
+  "index": 0,
+  "dateAdded": 1475084731479000,
+  "lastModified": 1475084731479000,
+  "id": 1,
+  "type": "text/x-moz-place-container",
+  "root": "placesRoot",
+  "children": [
+    {
+      "guid": "menu________",
+      "title": "Bookmarks Menu",
+      "index": 0,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731955000,
+      "id": 2,
+      "type": "text/x-moz-place-container",
+      "root": "bookmarksMenuFolder",
+      "children": [
+        {
+          "guid": "Utodo9b0oVws",
+          "title": "Firefox Accounts",
+          "index": 0,
+          "dateAdded": 1475084731955000,
+          "lastModified": 1475084731955000,
+          "id": 3,
+          "type": "text/x-moz-place",
+          "uri": "https://accounts.firefox.com/"
+        }
+      ]
+    },
+    {
+      "guid": "toolbar_____",
+      "title": "Bookmarks Toolbar",
+      "index": 1,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731938000,
+      "id": 4,
+      "type": "text/x-moz-place-container",
+      "root": "toolbarFolder"
+    },
+    {
+      "guid": "unfiled_____",
+      "title": "Other Bookmarks",
+      "index": 3,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731938000,
+      "id": 5,
+      "type": "text/x-moz-place-container",
+      "root": "unfiledBookmarksFolder"
+    },
+    {
+      "guid": "mobile______",
+      "title": "Mobile Bookmarks",
+      "index": 4,
+      "dateAdded": 1475084731479000,
+      "lastModified": 1475084731961000,
+      "id": 6,
+      "annos": [
+        { "name": "mobile/bookmarksRoot", "flags": 0, "expires": 4, "value": 1 }
+      ],
+      "type": "text/x-moz-place-container",
+      "root": "mobileFolder",
+      "children": [
+        {
+          "guid": "a17yW6-nTxEJ",
+          "title": "Mozilla",
+          "index": 0,
+          "dateAdded": 1475084731959000,
+          "lastModified": 1475084731959000,
+          "id": 7,
+          "type": "text/x-moz-place",
+          "uri": "https://mozilla.org/"
+        },
+        {
+          "guid": "xV10h9Wi3FBM",
+          "title": "Bugzilla",
+          "index": 1,
+          "dateAdded": 1475084731961000,
+          "lastModified": 1475084731961000,
+          "id": 8,
+          "type": "text/x-moz-place",
+          "uri": "https://bugzilla.mozilla.org/"
+        }
+      ]
+    }
+  ]
+}
diff --git a/toolkit/components/places/tests/unit/noRoot.sqlite b/toolkit/components/places/tests/unit/noRoot.sqlite
new file mode 100644
index 0000000000..02bd907891
Binary files /dev/null and b/toolkit/components/places/tests/unit/noRoot.sqlite differ
diff --git a/toolkit/components/places/tests/unit/places.sparse.sqlite b/toolkit/components/places/tests/unit/places.sparse.sqlite
new file mode 100644
index 0000000000..915089021c
Binary files /dev/null and b/toolkit/components/places/tests/unit/places.sparse.sqlite differ
diff --git a/toolkit/components/places/tests/unit/test_1085291.js b/toolkit/components/places/tests/unit/test_1085291.js
new file mode 100644
index 0000000000..b7ec4181f0
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_1085291.js
@@ -0,0 +1,48 @@
+add_task(async function () {
+  // test that nodes inserted by incremental update for bookmarks of all types
+  // have the extra bookmark properties (bookmarkGuid, dateAdded, lastModified).
+
+  // getFolderContents opens the root node.
+  let root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.toolbarGuid
+  ).root;
+
+  async function insertAndTest(bmInfo) {
+    bmInfo = await PlacesUtils.bookmarks.insert(bmInfo);
+    let node = root.getChild(root.childCount - 1);
+    Assert.equal(node.bookmarkGuid, bmInfo.guid);
+    Assert.equal(node.dateAdded, bmInfo.dateAdded * 1000);
+    Assert.equal(node.lastModified, bmInfo.lastModified * 1000);
+  }
+
+  // Normal bookmark.
+  await insertAndTest({
+    parentGuid: root.bookmarkGuid,
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    title: "Test Bookmark",
+    url: "http://test.url.tld",
+  });
+
+  // place: query
+  await insertAndTest({
+    parentGuid: root.bookmarkGuid,
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    title: "Test Query",
+    url: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
+  });
+
+  // folder
+  await insertAndTest({
+    parentGuid: root.bookmarkGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "Test Folder",
+  });
+
+  // separator
+  await insertAndTest({
+    parentGuid: root.bookmarkGuid,
+    type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+  });
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_1105208.js b/toolkit/components/places/tests/unit/test_1105208.js
new file mode 100644
index 0000000000..6b3f31f96a
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_1105208.js
@@ -0,0 +1,25 @@
+// Test that result node for folder shortcuts get the target folder title if
+// the shortcut itself has no title set.
+add_task(async function () {
+  let folder = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "fake",
+  });
+
+  let shortcutInfo = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: `place:parent=${folder.guid}`,
+  });
+
+  let unfiledRoot = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  let shortcutNode = unfiledRoot.getChild(unfiledRoot.childCount - 1);
+  Assert.equal(shortcutNode.bookmarkGuid, shortcutInfo.guid);
+
+  Assert.equal(shortcutNode.title, folder.title);
+
+  unfiledRoot.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_1105866.js b/toolkit/components/places/tests/unit/test_1105866.js
new file mode 100644
index 0000000000..76ecfb6b7d
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_1105866.js
@@ -0,0 +1,77 @@
+add_task(async function test_folder_shortcuts() {
+  let shortcutInfo = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: `place:parent=${PlacesUtils.bookmarks.toolbarGuid}`,
+  });
+
+  let unfiledRoot = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  let shortcutNode = unfiledRoot.getChild(unfiledRoot.childCount - 1);
+  Assert.strictEqual(
+    shortcutNode.itemId,
+    await PlacesUtils.promiseItemId(shortcutInfo.guid)
+  );
+  Assert.strictEqual(
+    PlacesUtils.asQuery(shortcutNode).folderItemId,
+    await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid)
+  );
+  Assert.strictEqual(shortcutNode.bookmarkGuid, shortcutInfo.guid);
+  Assert.strictEqual(
+    PlacesUtils.asQuery(shortcutNode).targetFolderGuid,
+    PlacesUtils.bookmarks.toolbarGuid
+  );
+
+  // test that a node added incrementally also behaves just as well.
+  shortcutInfo = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: `place:parent=${PlacesUtils.bookmarks.menuGuid}`,
+  });
+  shortcutNode = unfiledRoot.getChild(unfiledRoot.childCount - 1);
+  Assert.strictEqual(
+    shortcutNode.itemId,
+    await PlacesUtils.promiseItemId(shortcutInfo.guid)
+  );
+  Assert.strictEqual(
+    PlacesUtils.asQuery(shortcutNode).folderItemId,
+    await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.menuGuid)
+  );
+  Assert.strictEqual(shortcutNode.bookmarkGuid, shortcutInfo.guid);
+  Assert.strictEqual(
+    PlacesUtils.asQuery(shortcutNode).targetFolderGuid,
+    PlacesUtils.bookmarks.menuGuid
+  );
+
+  unfiledRoot.containerOpen = false;
+});
+
+add_task(async function test_plain_folder() {
+  let folderInfo = await PlacesUtils.bookmarks.insert({
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+
+  let unfiledRoot = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  let lastChild = unfiledRoot.getChild(unfiledRoot.childCount - 1);
+  Assert.strictEqual(lastChild.bookmarkGuid, folderInfo.guid);
+  Assert.strictEqual(
+    PlacesUtils.asQuery(lastChild).targetFolderGuid,
+    folderInfo.guid
+  );
+});
+
+add_task(async function test_non_item_query() {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  Assert.strictEqual(root.itemId, -1);
+  Assert.strictEqual(PlacesUtils.asQuery(root).folderItemId, -1);
+  Assert.strictEqual(root.bookmarkGuid, "");
+  Assert.strictEqual(PlacesUtils.asQuery(root).targetFolderGuid, "");
+});
diff --git a/toolkit/components/places/tests/unit/test_1606731.js b/toolkit/components/places/tests/unit/test_1606731.js
new file mode 100644
index 0000000000..89e8ec0498
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_1606731.js
@@ -0,0 +1,21 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+let hs = PlacesUtils.history;
+
+/* Test for diacritic-insensitive history search */
+
+add_task(async function test_execute() {
+  const TEST_URL = "http://example.net/El_%C3%81rea_51";
+  const SEARCH_TERM = "area";
+  await PlacesTestUtils.addVisits(uri(TEST_URL));
+  let query = hs.getNewQuery();
+  query.searchTerms = SEARCH_TERM;
+  let options = hs.getNewQueryOptions();
+  let result = hs.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.ok(result.root.childCount == 1);
+});
diff --git a/toolkit/components/places/tests/unit/test_331487.js b/toolkit/components/places/tests/unit/test_331487.js
new file mode 100644
index 0000000000..2d4f5f8279
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_331487.js
@@ -0,0 +1,113 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Get history service
+try {
+  var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+    Ci.nsINavHistoryService
+  );
+} catch (ex) {
+  do_throw("Could not get history service\n");
+}
+
+add_task(async function test_hierarchical_query() {
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        title: "test folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            title: "1 title",
+            url: "http://a1.com/",
+          },
+          {
+            title: "subfolder 1",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            children: [
+              {
+                title: "2 title",
+                url: "http://a2.com/",
+              },
+              {
+                title: "subfolder 2",
+                type: PlacesUtils.bookmarks.TYPE_FOLDER,
+                children: [
+                  {
+                    title: "3 title",
+                    url: "http://a3.com/",
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+
+  let [folderGuid, b1, sf1, b2, sf2, b3] = bookmarks.map(
+    bookmark => bookmark.guid
+  );
+
+  // bookmark query that should result in the "hierarchical" result
+  // because there is one query, one folder,
+  // no begin time, no end time, no domain, no uri, no search term
+  // and no max results.  See GetSimpleBookmarksQueryFolder()
+  // for more details.
+  var options = histsvc.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+  var query = histsvc.getNewQuery();
+  query.setParents([folderGuid]);
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.getChild(0).bookmarkGuid, b1);
+  Assert.equal(root.getChild(1).bookmarkGuid, sf1);
+
+  // check the contents of the subfolder
+  var sf1Node = root.getChild(1);
+  sf1Node = sf1Node.QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  sf1Node.containerOpen = true;
+  Assert.equal(sf1Node.childCount, 2);
+  Assert.equal(sf1Node.getChild(0).bookmarkGuid, b2);
+  Assert.equal(sf1Node.getChild(1).bookmarkGuid, sf2);
+
+  // check the contents of the subfolder's subfolder
+  var sf2Node = sf1Node.getChild(1);
+  sf2Node = sf2Node.QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  sf2Node.containerOpen = true;
+  Assert.equal(sf2Node.childCount, 1);
+  Assert.equal(sf2Node.getChild(0).bookmarkGuid, b3);
+
+  sf2Node.containerOpen = false;
+  sf1Node.containerOpen = false;
+  root.containerOpen = false;
+
+  // bookmark query that should result in a flat list
+  // because we specified max results
+  options = histsvc.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+  options.maxResults = 10;
+  query = histsvc.getNewQuery();
+  query.setParents([folderGuid]);
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 3);
+  Assert.equal(root.getChild(0).bookmarkGuid, b1);
+  Assert.equal(root.getChild(1).bookmarkGuid, b2);
+  Assert.equal(root.getChild(2).bookmarkGuid, b3);
+  root.containerOpen = false;
+
+  // XXX TODO
+  // test that if we have: more than one query,
+  // multiple folders, a begin time, an end time, a domain, a uri
+  // or a search term, that we get the (correct) flat list results
+  // (like we do when specified maxResults)
+});
diff --git a/toolkit/components/places/tests/unit/test_384370.js b/toolkit/components/places/tests/unit/test_384370.js
new file mode 100644
index 0000000000..88c0fd5976
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_384370.js
@@ -0,0 +1,188 @@
+var tagData = [
+  { uri: uri("http://slint.us"), tags: ["indie", "kentucky", "music"] },
+  {
+    uri: uri("http://en.wikipedia.org/wiki/Diplodocus"),
+    tags: ["dinosaur", "dj", "rad word"],
+  },
+];
+
+var bookmarkData = [
+  { uri: uri("http://slint.us"), title: "indie, kentucky, music" },
+  {
+    uri: uri("http://en.wikipedia.org/wiki/Diplodocus"),
+    title: "dinosaur, dj, rad word",
+  },
+];
+
+/*
+  HTML+FEATURES SUMMARY:
+  - import legacy bookmarks
+  - export as json, import, test (tests integrity of html > json)
+  - export as html, import, test (tests integrity of json > html)
+
+  BACKUP/RESTORE SUMMARY:
+  - create a bookmark in each root
+  - tag multiple URIs with multiple tags
+  - export as json, import, test
+*/
+add_task(async function () {
+  // Remove eventual bookmarks.exported.json.
+  let jsonFile = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.json"
+  );
+  await IOUtils.remove(jsonFile, { ignoreAbsent: true });
+
+  // Test importing a pre-Places canonical bookmarks file.
+  // Note: we do not empty the db before this import to catch bugs like 380999
+  let htmlFile = PathUtils.join(do_get_cwd().path, "bookmarks.preplaces.html");
+  await BookmarkHTMLUtils.importFromFile(htmlFile, { replace: true });
+
+  // Populate the database.
+  for (let { uri, tags } of tagData) {
+    PlacesUtils.tagging.tagURI(uri, tags);
+  }
+  for (let { uri, title } of bookmarkData) {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: uri,
+      title,
+    });
+  }
+  for (let { uri, title } of bookmarkData) {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      url: uri,
+      title,
+    });
+  }
+
+  await validate("initial database");
+
+  // Test exporting a Places canonical json file.
+  // 1. export to bookmarks.exported.json
+  await BookmarkJSONUtils.exportToFile(jsonFile);
+  info("exported json");
+
+  // 2. empty bookmarks db
+  // 3. import bookmarks.exported.json
+  await BookmarkJSONUtils.importFromFile(jsonFile, { replace: true });
+  info("imported json");
+
+  // 4. run the test-suite
+  await validate("re-imported json");
+  info("validated import");
+});
+
+async function validate(infoMsg) {
+  info(`Validating ${infoMsg}: testMenuBookmarks`);
+  await testMenuBookmarks();
+  info(`Validating ${infoMsg}: testToolbarBookmarks`);
+  await testToolbarBookmarks();
+  info(`Validating ${infoMsg}: testUnfiledBookmarks`);
+  testUnfiledBookmarks();
+  info(`Validating ${infoMsg}: testTags`);
+  testTags();
+  await PlacesTestUtils.promiseAsyncUpdates();
+}
+
+// Tests a bookmarks datastore that has a set of bookmarks, etc
+// that flex each supported field and feature.
+async function testMenuBookmarks() {
+  let root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.menuGuid).root;
+  Assert.equal(root.childCount, 3);
+
+  let separatorNode = root.getChild(1);
+  Assert.equal(separatorNode.type, separatorNode.RESULT_TYPE_SEPARATOR);
+
+  let folderNode = root.getChild(2);
+  Assert.equal(folderNode.type, folderNode.RESULT_TYPE_FOLDER);
+  Assert.equal(folderNode.title, "test");
+  let folder = await PlacesUtils.bookmarks.fetch(folderNode.bookmarkGuid);
+  Assert.equal(folder.dateAdded.getTime(), 1177541020000);
+
+  Assert.equal(PlacesUtils.asQuery(folderNode).hasChildren, true);
+
+  // open test folder, and test the children
+  folderNode.containerOpen = true;
+  Assert.equal(folderNode.childCount, 1);
+
+  let bookmarkNode = folderNode.getChild(0);
+  Assert.equal("http://test/post", bookmarkNode.uri);
+  Assert.equal("test post keyword", bookmarkNode.title);
+  Assert.equal(bookmarkNode.dateAdded, 1177375336000000);
+
+  let entry = await PlacesUtils.keywords.fetch({ url: bookmarkNode.uri });
+  Assert.equal("test", entry.keyword);
+  Assert.equal("hidden1%3Dbar&text1%3D%25s", entry.postData);
+
+  let pageInfo = await PlacesUtils.history.fetch(bookmarkNode.uri, {
+    includeAnnotations: true,
+  });
+  Assert.equal(
+    pageInfo.annotations.get(PlacesUtils.CHARSET_ANNO),
+    "ISO-8859-1",
+    "Should have the correct charset"
+  );
+
+  folderNode.containerOpen = false;
+  root.containerOpen = false;
+}
+
+async function testToolbarBookmarks() {
+  let root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.toolbarGuid
+  ).root;
+
+  // child count (add 2 for pre-existing items, one of the feeds is skipped
+  // because it doesn't have href)
+  Assert.equal(root.childCount, bookmarkData.length + 2);
+
+  // Livemarks are no more supported but may still exist in old html files.
+  let legacyLivemarkNode = root.getChild(1);
+  Assert.equal("Latest Headlines", legacyLivemarkNode.title);
+  Assert.equal(
+    "http://en-us.fxfeeds.mozilla.com/en-US/firefox/livebookmarks/",
+    legacyLivemarkNode.uri
+  );
+  Assert.equal(
+    legacyLivemarkNode.type,
+    Ci.nsINavHistoryResultNode.RESULT_TYPE_URI
+  );
+
+  // test added bookmark data
+  let bookmarkNode = root.getChild(2);
+  Assert.equal(bookmarkNode.uri, bookmarkData[0].uri.spec);
+  Assert.equal(bookmarkNode.title, bookmarkData[0].title);
+  bookmarkNode = root.getChild(3);
+  Assert.equal(bookmarkNode.uri, bookmarkData[1].uri.spec);
+  Assert.equal(bookmarkNode.title, bookmarkData[1].title);
+
+  root.containerOpen = false;
+}
+
+function testUnfiledBookmarks() {
+  let root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  ).root;
+  // child count (add 1 for pre-existing item)
+  Assert.equal(root.childCount, bookmarkData.length + 1);
+  for (let i = 1; i < root.childCount; ++i) {
+    let child = root.getChild(i);
+    Assert.equal(child.uri, bookmarkData[i - 1].uri.spec);
+    Assert.equal(child.title, bookmarkData[i - 1].title);
+    if (child.tags) {
+      Assert.equal(child.tags, bookmarkData[i - 1].title);
+    }
+  }
+  root.containerOpen = false;
+}
+
+function testTags() {
+  for (let { uri, tags } of tagData) {
+    info("Test tags for " + uri.spec + ": " + tags + "\n");
+    let foundTags = PlacesUtils.tagging.getTagsForURI(uri);
+    Assert.equal(foundTags.length, tags.length);
+    Assert.ok(tags.every(tag => foundTags.includes(tag)));
+  }
+}
diff --git a/toolkit/components/places/tests/unit/test_385397.js b/toolkit/components/places/tests/unit/test_385397.js
new file mode 100644
index 0000000000..7746b89657
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_385397.js
@@ -0,0 +1,152 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const TOTAL_SITES = 20;
+
+add_task(async function test_execute() {
+  let now = (Date.now() - 10000) * 1000;
+
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let site = "http://www.test-" + i + ".com/";
+    let testURI = uri(site);
+    let testImageURI = uri(site + "blank.gif");
+    let when = now + i * TOTAL_SITES * 1000;
+    await PlacesTestUtils.addVisits([
+      { uri: testURI, visitDate: when, transition: TRANSITION_TYPED },
+      {
+        uri: testImageURI,
+        visitDate: when + 1000,
+        transition: TRANSITION_EMBED,
+      },
+      {
+        uri: testImageURI,
+        visitDate: when + 2000,
+        transition: TRANSITION_FRAMED_LINK,
+      },
+      { uri: testURI, visitDate: when + 3000, transition: TRANSITION_LINK },
+    ]);
+  }
+
+  // verify our visits AS_VISIT, ordered by date descending
+  // including hidden
+  // we should get 80 visits:
+  // http://www.test-19.com/
+  // http://www.test-19.com/blank.gif
+  // http://www.test-19.com/
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-0.com/
+  // http://www.test-0.com/blank.gif
+  // http://www.test-0.com/
+  // http://www.test-0.com/
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+  options.includeHidden = true;
+  let root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  let cc = root.childCount;
+  // Embed visits are not added to the database, thus they won't appear.
+  Assert.equal(cc, 3 * TOTAL_SITES);
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let index = i * 3;
+    let node = root.getChild(index);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+    node = root.getChild(++index);
+    Assert.equal(node.uri, site + "blank.gif");
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+    node = root.getChild(++index);
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+
+  // verify our visits AS_VISIT, ordered by date descending
+  // we should get 40 visits:
+  // http://www.test-19.com/
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-0.com/
+  // http://www.test-0.com/
+  options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+  root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  cc = root.childCount;
+  // 2 * TOTAL_SITES because we count the TYPED and LINK, but not EMBED or FRAMED
+  Assert.equal(cc, 2 * TOTAL_SITES);
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let index = i * 2;
+    let node = root.getChild(index);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+    node = root.getChild(++index);
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+
+  // test our optimized query for the places menu
+  // place:type=0&sort=4&maxResults=10
+  // verify our visits AS_URI, ordered by date descending
+  // we should get 10 visits:
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-10.com/
+  options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.maxResults = 10;
+  options.resultType = options.RESULTS_AS_URI;
+  root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  cc = root.childCount;
+  Assert.equal(cc, options.maxResults);
+  for (let i = 0; i < cc; i++) {
+    let node = root.getChild(i);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+
+  // test without a maxResults, which executes a different query
+  // but the first 10 results should be the same.
+  // verify our visits AS_URI, ordered by date descending
+  // we should get 20 visits, but the first 10 should be
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-10.com/
+  options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+  root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  cc = root.childCount;
+  Assert.equal(cc, TOTAL_SITES);
+  for (let i = 0; i < 10; i++) {
+    let node = root.getChild(i);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_399266.js b/toolkit/components/places/tests/unit/test_399266.js
new file mode 100644
index 0000000000..6f99c710f8
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_399266.js
@@ -0,0 +1,82 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const TOTAL_SITES = 20;
+
+add_task(async function test_execute() {
+  let places = [];
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    for (let j = 0; j <= i; j++) {
+      places.push({
+        uri: uri("http://www.test-" + i + ".com/"),
+        transition: TRANSITION_TYPED,
+      });
+      // because these are embedded visits, they should not show up on our
+      // query results.  If they do, we have a problem.
+      places.push({
+        uri: uri("http://www.hidden.com/hidden.gif"),
+        transition: TRANSITION_EMBED,
+      });
+      places.push({
+        uri: uri("http://www.alsohidden.com/hidden.gif"),
+        transition: TRANSITION_FRAMED_LINK,
+      });
+    }
+  }
+  await PlacesTestUtils.addVisits(places);
+
+  // test our optimized query for the "Most Visited" item
+  // in the "Smart Bookmarks" folder
+  // place:queryType=0&sort=8&maxResults=10
+  // verify our visits AS_URI, ordered by visit count descending
+  // we should get 10 visits:
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-10.com/
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_VISITCOUNT_DESCENDING;
+  options.maxResults = 10;
+  options.resultType = options.RESULTS_AS_URI;
+  let root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  let cc = root.childCount;
+  Assert.equal(cc, options.maxResults);
+  for (let i = 0; i < cc; i++) {
+    let node = root.getChild(i);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+
+  // test without a maxResults, which executes a different query
+  // but the first 10 results should be the same.
+  // verify our visits AS_URI, ordered by visit count descending
+  // we should get 20 visits, but the first 10 should be
+  // http://www.test-19.com/
+  // ...
+  // http://www.test-10.com/
+  options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_VISITCOUNT_DESCENDING;
+  options.resultType = options.RESULTS_AS_URI;
+  root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  cc = root.childCount;
+  Assert.equal(cc, TOTAL_SITES);
+  for (let i = 0; i < 10; i++) {
+    let node = root.getChild(i);
+    let site = "http://www.test-" + (TOTAL_SITES - 1 - i) + ".com/";
+    Assert.equal(node.uri, site);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_402799.js b/toolkit/components/places/tests/unit/test_402799.js
new file mode 100644
index 0000000000..f621911cce
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_402799.js
@@ -0,0 +1,60 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Get history services
+try {
+  var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+    Ci.nsINavHistoryService
+  );
+} catch (ex) {
+  do_throw("Could not get history services\n");
+}
+
+// Get tagging service
+try {
+  var tagssvc = Cc["@mozilla.org/browser/tagging-service;1"].getService(
+    Ci.nsITaggingService
+  );
+} catch (ex) {
+  do_throw("Could not get tagging service\n");
+}
+
+add_task(async function test_query_only_returns_bookmarks_not_tags() {
+  const url = "http://foo.bar/";
+
+  // create 2 bookmarks on the same uri
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "title 1",
+    url,
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "title 2",
+    url,
+  });
+  // add some tags
+  tagssvc.tagURI(uri(url), ["foo", "bar", "foobar", "foo bar"]);
+
+  // check that a generic bookmark query returns only real bookmarks
+  let options = histsvc.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+
+  let query = histsvc.getNewQuery();
+  let result = histsvc.executeQuery(query, options);
+  let root = result.root;
+
+  root.containerOpen = true;
+  let cc = root.childCount;
+  Assert.equal(cc, 2);
+  let node1 = root.getChild(0);
+  node1 = await PlacesUtils.bookmarks.fetch(node1.bookmarkGuid);
+  Assert.equal(node1.parentGuid, PlacesUtils.bookmarks.menuGuid);
+  let node2 = root.getChild(1);
+  node2 = await PlacesUtils.bookmarks.fetch(node2.bookmarkGuid);
+  Assert.equal(node2.parentGuid, PlacesUtils.bookmarks.toolbarGuid);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_412132.js b/toolkit/components/places/tests/unit/test_412132.js
new file mode 100644
index 0000000000..65653b5c0a
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_412132.js
@@ -0,0 +1,181 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * TEST DESCRIPTION:
+ *
+ * Tests patch to Bug 412132:
+ * https://bugzilla.mozilla.org/show_bug.cgi?id=412132
+ */
+
+const TEST_URL0 = "http://example.com/";
+const TEST_URL1 = "http://example.com/1";
+const TEST_URL2 = "http://example.com/2";
+
+add_task(async function changeuri_unvisited_bookmark() {
+  info(
+    "After changing URI of bookmark, frecency of bookmark's " +
+      "original URI should be zero if original URI is unvisited and " +
+      "no longer bookmarked."
+  );
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URL1,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL1,
+    }),
+    0,
+    "Bookmarked => frecency of URI should be != 0"
+  );
+
+  await PlacesUtils.bookmarks.update({
+    guid: bookmark.guid,
+    url: TEST_URL2,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL1,
+    }),
+    0,
+    "Unvisited URI no longer bookmarked => frecency should = 0"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function changeuri_visited_bookmark() {
+  info(
+    "After changing URI of bookmark, frecency of bookmark's " +
+      "original URI should not be zero if original URI is visited."
+  );
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URL1,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL1,
+    }),
+    0,
+    "Bookmarked => frecency of URI should be != 0"
+  );
+
+  await PlacesTestUtils.addVisits(TEST_URL1);
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  await PlacesUtils.bookmarks.update({
+    guid: bookmark.guid,
+    url: TEST_URL2,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL1,
+    }),
+    0,
+    "*Visited* URI no longer bookmarked => frecency should != 0"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function changeuri_bookmark_still_bookmarked() {
+  info(
+    "After changing URI of bookmark, frecency of bookmark's " +
+      "original URI should not be zero if original URI is still " +
+      "bookmarked."
+  );
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark 1 title",
+    url: TEST_URL1,
+  });
+
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark 2 title",
+    url: TEST_URL1,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL1,
+    }),
+    0,
+    "Bookmarked => frecency of URI should be != 0"
+  );
+
+  await PlacesUtils.bookmarks.update({
+    guid: bookmark.guid,
+    url: TEST_URL2,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  info("URI still bookmarked => frecency should != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URL2,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function changeuri_nonexistent_bookmark() {
+  // Try a bogus guid.
+  await Assert.rejects(
+    PlacesUtils.bookmarks.update({
+      guid: "ABCDEDFGHIJK",
+      url: TEST_URL2,
+    }),
+    /No bookmarks found for the provided GUID/,
+    "Changing the URI of a non-existent bookmark should fail."
+  );
+
+  // Now add a bookmark, delete it, and check.
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URL0,
+  });
+
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+
+  await Assert.rejects(
+    PlacesUtils.bookmarks.update({
+      guid: bookmark.guid,
+      url: TEST_URL2,
+    }),
+    /No bookmarks found for the provided GUID/,
+    "Changing the URI of a non-existent bookmark should fail."
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/unit/test_415460.js b/toolkit/components/places/tests/unit/test_415460.js
new file mode 100644
index 0000000000..3f0f7a1edb
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_415460.js
@@ -0,0 +1,37 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var hs = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+  Ci.nsINavHistoryService
+);
+
+/**
+ * Checks to see that a search has exactly one result in the database.
+ *
+ * @param aTerms
+ *        The terms to search for.
+ * @returns true if the search returns one result, false otherwise.
+ */
+function search_has_result(aTerms) {
+  var options = hs.getNewQueryOptions();
+  options.maxResults = 1;
+  options.resultType = options.RESULTS_AS_URI;
+  var query = hs.getNewQuery();
+  query.searchTerms = aTerms;
+  var result = hs.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var cc = root.childCount;
+  root.containerOpen = false;
+  return cc == 1;
+}
+
+add_task(async function test_execute() {
+  const SEARCH_TERM = "ユニコード";
+  const TEST_URL = "http://example.com/" + SEARCH_TERM + "/";
+  await PlacesTestUtils.addVisits(uri(TEST_URL));
+  Assert.ok(search_has_result(SEARCH_TERM));
+});
diff --git a/toolkit/components/places/tests/unit/test_415757.js b/toolkit/components/places/tests/unit/test_415757.js
new file mode 100644
index 0000000000..a069bc6aa3
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_415757.js
@@ -0,0 +1,92 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Checks to see that a URI is in the database.
+ *
+ * @param aURI
+ *        The URI to check.
+ * @returns true if the URI is in the DB, false otherwise.
+ */
+function uri_in_db(aURI) {
+  var options = PlacesUtils.history.getNewQueryOptions();
+  options.maxResults = 1;
+  options.resultType = options.RESULTS_AS_URI;
+  var query = PlacesUtils.history.getNewQuery();
+  query.uri = aURI;
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var cc = root.childCount;
+  root.containerOpen = false;
+  return cc == 1;
+}
+
+const TOTAL_SITES = 20;
+
+// main
+add_task(async function test_execute() {
+  // add pages to global history
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let uri = "http://www.test-" + i + ".com/";
+    let when = Date.now() * 1000 + i * TOTAL_SITES;
+    await PlacesTestUtils.addVisits({ uri, visitDate: when });
+  }
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let uri = "http://www.test.com/" + i + "/";
+    let when = Date.now() * 1000 + i * TOTAL_SITES;
+    await PlacesTestUtils.addVisits({ uri, visitDate: when });
+  }
+
+  // set a page annotation on one of the urls that will be removed
+  var testAnnoDeletedURI = "http://www.test.com/1/";
+  var testAnnoDeletedName = "foo";
+  var testAnnoDeletedValue = "bar";
+  await PlacesUtils.history.update({
+    url: testAnnoDeletedURI,
+    annotations: new Map([[testAnnoDeletedName, testAnnoDeletedValue]]),
+  });
+
+  // set a page annotation on one of the urls that will NOT be removed
+  var testAnnoRetainedURI = "http://www.test-1.com/";
+  var testAnnoRetainedName = "foo";
+  var testAnnoRetainedValue = "bar";
+  await PlacesUtils.history.update({
+    url: testAnnoRetainedURI,
+    annotations: new Map([[testAnnoRetainedName, testAnnoRetainedValue]]),
+  });
+
+  // remove pages from www.test.com
+  await PlacesUtils.history.removeByFilter({ host: "www.test.com" });
+
+  // check that all pages in www.test.com have been removed
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let site = "http://www.test.com/" + i + "/";
+    let testURI = uri(site);
+    Assert.ok(!uri_in_db(testURI));
+  }
+
+  // check that all pages in www.test-X.com have NOT been removed
+  for (let i = 0; i < TOTAL_SITES; i++) {
+    let site = "http://www.test-" + i + ".com/";
+    let testURI = uri(site);
+    Assert.ok(uri_in_db(testURI));
+  }
+
+  // check that annotation on the removed item does not exists
+  await assertNoOrphanPageAnnotations();
+
+  // check that annotation on the NOT removed item still exists
+  let pageInfo = await PlacesUtils.history.fetch(testAnnoRetainedURI, {
+    includeAnnotations: true,
+  });
+
+  Assert.equal(
+    pageInfo.annotations.get(testAnnoRetainedName),
+    testAnnoRetainedValue,
+    "Should have kept the annotation for the non-removed items"
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_419792_node_tags_property.js b/toolkit/components/places/tests/unit/test_419792_node_tags_property.js
new file mode 100644
index 0000000000..5efadb39fe
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_419792_node_tags_property.js
@@ -0,0 +1,52 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// get services
+var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+  Ci.nsINavHistoryService
+);
+var tagssvc = Cc["@mozilla.org/browser/tagging-service;1"].getService(
+  Ci.nsITaggingService
+);
+
+add_task(async function test_query_node_tags_property() {
+  // get toolbar node
+  var options = histsvc.getNewQueryOptions();
+  var query = histsvc.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var result = histsvc.executeQuery(query, options);
+  var toolbarNode = result.root;
+  toolbarNode.containerOpen = true;
+
+  // add a bookmark
+  var bookmarkURI = uri("http://foo.com");
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "",
+    url: bookmarkURI,
+  });
+
+  // get the node for the new bookmark
+  var node = toolbarNode.getChild(toolbarNode.childCount - 1);
+  Assert.equal(node.bookmarkGuid, bookmark.guid);
+
+  // confirm there's no tags via the .tags property
+  Assert.equal(node.tags, null);
+
+  // add a tag
+  tagssvc.tagURI(bookmarkURI, ["foo"]);
+  Assert.equal(node.tags, "foo");
+
+  // add another tag, to test delimiter and sorting
+  tagssvc.tagURI(bookmarkURI, ["bar"]);
+  Assert.equal(node.tags, "bar, foo");
+
+  // remove the tags, confirming the property is cleared
+  tagssvc.untagURI(bookmarkURI, null);
+  Assert.equal(node.tags, null);
+
+  toolbarNode.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_425563.js b/toolkit/components/places/tests/unit/test_425563.js
new file mode 100644
index 0000000000..8ecf8bbcc8
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_425563.js
@@ -0,0 +1,76 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function test_execute() {
+  let count_visited_URIs = [
+    "http://www.test-link.com/",
+    "http://www.test-typed.com/",
+    "http://www.test-bookmark.com/",
+    "http://www.test-redirect-permanent.com/",
+    "http://www.test-redirect-temporary.com/",
+  ];
+
+  let notcount_visited_URIs = [
+    "http://www.test-download.com/",
+    "http://www.test-framed.com/",
+    "http://www.test-reload.com/",
+  ];
+
+  // add visits, one for each transition type
+  await PlacesTestUtils.addVisits([
+    { uri: uri("http://www.test-link.com/"), transition: TRANSITION_LINK },
+    { uri: uri("http://www.test-typed.com/"), transition: TRANSITION_TYPED },
+    {
+      uri: uri("http://www.test-bookmark.com/"),
+      transition: TRANSITION_BOOKMARK,
+    },
+    {
+      uri: uri("http://www.test-framed.com/"),
+      transition: TRANSITION_FRAMED_LINK,
+    },
+    {
+      uri: uri("http://www.test-redirect-permanent.com/"),
+      transition: TRANSITION_REDIRECT_PERMANENT,
+    },
+    {
+      uri: uri("http://www.test-redirect-temporary.com/"),
+      transition: TRANSITION_REDIRECT_TEMPORARY,
+    },
+    {
+      uri: uri("http://www.test-download.com/"),
+      transition: TRANSITION_DOWNLOAD,
+    },
+    { uri: uri("http://www.test-reload.com/"), transition: TRANSITION_RELOAD },
+  ]);
+
+  // check that all links are marked as visited
+  for (let visited_uri of count_visited_URIs) {
+    Assert.ok(await PlacesUtils.history.hasVisits(uri(visited_uri)));
+  }
+  for (let visited_uri of notcount_visited_URIs) {
+    Assert.ok(await PlacesUtils.history.hasVisits(uri(visited_uri)));
+  }
+
+  // check that visit_count does not take in count embed and downloads
+  // maxVisits query are directly binded to visit_count
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_VISITCOUNT_DESCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+  options.includeHidden = true;
+  let query = PlacesUtils.history.getNewQuery();
+  query.minVisits = 1;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+
+  root.containerOpen = true;
+  let cc = root.childCount;
+  Assert.equal(cc, count_visited_URIs.length);
+
+  for (let i = 0; i < cc; i++) {
+    let node = root.getChild(i);
+    Assert.notEqual(count_visited_URIs.indexOf(node.uri), -1);
+  }
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_429505_remove_shortcuts.js b/toolkit/components/places/tests/unit/test_429505_remove_shortcuts.js
new file mode 100644
index 0000000000..d5926d0c17
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_429505_remove_shortcuts.js
@@ -0,0 +1,45 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+
+- add a folder
+- add a folder-shortcut to the new folder
+- query for the shortcut
+- remove the folder-shortcut
+- confirm the shortcut is removed from the query results
+
+*/
+
+add_task(async function test_query_with_remove_shortcut() {
+  let folder = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+  });
+
+  let query = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "",
+    url: `place:parent=${folder.guid}`,
+  });
+
+  var root = PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.toolbarGuid,
+    false,
+    true
+  ).root;
+
+  var oldCount = root.childCount;
+
+  await PlacesUtils.bookmarks.remove(query.guid);
+
+  Assert.equal(root.childCount, oldCount - 1);
+
+  root.containerOpen = false;
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
diff --git a/toolkit/components/places/tests/unit/test_433317_query_title_update.js b/toolkit/components/places/tests/unit/test_433317_query_title_update.js
new file mode 100644
index 0000000000..d8f69064d9
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_433317_query_title_update.js
@@ -0,0 +1,43 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function test_query_title_update() {
+  try {
+    var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+      Ci.nsINavHistoryService
+    );
+  } catch (ex) {
+    do_throw("Unable to initialize Places services");
+  }
+
+  // create a query bookmark
+  let bmQuery = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "test query",
+    url: "place:",
+  });
+
+  // query for that query
+  var options = histsvc.getNewQueryOptions();
+  let query = histsvc.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var queryNode = root.getChild(0);
+  Assert.equal(queryNode.title, "test query");
+
+  // change the title
+  await PlacesUtils.bookmarks.update({
+    guid: bmQuery.guid,
+    title: "foo",
+  });
+
+  // confirm the node was updated
+  Assert.equal(queryNode.title, "foo");
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_433525_hasChildren_crash.js b/toolkit/components/places/tests/unit/test_433525_hasChildren_crash.js
new file mode 100644
index 0000000000..231722e72b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_433525_hasChildren_crash.js
@@ -0,0 +1,52 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function test_execute() {
+  try {
+    var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+      Ci.nsINavHistoryService
+    );
+  } catch (ex) {
+    do_throw("Unable to initialize Places services");
+  }
+
+  // add a visit
+  var testURI = uri("http://test");
+  await PlacesTestUtils.addVisits(testURI);
+
+  // query for the visit
+  var options = histsvc.getNewQueryOptions();
+  options.maxResults = 1;
+  options.resultType = options.RESULTS_AS_URI;
+  var query = histsvc.getNewQuery();
+  query.uri = testURI;
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+
+  // check hasChildren while the container is closed
+  Assert.equal(root.hasChildren, true);
+
+  // now check via the saved search path
+  var queryURI = histsvc.queryToQueryString(query, options);
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "test query",
+    url: queryURI,
+  });
+
+  // query for that query
+  options = histsvc.getNewQueryOptions();
+  query = histsvc.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  var queryNode = root.getChild(0);
+  Assert.equal(queryNode.title, "test query");
+  queryNode.QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  Assert.equal(queryNode.hasChildren, true);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_454977.js b/toolkit/components/places/tests/unit/test_454977.js
new file mode 100644
index 0000000000..aa6437988f
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_454977.js
@@ -0,0 +1,121 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Cache actual visit_count value, filled by add_visit, used by check_results
+var visit_count = 0;
+
+// Returns the Place ID corresponding to an added visit.
+async function task_add_visit(aURI, aVisitType) {
+  // Wait for a visits notification and get the visitId.
+  let visitId;
+  let visitsPromise = PlacesTestUtils.waitForNotification(
+    "page-visited",
+    visits => {
+      visitId = visits[0].visitId;
+      let { url } = visits[0];
+      return url == aURI.spec;
+    }
+  );
+
+  // Add visits.
+  await PlacesTestUtils.addVisits([
+    {
+      uri: aURI,
+      transition: aVisitType,
+    },
+  ]);
+
+  if (aVisitType != TRANSITION_EMBED) {
+    await visitsPromise;
+  }
+
+  // Increase visit_count if applicable
+  if (
+    aVisitType != 0 &&
+    aVisitType != TRANSITION_EMBED &&
+    aVisitType != TRANSITION_FRAMED_LINK &&
+    aVisitType != TRANSITION_DOWNLOAD &&
+    aVisitType != TRANSITION_RELOAD
+  ) {
+    visit_count++;
+  }
+
+  // Get the place id
+  if (visitId > 0) {
+    let sql = "SELECT place_id FROM moz_historyvisits WHERE id = ?1";
+    let stmt = DBConn().createStatement(sql);
+    stmt.bindByIndex(0, visitId);
+    Assert.ok(stmt.executeStep());
+    let placeId = stmt.getInt64(0);
+    stmt.finalize();
+    Assert.ok(placeId > 0);
+    return placeId;
+  }
+  return 0;
+}
+
+/**
+ * Checks for results consistency, using visit_count as constraint
+ * @param   aExpectedCount
+ *          Number of history results we are expecting (excluded hidden ones)
+ * @param   aExpectedCountWithHidden
+ *          Number of history results we are expecting (included hidden ones)
+ */
+function check_results(aExpectedCount, aExpectedCountWithHidden) {
+  let query = PlacesUtils.history.getNewQuery();
+  // used to check visit_count
+  query.minVisits = visit_count;
+  query.maxVisits = visit_count;
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_HISTORY;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  // Children without hidden ones
+  Assert.equal(root.childCount, aExpectedCount);
+  root.containerOpen = false;
+
+  // Execute again with includeHidden = true
+  // This will ensure visit_count is correct
+  options.includeHidden = true;
+  root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  // Children with hidden ones
+  Assert.equal(root.childCount, aExpectedCountWithHidden);
+  root.containerOpen = false;
+}
+
+// main
+
+add_task(async function test_execute() {
+  const TEST_URI = uri("http://test.mozilla.org/");
+
+  // Add a visit that force hidden
+  await task_add_visit(TEST_URI, TRANSITION_EMBED);
+  check_results(0, 0);
+
+  let placeId = await task_add_visit(TEST_URI, TRANSITION_FRAMED_LINK);
+  check_results(0, 1);
+
+  // Add a visit that force unhide and check the place id.
+  // - We expect that the place gets hidden = 0 while retaining the same
+  //   place id and a correct visit_count.
+  Assert.equal(await task_add_visit(TEST_URI, TRANSITION_TYPED), placeId);
+  check_results(1, 1);
+
+  // Add a visit that should not increase visit_count
+  Assert.equal(await task_add_visit(TEST_URI, TRANSITION_RELOAD), placeId);
+  check_results(1, 1);
+
+  // Add a visit that should not increase visit_count
+  Assert.equal(await task_add_visit(TEST_URI, TRANSITION_DOWNLOAD), placeId);
+  check_results(1, 1);
+
+  // Add a visit, check that hidden is not overwritten
+  // - We expect that the place has still hidden = 0, while retaining
+  //   correct visit_count.
+  await task_add_visit(TEST_URI, TRANSITION_EMBED);
+  check_results(1, 1);
+});
diff --git a/toolkit/components/places/tests/unit/test_463863.js b/toolkit/components/places/tests/unit/test_463863.js
new file mode 100644
index 0000000000..d524c00cd3
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_463863.js
@@ -0,0 +1,56 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * TEST DESCRIPTION:
+ *
+ * This test checks that in a basic history query all transition types visits
+ * appear but TRANSITION_EMBED and TRANSITION_FRAMED_LINK ones.
+ */
+
+var transitions = [
+  TRANSITION_LINK,
+  TRANSITION_TYPED,
+  TRANSITION_BOOKMARK,
+  TRANSITION_EMBED,
+  TRANSITION_FRAMED_LINK,
+  TRANSITION_REDIRECT_PERMANENT,
+  TRANSITION_REDIRECT_TEMPORARY,
+  TRANSITION_DOWNLOAD,
+];
+
+function runQuery(aResultType) {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = aResultType;
+  let root = PlacesUtils.history.executeQuery(
+    PlacesUtils.history.getNewQuery(),
+    options
+  ).root;
+  root.containerOpen = true;
+  let cc = root.childCount;
+  Assert.equal(cc, transitions.length - 2);
+
+  for (let i = 0; i < cc; i++) {
+    let node = root.getChild(i);
+    // Check that all transition types but EMBED and FRAMED appear in results
+    Assert.notEqual(node.uri.substr(6, 1), TRANSITION_EMBED);
+    Assert.notEqual(node.uri.substr(6, 1), TRANSITION_FRAMED_LINK);
+  }
+  root.containerOpen = false;
+}
+
+add_task(async function test_execute() {
+  // add visits, one for each transition type
+  for (let transition of transitions) {
+    await PlacesTestUtils.addVisits({
+      uri: uri("http://" + transition + ".mozilla.org/"),
+      transition,
+    });
+  }
+
+  runQuery(Ci.nsINavHistoryQueryOptions.RESULTS_AS_VISIT);
+  runQuery(Ci.nsINavHistoryQueryOptions.RESULTS_AS_URI);
+});
diff --git a/toolkit/components/places/tests/unit/test_485442_crash_bug_nsNavHistoryQuery_GetUri.js b/toolkit/components/places/tests/unit/test_485442_crash_bug_nsNavHistoryQuery_GetUri.js
new file mode 100644
index 0000000000..73bb5e9af4
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_485442_crash_bug_nsNavHistoryQuery_GetUri.js
@@ -0,0 +1,19 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function run_test() {
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = options.RESULT_TYPE_QUERY;
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(
+    PlacesUtils.asQuery(root).query.uri,
+    null,
+    "Should be null and not crash the browser"
+  );
+  root.containerOpen = false;
+}
diff --git a/toolkit/components/places/tests/unit/test_486978_sort_by_date_queries.js b/toolkit/components/places/tests/unit/test_486978_sort_by_date_queries.js
new file mode 100644
index 0000000000..221377e184
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_486978_sort_by_date_queries.js
@@ -0,0 +1,130 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * TEST DESCRIPTION:
+ *
+ * This test checks that setting a sort on a RESULTS_AS_DATE_QUERY query,
+ * children of inside containers are sorted accordingly.
+ */
+
+var hs = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+  Ci.nsINavHistoryService
+);
+
+// Will be inserted in this order, so last one will be the newest visit.
+var pages = [
+  "http://a.mozilla.org/1/",
+  "http://a.mozilla.org/2/",
+  "http://a.mozilla.org/3/",
+  "http://a.mozilla.org/4/",
+  "http://b.mozilla.org/5/",
+  "http://b.mozilla.org/6/",
+  "http://b.mozilla.org/7/",
+  "http://b.mozilla.org/8/",
+];
+
+add_task(async function test_initialize() {
+  // Add visits.
+  let now = new Date();
+  for (let pageIndex = 0; pageIndex < pages.length; ++pageIndex) {
+    let page = pages[pageIndex];
+    await PlacesTestUtils.addVisits({
+      uri: uri(page),
+      visitDate: new Date(now - (pages.length - pageIndex)),
+    });
+  }
+});
+
+/**
+ * Tests that sorting date query by none will sort by title asc.
+ */
+add_task(function () {
+  var options = hs.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_DATE_QUERY;
+  // This should sort by title asc.
+  options.sortingMode = options.SORT_BY_NONE;
+  var query = hs.getNewQuery();
+  var result = hs.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var dayContainer = root
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  dayContainer.containerOpen = true;
+
+  var cc = dayContainer.childCount;
+  Assert.equal(cc, pages.length);
+  for (var i = 0; i < cc; i++) {
+    var node = dayContainer.getChild(i);
+    Assert.equal(pages[i], node.uri);
+  }
+
+  dayContainer.containerOpen = false;
+  root.containerOpen = false;
+});
+
+/**
+ * Tests that sorting date query by date will sort accordingly.
+ */
+add_task(function () {
+  var options = hs.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_DATE_QUERY;
+  // This should sort by title asc.
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  var query = hs.getNewQuery();
+  var result = hs.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var dayContainer = root
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  dayContainer.containerOpen = true;
+
+  var cc = dayContainer.childCount;
+  Assert.equal(cc, pages.length);
+  for (var i = 0; i < cc; i++) {
+    var node = dayContainer.getChild(i);
+    Assert.equal(pages[pages.length - i - 1], node.uri);
+  }
+
+  dayContainer.containerOpen = false;
+  root.containerOpen = false;
+});
+
+/**
+ * Tests that sorting date site query by date will still sort by title asc.
+ */
+add_task(function () {
+  var options = hs.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_DATE_SITE_QUERY;
+  // This should sort by title asc.
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  var query = hs.getNewQuery();
+  var result = hs.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var dayContainer = root
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  dayContainer.containerOpen = true;
+  var siteContainer = dayContainer
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  Assert.equal(siteContainer.title, "a.mozilla.org");
+  siteContainer.containerOpen = true;
+
+  var cc = siteContainer.childCount;
+  Assert.equal(cc, pages.length / 2);
+  for (var i = 0; i < cc / 2; i++) {
+    var node = siteContainer.getChild(i);
+    Assert.equal(pages[i], node.uri);
+  }
+
+  siteContainer.containerOpen = false;
+  dayContainer.containerOpen = false;
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_536081.js b/toolkit/components/places/tests/unit/test_536081.js
new file mode 100644
index 0000000000..4105b09d1b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_536081.js
@@ -0,0 +1,35 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const TEST_URL = {
+  u: "http://www.google.com/search?q=testing%3Bthis&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-US:unofficial&client=firefox-a",
+  s: "goog",
+};
+
+add_task(async function () {
+  print("Testing url: " + TEST_URL.u);
+  await PlacesTestUtils.addVisits(uri(TEST_URL.u));
+
+  let query = PlacesUtils.history.getNewQuery();
+  query.searchTerms = TEST_URL.s;
+  let options = PlacesUtils.history.getNewQueryOptions();
+
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  let cc = root.childCount;
+  Assert.equal(cc, 1);
+
+  print("Checking url is in the query.");
+  let node = root.getChild(0);
+  print("Found " + node.uri);
+
+  Assert.ok(await PlacesTestUtils.isPageInDB(TEST_URL.u));
+
+  root.containerOpen = false;
+  await PlacesUtils.history.remove(node.uri);
+
+  Assert.equal(false, await PlacesTestUtils.isPageInDB(TEST_URL.u));
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesDBUtils_removeOldCorruptDBs.js b/toolkit/components/places/tests/unit/test_PlacesDBUtils_removeOldCorruptDBs.js
new file mode 100644
index 0000000000..26ec85ffc4
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesDBUtils_removeOldCorruptDBs.js
@@ -0,0 +1,42 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const TEMP_FILES_TO_CREATE = 5;
+const LAST_MODIFICATION_DAY = [5, 10, 15, 20, 25];
+const TEST_CURRENT_TIME = Date.now();
+const MS_PER_DAY = 86400000;
+const RETAIN_DAYS = 14;
+
+async function createfiles() {
+  for (let i = 0; i < TEMP_FILES_TO_CREATE; i++) {
+    let setTime = TEST_CURRENT_TIME;
+    setTime -= LAST_MODIFICATION_DAY[i] * MS_PER_DAY;
+    let fileName = "places.sqlite" + (i > 0 ? "-" + i : "") + ".corrupt";
+    let filePath = PathUtils.join(PathUtils.profileDir, fileName);
+    await IOUtils.writeUTF8(filePath, "test-file-delete-me", {
+      tmpPath: filePath + ".tmp",
+    });
+    Assert.ok(await IOUtils.exists(filePath), "file created: " + filePath);
+    await IOUtils.setModificationTime(filePath, setTime);
+  }
+}
+
+add_task(async function removefiles() {
+  await createfiles();
+  await PlacesDBUtils.runTasks([PlacesDBUtils.removeOldCorruptDBs]);
+  for (let i = 0; i < TEMP_FILES_TO_CREATE; i++) {
+    let fileName = "places.sqlite" + (i > 0 ? "-" + i : "") + ".corrupt";
+    let filePath = PathUtils.join(PathUtils.profileDir, fileName);
+    if (LAST_MODIFICATION_DAY[i] >= RETAIN_DAYS) {
+      Assert.ok(
+        !(await IOUtils.exists(filePath)),
+        "Old corrupt file has been removed" + filePath
+      );
+    } else {
+      Assert.ok(
+        await IOUtils.exists(filePath),
+        "Files that are not old are not removed" + filePath
+      );
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesQuery_history.js b/toolkit/components/places/tests/unit/test_PlacesQuery_history.js
new file mode 100644
index 0000000000..02648ead05
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesQuery_history.js
@@ -0,0 +1,100 @@
+/* Any copyright is dedicated to the Public Domain.
+http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { PlacesQuery } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesQuery.sys.mjs"
+);
+
+add_task(async function test_visits_cache_is_updated() {
+  const placesQuery = new PlacesQuery();
+  const now = new Date();
+  info("Insert the first visit.");
+  await PlacesUtils.history.insert({
+    url: "https://www.example.com/",
+    title: "Example Domain",
+    visits: [{ date: now }],
+  });
+  let history = await placesQuery.getHistory();
+  Assert.equal(history.length, 1);
+  Assert.equal(history[0].url, "https://www.example.com/");
+  Assert.equal(history[0].date.getTime(), now.getTime());
+  Assert.equal(history[0].title, "Example Domain");
+
+  info("Insert the next visit.");
+  let historyUpdated = PromiseUtils.defer();
+  placesQuery.observeHistory(newHistory => {
+    history = newHistory;
+    historyUpdated.resolve();
+  });
+  await PlacesUtils.history.insert({
+    url: "https://example.net/",
+    visits: [{ date: now }],
+  });
+  await historyUpdated.promise;
+  Assert.equal(history.length, 2);
+  Assert.equal(
+    history[0].url,
+    "https://example.net/",
+    "The most recent visit should come first."
+  );
+  Assert.equal(history[0].date.getTime(), now.getTime());
+
+  info("Remove the first visit.");
+  historyUpdated = PromiseUtils.defer();
+  await PlacesUtils.history.remove("https://www.example.com/");
+  await historyUpdated.promise;
+  Assert.equal(history.length, 1);
+  Assert.equal(history[0].url, "https://example.net/");
+
+  info("Remove all visits.");
+  historyUpdated = PromiseUtils.defer();
+  await PlacesUtils.history.clear();
+  await historyUpdated.promise;
+  Assert.equal(history.length, 0);
+  placesQuery.close();
+});
+
+add_task(async function test_filter_visits_by_age() {
+  const placesQuery = new PlacesQuery();
+  await PlacesUtils.history.insertMany([
+    {
+      url: "https://www.example.com/",
+      visits: [{ date: new Date("2000-01-01T12:00:00") }],
+    },
+    {
+      url: "https://example.net/",
+      visits: [{ date: new Date() }],
+    },
+  ]);
+  const history = await placesQuery.getHistory({ daysOld: 1 });
+  Assert.equal(history.length, 1, "The older visit should be excluded.");
+  Assert.equal(history[0].url, "https://example.net/");
+  await PlacesUtils.history.clear();
+  placesQuery.close();
+});
+
+add_task(async function test_filter_redirecting_visits() {
+  const placesQuery = new PlacesQuery();
+  await PlacesUtils.history.insertMany([
+    {
+      url: "http://google.com/",
+      visits: [{ transition: PlacesUtils.history.TRANSITIONS.TYPED }],
+    },
+    {
+      url: "https://www.google.com/",
+      visits: [
+        {
+          transition: PlacesUtils.history.TRANSITIONS.REDIRECT_PERMANENT,
+          referrer: "http://google.com/",
+        },
+      ],
+    },
+  ]);
+  const history = await placesQuery.getHistory();
+  Assert.equal(history.length, 1, "Redirecting visits should be excluded.");
+  Assert.equal(history[0].url, "https://www.google.com/");
+  await PlacesUtils.history.clear();
+  placesQuery.close();
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuidFor.js b/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuidFor.js
new file mode 100644
index 0000000000..8ec7eccfee
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuidFor.js
@@ -0,0 +1,32 @@
+add_task(async function () {
+  info("Add a bookmark.");
+  let bm = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  let id = await PlacesUtils.promiseItemId(bm.guid);
+  Assert.equal(await PlacesUtils.promiseItemGuid(id), bm.guid);
+
+  // Ensure invalidating a non-existent itemId doesn't throw.
+  PlacesUtils.invalidateCachedGuidFor(null);
+  PlacesUtils.invalidateCachedGuidFor(9999);
+
+  info("Change the GUID.");
+  await PlacesUtils.withConnectionWrapper("test", async function (db) {
+    await db.execute("UPDATE moz_bookmarks SET guid = :guid WHERE id = :id", {
+      guid: "123456789012",
+      id,
+    });
+  });
+  // The cache should still point to the wrong id.
+  Assert.equal(await PlacesUtils.promiseItemGuid(id), bm.guid);
+
+  info("Invalidate the cache.");
+  PlacesUtils.invalidateCachedGuidFor(id);
+  Assert.equal(await PlacesUtils.promiseItemGuid(id), "123456789012");
+  Assert.equal(await PlacesUtils.promiseItemId("123456789012"), id);
+  await Assert.rejects(
+    PlacesUtils.promiseItemId(bm.guid),
+    /no item found for the given GUID/
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuids.js b/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuids.js
new file mode 100644
index 0000000000..72304e8524
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesUtils_invalidateCachedGuids.js
@@ -0,0 +1,86 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+async function changeGuid(db, id, newGuid) {
+  await db.execute(
+    `UPDATE moz_bookmarks SET
+       guid = :newGuid
+     WHERE id = :id`,
+    { id, newGuid }
+  );
+}
+
+add_task(async function test_invalidateCachedGuids() {
+  info("Add bookmarks");
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        guid: "bookmarkAAAA",
+        url: "http://example.com/a",
+        title: "A",
+      },
+      {
+        guid: "bookmarkBBBB",
+        url: "http://example.com/b",
+        title: "B",
+      },
+    ],
+  });
+
+  let ids = await PlacesUtils.promiseManyItemIds([
+    "bookmarkAAAA",
+    "bookmarkBBBB",
+  ]);
+  Assert.equal(
+    await PlacesUtils.promiseItemGuid(ids.get("bookmarkAAAA")),
+    "bookmarkAAAA"
+  );
+  Assert.equal(
+    await PlacesUtils.promiseItemGuid(ids.get("bookmarkBBBB")),
+    "bookmarkBBBB"
+  );
+
+  info("Change GUIDs");
+  await PlacesUtils.withConnectionWrapper(
+    "test_invalidateCachedGuids",
+    async function (db) {
+      await db.executeTransaction(async function () {
+        await changeGuid(db, ids.get("bookmarkAAAA"), "bookmarkCCCC");
+        await changeGuid(db, ids.get("bookmarkBBBB"), "bookmarkDDDD");
+      });
+    }
+  );
+  Assert.equal(
+    await PlacesUtils.promiseItemId("bookmarkAAAA"),
+    ids.get("bookmarkAAAA")
+  );
+  Assert.equal(
+    await PlacesUtils.promiseItemId("bookmarkBBBB"),
+    ids.get("bookmarkBBBB")
+  );
+
+  info("Invalidate the cache");
+  PlacesUtils.invalidateCachedGuids();
+
+  let newIds = await PlacesUtils.promiseManyItemIds([
+    "bookmarkCCCC",
+    "bookmarkDDDD",
+  ]);
+  Assert.equal(
+    await PlacesUtils.promiseItemGuid(newIds.get("bookmarkCCCC")),
+    "bookmarkCCCC"
+  );
+  Assert.equal(
+    await PlacesUtils.promiseItemGuid(newIds.get("bookmarkDDDD")),
+    "bookmarkDDDD"
+  );
+  await Assert.rejects(
+    PlacesUtils.promiseItemId("bookmarkAAAA"),
+    /no item found for the given GUID/
+  );
+  await Assert.rejects(
+    PlacesUtils.promiseItemId("bookmarkBBBB"),
+    /no item found for the given GUID/
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesUtils_isRootItem.js b/toolkit/components/places/tests/unit/test_PlacesUtils_isRootItem.js
new file mode 100644
index 0000000000..f626d437e0
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesUtils_isRootItem.js
@@ -0,0 +1,21 @@
+"use strict";
+
+const GUIDS = [
+  PlacesUtils.bookmarks.rootGuid,
+  ...PlacesUtils.bookmarks.userContentRoots,
+  PlacesUtils.bookmarks.tagsGuid,
+];
+
+add_task(async function test_isRootItem() {
+  for (let guid of GUIDS) {
+    Assert.ok(
+      PlacesUtils.isRootItem(guid),
+      `Should correctly identify root item ${guid}`
+    );
+  }
+
+  Assert.ok(
+    !PlacesUtils.isRootItem("fakeguid1234"),
+    "Should not identify other items as root."
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_PlacesUtils_unwrapNodes_place.js b/toolkit/components/places/tests/unit/test_PlacesUtils_unwrapNodes_place.js
new file mode 100644
index 0000000000..8944dc22f4
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_PlacesUtils_unwrapNodes_place.js
@@ -0,0 +1,34 @@
+/* Any copyright is dedicated to the Public Domain.
+ * https://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that unwrapNodes properly filters out place: uris from text flavors.
+
+add_task(function () {
+  let tests = [
+    // Single url.
+    ["place:type=0&sort=1:", PlacesUtils.TYPE_X_MOZ_URL],
+    // Multiple urls.
+    [
+      "place:type=0&sort=1:\nfirst\nplace:type=0&sort=1\nsecond",
+      PlacesUtils.TYPE_X_MOZ_URL,
+    ],
+    // Url == title.
+    ["place:type=0&sort=1:\nplace:type=0&sort=1", PlacesUtils.TYPE_X_MOZ_URL],
+    // Malformed.
+    [
+      "place:type=0&sort=1:\nplace:type=0&sort=1\nmalformed",
+      PlacesUtils.TYPE_X_MOZ_URL,
+    ],
+    // Single url.
+    ["place:type=0&sort=1:", PlacesUtils.TYPE_PLAINTEXT],
+    // Multiple urls.
+    ["place:type=0&sort=1:\nplace:type=0&sort=1", PlacesUtils.TYPE_PLAINTEXT],
+  ];
+  for (let [blob, type] of tests) {
+    Assert.deepEqual(
+      PlacesUtils.unwrapNodes(blob, type),
+      [],
+      "No valid entries should be found"
+    );
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_asyncExecuteLegacyQueries.js b/toolkit/components/places/tests/unit/test_asyncExecuteLegacyQueries.js
new file mode 100644
index 0000000000..084415fb37
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_asyncExecuteLegacyQueries.js
@@ -0,0 +1,84 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+// This is a test for asyncExecuteLegacyQuery API.
+
+add_task(async function test_history_query() {
+  let uri = "http://test.visit.mozilla.com/";
+  let title = "Test visit";
+  await PlacesTestUtils.addVisits({ uri, title });
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_DATE_DESCENDING;
+  let query = PlacesUtils.history.getNewQuery();
+
+  return new Promise(resolve => {
+    PlacesUtils.history.asyncExecuteLegacyQuery(query, options, {
+      handleResult(aResultSet) {
+        for (let row; (row = aResultSet.getNextRow()); ) {
+          try {
+            Assert.equal(row.getResultByIndex(1), uri);
+            Assert.equal(row.getResultByIndex(2), title);
+          } catch (e) {
+            do_throw("Error while fetching page data.");
+          }
+        }
+      },
+      handleError(aError) {
+        do_throw(
+          "Async execution error (" + aError.result + "): " + aError.message
+        );
+      },
+      handleCompletion(aReason) {
+        cleanupTest().then(resolve);
+      },
+    });
+  });
+});
+
+add_task(async function test_bookmarks_query() {
+  let url = "http://test.bookmark.mozilla.com/";
+  let title = "Test bookmark";
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title,
+    url,
+  });
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode =
+    Ci.nsINavHistoryQueryOptions.SORT_BY_LASTMODIFIED_DESCENDING;
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let query = PlacesUtils.history.getNewQuery();
+
+  return new Promise(resolve => {
+    PlacesUtils.history.asyncExecuteLegacyQuery(query, options, {
+      handleResult(aResultSet) {
+        for (let row; (row = aResultSet.getNextRow()); ) {
+          try {
+            Assert.equal(row.getResultByIndex(1), url);
+            Assert.equal(row.getResultByIndex(2), title);
+          } catch (e) {
+            do_throw("Error while fetching page data.");
+          }
+        }
+      },
+      handleError(aError) {
+        do_throw(
+          "Async execution error (" + aError.result + "): " + aError.message
+        );
+      },
+      handleCompletion(aReason) {
+        cleanupTest().then(resolve);
+      },
+    });
+  });
+});
+
+function cleanupTest() {
+  return Promise.all([
+    PlacesUtils.history.clear(),
+    PlacesUtils.bookmarks.eraseEverything(),
+  ]);
+}
diff --git a/toolkit/components/places/tests/unit/test_async_transactions.js b/toolkit/components/places/tests/unit/test_async_transactions.js
new file mode 100644
index 0000000000..71701ec540
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_async_transactions.js
@@ -0,0 +1,2214 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const bmsvc = PlacesUtils.bookmarks;
+const obsvc = PlacesUtils.observers;
+const tagssvc = PlacesUtils.tagging;
+const PT = PlacesTransactions;
+const menuGuid = PlacesUtils.bookmarks.menuGuid;
+
+ChromeUtils.defineESModuleGetters(this, {
+  Preferences: "resource://gre/modules/Preferences.sys.mjs",
+});
+
+// Create and add bookmarks observer.
+var observer = {
+  tagRelatedGuids: new Set(),
+
+  reset() {
+    this.itemsAdded = new Map();
+    this.itemsRemoved = new Map();
+    this.itemsMoved = new Map();
+    this.itemsKeywordChanged = new Map();
+    this.itemsTitleChanged = new Map();
+    this.itemsUrlChanged = new Map();
+  },
+
+  handlePlacesEvents(events) {
+    for (let event of events) {
+      switch (event.type) {
+        case "bookmark-added":
+          // Ignore tag items.
+          if (event.isTagging) {
+            this.tagRelatedGuids.add(event.guid);
+            return;
+          }
+
+          this.itemsAdded.set(event.guid, {
+            itemId: event.id,
+            parentGuid: event.parentGuid,
+            index: event.index,
+            itemType: event.itemType,
+            title: event.title,
+            url: event.url,
+          });
+          break;
+        case "bookmark-removed":
+          if (this.tagRelatedGuids.has(event.guid)) {
+            return;
+          }
+
+          this.itemsRemoved.set(event.guid, {
+            parentGuid: event.parentGuid,
+            index: event.index,
+            itemType: event.itemType,
+          });
+          break;
+        case "bookmark-moved":
+          this.itemsMoved.set(event.guid, {
+            oldParentGuid: event.oldParentGuid,
+            oldIndex: event.oldIndex,
+            newParentGuid: event.parentGuid,
+            newIndex: event.index,
+            itemType: event.itemType,
+          });
+          break;
+        case "bookmark-keyword-changed":
+          if (this.tagRelatedGuids.has(event.guid)) {
+            return;
+          }
+
+          this.itemsKeywordChanged.set(event.guid, {
+            keyword: event.keyword,
+          });
+          break;
+        case "bookmark-title-changed":
+          if (this.tagRelatedGuids.has(event.guid)) {
+            return;
+          }
+
+          this.itemsTitleChanged.set(event.guid, {
+            title: event.title,
+            parentGuid: event.parentGuid,
+          });
+          break;
+        case "bookmark-url-changed":
+          if (this.tagRelatedGuids.has(event.guid)) {
+            return;
+          }
+
+          this.itemsUrlChanged.set(event.guid, {
+            url: event.url,
+          });
+          break;
+      }
+    }
+  },
+};
+observer.reset();
+
+// index at which items should begin
+var bmStartIndex = 0;
+
+function run_test() {
+  observer.handlePlacesEvents = observer.handlePlacesEvents.bind(observer);
+  obsvc.addListener(
+    [
+      "bookmark-added",
+      "bookmark-removed",
+      "bookmark-moved",
+      "bookmark-keyword-changed",
+      "bookmark-title-changed",
+      "bookmark-url-changed",
+    ],
+    observer.handlePlacesEvents
+  );
+  registerCleanupFunction(function () {
+    obsvc.removeListener(
+      [
+        "bookmark-added",
+        "bookmark-removed",
+        "bookmark-moved",
+        "bookmark-keyword-changed",
+        "bookmark-title-changed",
+        "bookmark-url-changed",
+      ],
+      observer.handlePlacesEvents
+    );
+  });
+
+  run_next_test();
+}
+
+function sanityCheckTransactionHistory() {
+  Assert.ok(PT.undoPosition <= PT.length);
+
+  let check_entry_throws = f => {
+    try {
+      f();
+      do_throw("PT.entry should throw for invalid input");
+    } catch (ex) {}
+  };
+  check_entry_throws(() => PT.entry(-1));
+  check_entry_throws(() => PT.entry({}));
+  check_entry_throws(() => PT.entry(PT.length));
+
+  if (PT.undoPosition < PT.length) {
+    Assert.equal(PT.topUndoEntry, PT.entry(PT.undoPosition));
+  } else {
+    Assert.equal(null, PT.topUndoEntry);
+  }
+  if (PT.undoPosition > 0) {
+    Assert.equal(PT.topRedoEntry, PT.entry(PT.undoPosition - 1));
+  } else {
+    Assert.equal(null, PT.topRedoEntry);
+  }
+}
+
+function getTransactionsHistoryState() {
+  let history = [];
+  for (let i = 0; i < PT.length; i++) {
+    history.push(PT.entry(i));
+  }
+  return [history, PT.undoPosition];
+}
+
+function ensureUndoState(aExpectedEntries = [], aExpectedUndoPosition = 0) {
+  // ensureUndoState is called in various places during this test, so it's
+  // a good places to sanity-check the transaction-history APIs in all
+  // cases.
+  sanityCheckTransactionHistory();
+
+  let [actualEntries, actualUndoPosition] = getTransactionsHistoryState();
+  Assert.equal(actualEntries.length, aExpectedEntries.length);
+  Assert.equal(actualUndoPosition, aExpectedUndoPosition);
+
+  function checkEqualEntries(aExpectedEntry, aActualEntry) {
+    Assert.equal(aExpectedEntry.length, aActualEntry.length);
+    aExpectedEntry.forEach((t, i) => Assert.equal(t, aActualEntry[i]));
+  }
+  aExpectedEntries.forEach((e, i) => checkEqualEntries(e, actualEntries[i]));
+}
+
+function ensureItemsAdded(...items) {
+  let expectedResultsCount = items.length;
+
+  for (let item of items) {
+    if ("children" in item) {
+      expectedResultsCount += item.children.length;
+    }
+    Assert.ok(
+      observer.itemsAdded.has(item.guid),
+      `Should have the expected guid ${item.guid}`
+    );
+    let info = observer.itemsAdded.get(item.guid);
+    Assert.equal(
+      info.parentGuid,
+      item.parentGuid,
+      "Should have notified the correct parentGuid"
+    );
+    for (let propName of ["title", "index", "itemType"]) {
+      if (propName in item) {
+        Assert.equal(info[propName], item[propName]);
+      }
+    }
+    if ("url" in item) {
+      Assert.ok(
+        Services.io.newURI(info.url).equals(Services.io.newURI(item.url)),
+        "Should have the correct url"
+      );
+    }
+  }
+
+  Assert.equal(
+    observer.itemsAdded.size,
+    expectedResultsCount,
+    "Should have added the correct number of items"
+  );
+}
+
+function ensureItemsRemoved(...items) {
+  let expectedResultsCount = items.length;
+
+  for (let item of items) {
+    // We accept both guids and full info object here.
+    if (typeof item == "string") {
+      Assert.ok(
+        observer.itemsRemoved.has(item),
+        `Should have removed the expected guid ${item}`
+      );
+    } else {
+      if ("children" in item) {
+        expectedResultsCount += item.children.length;
+      }
+
+      Assert.ok(
+        observer.itemsRemoved.has(item.guid),
+        `Should have removed expected guid ${item.guid}`
+      );
+      let info = observer.itemsRemoved.get(item.guid);
+      Assert.equal(
+        info.parentGuid,
+        item.parentGuid,
+        "Should have notified the correct parentGuid"
+      );
+      if ("index" in item) {
+        Assert.equal(info.index, item.index);
+      }
+    }
+  }
+
+  Assert.equal(
+    observer.itemsRemoved.size,
+    expectedResultsCount,
+    "Should have removed the correct number of items"
+  );
+}
+
+function ensureItemsMoved(...items) {
+  Assert.equal(
+    observer.itemsMoved.size,
+    items.length,
+    "Should have received the correct number of moved notifications"
+  );
+  for (let item of items) {
+    Assert.ok(
+      observer.itemsMoved.has(item.guid),
+      `Observer should have a move for ${item.guid}`
+    );
+    let info = observer.itemsMoved.get(item.guid);
+    Assert.equal(
+      info.oldParentGuid,
+      item.oldParentGuid,
+      "Should have the correct old parent guid"
+    );
+    Assert.equal(
+      info.oldIndex,
+      item.oldIndex,
+      "Should have the correct old index"
+    );
+    Assert.equal(
+      info.newParentGuid,
+      item.newParentGuid,
+      "Should have the correct new parent guid"
+    );
+    Assert.equal(
+      info.newIndex,
+      item.newIndex,
+      "Should have the correct new index"
+    );
+  }
+}
+
+function ensureItemsKeywordChanged(...items) {
+  for (const item of items) {
+    Assert.ok(
+      observer.itemsKeywordChanged.has(item.guid),
+      `Observer should have a keyword changed for ${item.guid}`
+    );
+    const info = observer.itemsKeywordChanged.get(item.guid);
+    Assert.equal(info.keyword, item.keyword, "Should have the correct keyword");
+  }
+}
+
+function ensureItemsTitleChanged(...items) {
+  Assert.equal(
+    observer.itemsTitleChanged.size,
+    items.length,
+    "Should have received the correct number of bookmark-title-changed notifications"
+  );
+  for (const item of items) {
+    Assert.ok(
+      observer.itemsTitleChanged.has(item.guid),
+      `Observer should have a title changed for ${item.guid}`
+    );
+    const info = observer.itemsTitleChanged.get(item.guid);
+    Assert.equal(info.title, item.title, "Should have the correct title");
+    Assert.equal(
+      info.parentGuid,
+      item.parentGuid,
+      "Should have the correct parent guid"
+    );
+  }
+}
+
+function ensureItemsUrlChanged(...items) {
+  Assert.equal(
+    observer.itemsUrlChanged.size,
+    items.length,
+    "Should have received the correct number of bookmark-url-changed notifications"
+  );
+  for (const item of items) {
+    Assert.ok(
+      observer.itemsUrlChanged.has(item.guid),
+      `Observer should have a title changed for ${item.guid}`
+    );
+    const info = observer.itemsUrlChanged.get(item.guid);
+    Assert.equal(info.url, item.url, "Should have the correct url");
+  }
+}
+
+function ensureTagsForURI(aURI, aTags) {
+  let tagsSet = tagssvc.getTagsForURI(Services.io.newURI(aURI));
+  Assert.equal(tagsSet.length, aTags.length);
+  Assert.ok(aTags.every(t => tagsSet.includes(t)));
+}
+
+function createTestFolderInfo(
+  title = "Test Folder",
+  parentGuid = menuGuid,
+  children = undefined
+) {
+  let info = { parentGuid, title };
+  if (children) {
+    info.children = children;
+  }
+  return info;
+}
+
+function removeAllDatesInTree(tree) {
+  if ("lastModified" in tree) {
+    delete tree.lastModified;
+  }
+  if ("dateAdded" in tree) {
+    delete tree.dateAdded;
+  }
+
+  if (!tree.children) {
+    return;
+  }
+
+  for (let child of tree.children) {
+    removeAllDatesInTree(child);
+  }
+}
+
+// Checks if two bookmark trees (as returned by promiseBookmarksTree) are the
+// same.
+// false value for aCheckParentAndPosition is ignored if aIsRestoredItem is set.
+async function ensureEqualBookmarksTrees(
+  aOriginal,
+  aNew,
+  aIsRestoredItem = true,
+  aCheckParentAndPosition = false,
+  aIgnoreAllDates = false
+) {
+  // Note "id" is not-enumerable, and is therefore skipped by Object.keys (both
+  // ours and the one at deepEqual). This is fine for us because ids are not
+  // restored by Redo.
+  if (aIsRestoredItem) {
+    if (aIgnoreAllDates) {
+      removeAllDatesInTree(aOriginal);
+      removeAllDatesInTree(aNew);
+    } else if (!aOriginal.lastModified) {
+      // Ignore lastModified for newly created items, for performance reasons.
+      aNew.lastModified = aOriginal.lastModified;
+    }
+    Assert.deepEqual(aOriginal, aNew);
+    return;
+  }
+
+  for (let property of Object.keys(aOriginal)) {
+    if (property == "children") {
+      Assert.equal(aOriginal.children.length, aNew.children.length);
+      for (let i = 0; i < aOriginal.children.length; i++) {
+        await ensureEqualBookmarksTrees(
+          aOriginal.children[i],
+          aNew.children[i],
+          false,
+          true,
+          aIgnoreAllDates
+        );
+      }
+    } else if (property == "guid") {
+      // guid shouldn't be copied if the item was not restored.
+      Assert.notEqual(aOriginal.guid, aNew.guid);
+    } else if (property == "dateAdded") {
+      // dateAdded shouldn't be copied if the item was not restored.
+      Assert.ok(is_time_ordered(aOriginal.dateAdded, aNew.dateAdded));
+    } else if (property == "lastModified") {
+      // same same, except for the never-changed case
+      if (!aOriginal.lastModified) {
+        Assert.ok(!aNew.lastModified);
+      } else {
+        Assert.ok(is_time_ordered(aOriginal.lastModified, aNew.lastModified));
+      }
+    } else if (
+      aCheckParentAndPosition ||
+      (property != "parentGuid" && property != "index")
+    ) {
+      Assert.deepEqual(aOriginal[property], aNew[property]);
+    }
+  }
+}
+
+async function ensureBookmarksTreeRestoredCorrectly(
+  ...aOriginalBookmarksTrees
+) {
+  for (let originalTree of aOriginalBookmarksTrees) {
+    let restoredTree = await PlacesUtils.promiseBookmarksTree(
+      originalTree.guid
+    );
+    await ensureEqualBookmarksTrees(originalTree, restoredTree);
+  }
+}
+
+async function ensureBookmarksTreeRestoredCorrectlyExceptDates(
+  ...aOriginalBookmarksTrees
+) {
+  for (let originalTree of aOriginalBookmarksTrees) {
+    let restoredTree = await PlacesUtils.promiseBookmarksTree(
+      originalTree.guid
+    );
+    await ensureEqualBookmarksTrees(
+      originalTree,
+      restoredTree,
+      true,
+      false,
+      true
+    );
+  }
+}
+
+async function ensureNonExistent(...aGuids) {
+  for (let guid of aGuids) {
+    Assert.strictEqual(await PlacesUtils.promiseBookmarksTree(guid), null);
+  }
+}
+
+add_task(async function test_recycled_transactions() {
+  async function ensureTransactThrowsFor(aTransaction) {
+    let [txns, undoPosition] = getTransactionsHistoryState();
+    try {
+      await aTransaction.transact();
+      do_throw("Shouldn't be able to use the same transaction twice");
+    } catch (ex) {}
+    ensureUndoState(txns, undoPosition);
+  }
+
+  let txn_a = PT.NewFolder(createTestFolderInfo());
+  await txn_a.transact();
+  ensureUndoState([[txn_a]], 0);
+  await ensureTransactThrowsFor(txn_a);
+
+  await PT.undo();
+  ensureUndoState([[txn_a]], 1);
+  ensureTransactThrowsFor(txn_a);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+  ensureTransactThrowsFor(txn_a);
+
+  let txn_b = PT.NewFolder(createTestFolderInfo());
+  await PT.batch(async function () {
+    try {
+      await txn_a.transact();
+      do_throw("Shouldn't be able to use the same transaction twice");
+    } catch (ex) {}
+    ensureUndoState();
+    await txn_b.transact();
+  });
+  ensureUndoState([[txn_b]], 0);
+
+  await PT.undo();
+  ensureUndoState([[txn_b]], 1);
+  ensureTransactThrowsFor(txn_a);
+  ensureTransactThrowsFor(txn_b);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+  observer.reset();
+});
+
+add_task(async function test_new_folder_with_children() {
+  let folder_info = createTestFolderInfo(
+    "Test folder",
+    PlacesUtils.bookmarks.menuGuid,
+    [
+      {
+        url: "http://test_create_item.com",
+        title: "Test creating an item",
+      },
+    ]
+  );
+  ensureUndoState();
+  let txn = PT.NewFolder(folder_info);
+  folder_info.guid = await txn.transact();
+  let originalInfo = await PlacesUtils.promiseBookmarksTree(folder_info.guid);
+  let ensureDo = async function (aRedo = false) {
+    ensureUndoState([[txn]], 0);
+    ensureItemsAdded(folder_info);
+    if (aRedo) {
+      // Ignore lastModified in the comparison, for performance reasons.
+      originalInfo.lastModified = null;
+      await ensureBookmarksTreeRestoredCorrectlyExceptDates(originalInfo);
+    }
+    observer.reset();
+  };
+
+  let ensureUndo = () => {
+    ensureUndoState([[txn]], 1);
+    ensureItemsRemoved({
+      guid: folder_info.guid,
+      parentGuid: folder_info.parentGuid,
+      index: bmStartIndex,
+      children: [
+        {
+          title: "Test creating an item",
+          url: "http://test_create_item.com",
+        },
+      ],
+    });
+    observer.reset();
+  };
+
+  await ensureDo();
+  await PT.undo();
+  await ensureUndo();
+  await PT.redo();
+  await ensureDo(true);
+  await PT.undo();
+  ensureUndo();
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_new_bookmark() {
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test_create_item.com",
+    index: bmStartIndex,
+    title: "Test creating an item",
+  };
+
+  ensureUndoState();
+  let txn = PT.NewBookmark(bm_info);
+  bm_info.guid = await txn.transact();
+
+  let originalInfo = await PlacesUtils.promiseBookmarksTree(bm_info.guid);
+  let ensureDo = async function (aRedo = false) {
+    ensureUndoState([[txn]], 0);
+    await ensureItemsAdded(bm_info);
+    if (aRedo) {
+      await ensureBookmarksTreeRestoredCorrectly(originalInfo);
+    }
+    observer.reset();
+  };
+  let ensureUndo = () => {
+    ensureUndoState([[txn]], 1);
+    ensureItemsRemoved({
+      guid: bm_info.guid,
+      parentGuid: bm_info.parentGuid,
+      index: bmStartIndex,
+    });
+    observer.reset();
+  };
+
+  await ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo(true);
+  await ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_merge_create_folder_and_item() {
+  let folder_info = createTestFolderInfo();
+  let bm_info = {
+    url: "http://test_create_item_to_folder.com",
+    title: "Test Bookmark",
+    index: bmStartIndex,
+  };
+
+  let [folderTxnResult, bkmTxnResult] = await PT.batch(async function () {
+    let folderTxn = PT.NewFolder(folder_info);
+    folder_info.guid = bm_info.parentGuid = await folderTxn.transact();
+    let bkmTxn = PT.NewBookmark(bm_info);
+    bm_info.guid = await bkmTxn.transact();
+    return [folderTxn, bkmTxn];
+  });
+
+  let ensureDo = async function () {
+    ensureUndoState([[bkmTxnResult, folderTxnResult]], 0);
+    await ensureItemsAdded(folder_info, bm_info);
+    observer.reset();
+  };
+
+  let ensureUndo = () => {
+    ensureUndoState([[bkmTxnResult, folderTxnResult]], 1);
+    ensureItemsRemoved(folder_info, bm_info);
+    observer.reset();
+  };
+
+  await ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo();
+  await ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_move_items_to_folder() {
+  let folder_a_info = createTestFolderInfo("Folder A");
+  let bkm_a_info = { url: "http://test_move_items.com", title: "Bookmark A" };
+  let bkm_b_info = { url: "http://test_move_items.com", title: "Bookmark B" };
+
+  // Test moving items within the same folder.
+  let [folder_a_txn_result, bkm_a_txn_result, bkm_b_txn_result] =
+    await PT.batch(async function () {
+      let folder_a_txn = PT.NewFolder(folder_a_info);
+
+      folder_a_info.guid =
+        bkm_a_info.parentGuid =
+        bkm_b_info.parentGuid =
+          await folder_a_txn.transact();
+      let bkm_a_txn = PT.NewBookmark(bkm_a_info);
+      bkm_a_info.guid = await bkm_a_txn.transact();
+      let bkm_b_txn = PT.NewBookmark(bkm_b_info);
+      bkm_b_info.guid = await bkm_b_txn.transact();
+      return [folder_a_txn, bkm_a_txn, bkm_b_txn];
+    });
+
+  ensureUndoState(
+    [[bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result]],
+    0
+  );
+
+  let moveTxn = PT.Move({
+    guid: bkm_a_info.guid,
+    newParentGuid: folder_a_info.guid,
+  });
+  await moveTxn.transact();
+
+  let ensureDo = () => {
+    ensureUndoState(
+      [[moveTxn], [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result]],
+      0
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_a_info.guid,
+      newParentGuid: folder_a_info.guid,
+      oldIndex: 0,
+      newIndex: 1,
+    });
+    observer.reset();
+  };
+  let ensureUndo = () => {
+    ensureUndoState(
+      [[moveTxn], [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result]],
+      1
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_a_info.guid,
+      newParentGuid: folder_a_info.guid,
+      oldIndex: 1,
+      newIndex: 0,
+    });
+    observer.reset();
+  };
+
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo();
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  await PT.clearTransactionsHistory(false, true);
+  ensureUndoState(
+    [[bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result]],
+    0
+  );
+
+  // Test moving items between folders.
+  let folder_b_info = createTestFolderInfo("Folder B");
+  let folder_b_txn = PT.NewFolder(folder_b_info);
+  folder_b_info.guid = await folder_b_txn.transact();
+  ensureUndoState(
+    [[folder_b_txn], [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result]],
+    0
+  );
+
+  moveTxn = PT.Move({
+    guid: bkm_a_info.guid,
+    newParentGuid: folder_b_info.guid,
+    newIndex: bmsvc.DEFAULT_INDEX,
+  });
+  await moveTxn.transact();
+
+  ensureDo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [folder_b_txn],
+        [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result],
+      ],
+      0
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_a_info.guid,
+      newParentGuid: folder_b_info.guid,
+      oldIndex: 0,
+      newIndex: 0,
+    });
+    observer.reset();
+  };
+  ensureUndo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [folder_b_txn],
+        [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result],
+      ],
+      1
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_b_info.guid,
+      newParentGuid: folder_a_info.guid,
+      oldIndex: 0,
+      newIndex: 0,
+    });
+    observer.reset();
+  };
+
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo();
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  // Clean up
+  await PT.undo(); // folder_b_txn
+  await PT.undo(); // folder_a_txn + the bookmarks;
+  Assert.equal(observer.itemsRemoved.size, 4);
+  ensureUndoState(
+    [
+      [moveTxn],
+      [folder_b_txn],
+      [bkm_b_txn_result, bkm_a_txn_result, folder_a_txn_result],
+    ],
+    3
+  );
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_move_multiple_items_to_folder() {
+  let folder_a_info = createTestFolderInfo("Folder A");
+  let bkm_a_info = { url: "http://test_move_items.com", title: "Bookmark A" };
+  let bkm_b_info = { url: "http://test_move_items.com", title: "Bookmark B" };
+  let bkm_c_info = { url: "http://test_move_items.com", title: "Bookmark C" };
+
+  // Test moving items within the same folder.
+  let [
+    folder_a_txn_result,
+    bkm_a_txn_result,
+    bkm_b_txn_result,
+    bkm_c_txn_result,
+  ] = await PT.batch(async function () {
+    let folder_a_txn = PT.NewFolder(folder_a_info);
+
+    folder_a_info.guid =
+      bkm_a_info.parentGuid =
+      bkm_b_info.parentGuid =
+      bkm_c_info.parentGuid =
+        await folder_a_txn.transact();
+    let bkm_a_txn = PT.NewBookmark(bkm_a_info);
+    bkm_a_info.guid = await bkm_a_txn.transact();
+    let bkm_b_txn = PT.NewBookmark(bkm_b_info);
+    bkm_b_info.guid = await bkm_b_txn.transact();
+    let bkm_c_txn = PT.NewBookmark(bkm_c_info);
+    bkm_c_info.guid = await bkm_c_txn.transact();
+    return [folder_a_txn, bkm_a_txn, bkm_b_txn, bkm_c_txn];
+  });
+
+  ensureUndoState(
+    [
+      [
+        bkm_c_txn_result,
+        bkm_b_txn_result,
+        bkm_a_txn_result,
+        folder_a_txn_result,
+      ],
+    ],
+    0
+  );
+
+  let moveTxn = PT.Move({
+    guids: [bkm_a_info.guid, bkm_b_info.guid],
+    newParentGuid: folder_a_info.guid,
+  });
+  await moveTxn.transact();
+
+  let ensureDo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [
+          bkm_c_txn_result,
+          bkm_b_txn_result,
+          bkm_a_txn_result,
+          folder_a_txn_result,
+        ],
+      ],
+      0
+    );
+    ensureItemsMoved(
+      {
+        guid: bkm_a_info.guid,
+        oldParentGuid: folder_a_info.guid,
+        newParentGuid: folder_a_info.guid,
+        oldIndex: 0,
+        newIndex: 2,
+      },
+      {
+        guid: bkm_b_info.guid,
+        oldParentGuid: folder_a_info.guid,
+        newParentGuid: folder_a_info.guid,
+        oldIndex: 1,
+        newIndex: 2,
+      }
+    );
+    observer.reset();
+  };
+  let ensureUndo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [
+          bkm_c_txn_result,
+          bkm_b_txn_result,
+          bkm_a_txn_result,
+          folder_a_txn_result,
+        ],
+      ],
+      1
+    );
+    ensureItemsMoved(
+      {
+        guid: bkm_a_info.guid,
+        oldParentGuid: folder_a_info.guid,
+        newParentGuid: folder_a_info.guid,
+        oldIndex: 1,
+        newIndex: 0,
+      },
+      {
+        guid: bkm_b_info.guid,
+        oldParentGuid: folder_a_info.guid,
+        newParentGuid: folder_a_info.guid,
+        oldIndex: 2,
+        newIndex: 1,
+      }
+    );
+    observer.reset();
+  };
+
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo();
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  await PT.clearTransactionsHistory(false, true);
+  ensureUndoState(
+    [
+      [
+        bkm_c_txn_result,
+        bkm_b_txn_result,
+        bkm_a_txn_result,
+        folder_a_txn_result,
+      ],
+    ],
+    0
+  );
+
+  // Test moving items between folders.
+  let folder_b_info = createTestFolderInfo("Folder B");
+  let folder_b_txn = PT.NewFolder(folder_b_info);
+  folder_b_info.guid = await folder_b_txn.transact();
+  ensureUndoState(
+    [
+      [folder_b_txn],
+      [
+        bkm_c_txn_result,
+        bkm_b_txn_result,
+        bkm_a_txn_result,
+        folder_a_txn_result,
+      ],
+    ],
+    0
+  );
+
+  moveTxn = PT.Move({
+    guid: bkm_a_info.guid,
+    newParentGuid: folder_b_info.guid,
+    newIndex: bmsvc.DEFAULT_INDEX,
+  });
+  await moveTxn.transact();
+
+  ensureDo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [folder_b_txn],
+        [
+          bkm_c_txn_result,
+          bkm_b_txn_result,
+          bkm_a_txn_result,
+          folder_a_txn_result,
+        ],
+      ],
+      0
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_a_info.guid,
+      newParentGuid: folder_b_info.guid,
+      oldIndex: 0,
+      newIndex: 0,
+    });
+    observer.reset();
+  };
+  ensureUndo = () => {
+    ensureUndoState(
+      [
+        [moveTxn],
+        [folder_b_txn],
+        [
+          bkm_c_txn_result,
+          bkm_b_txn_result,
+          bkm_a_txn_result,
+          folder_a_txn_result,
+        ],
+      ],
+      1
+    );
+    ensureItemsMoved({
+      guid: bkm_a_info.guid,
+      oldParentGuid: folder_b_info.guid,
+      newParentGuid: folder_a_info.guid,
+      oldIndex: 0,
+      newIndex: 0,
+    });
+    observer.reset();
+  };
+
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+  await PT.redo();
+  ensureDo();
+  await PT.undo();
+  ensureUndo();
+
+  // Clean up
+  await PT.undo(); // folder_b_txn
+  await PT.undo(); // folder_a_txn + the bookmarks;
+  Assert.equal(observer.itemsRemoved.size, 5);
+  ensureUndoState(
+    [
+      [moveTxn],
+      [folder_b_txn],
+      [
+        bkm_c_txn_result,
+        bkm_b_txn_result,
+        bkm_a_txn_result,
+        folder_a_txn_result,
+      ],
+    ],
+    3
+  );
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_remove_folder() {
+  let folder_level_1_info = createTestFolderInfo("Folder Level 1");
+  let folder_level_2_info = { title: "Folder Level 2" };
+  let [folder_level_1_txn_result, folder_level_2_txn_result] = await PT.batch(
+    async function () {
+      let folder_level_1_txn = PT.NewFolder(folder_level_1_info);
+      folder_level_1_info.guid = await folder_level_1_txn.transact();
+      folder_level_2_info.parentGuid = folder_level_1_info.guid;
+      let folder_level_2_txn = PT.NewFolder(folder_level_2_info);
+      folder_level_2_info.guid = await folder_level_2_txn.transact();
+      return [folder_level_1_txn, folder_level_2_txn];
+    }
+  );
+
+  let original_folder_level_1_tree = await PlacesUtils.promiseBookmarksTree(
+    folder_level_1_info.guid
+  );
+  let original_folder_level_2_tree = Object.assign(
+    { parentGuid: original_folder_level_1_tree.guid },
+    original_folder_level_1_tree.children[0]
+  );
+
+  ensureUndoState([[folder_level_2_txn_result, folder_level_1_txn_result]]);
+  await ensureItemsAdded(folder_level_1_info, folder_level_2_info);
+  observer.reset();
+
+  let remove_folder_2_txn = PT.Remove(folder_level_2_info);
+  await remove_folder_2_txn.transact();
+
+  ensureUndoState([
+    [remove_folder_2_txn],
+    [folder_level_2_txn_result, folder_level_1_txn_result],
+  ]);
+  await ensureItemsRemoved(folder_level_2_info);
+
+  // Undo Remove "Folder Level 2"
+  await PT.undo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    1
+  );
+  await ensureItemsAdded(folder_level_2_info);
+  await ensureBookmarksTreeRestoredCorrectly(original_folder_level_2_tree);
+  observer.reset();
+
+  // Redo Remove "Folder Level 2"
+  await PT.redo();
+  ensureUndoState([
+    [remove_folder_2_txn],
+    [folder_level_2_txn_result, folder_level_1_txn_result],
+  ]);
+  await ensureItemsRemoved(folder_level_2_info);
+  observer.reset();
+
+  // Undo it again
+  await PT.undo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    1
+  );
+  await ensureItemsAdded(folder_level_2_info);
+  await ensureBookmarksTreeRestoredCorrectly(original_folder_level_2_tree);
+  observer.reset();
+
+  // Undo the creation of both folders
+  await PT.undo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    2
+  );
+  await ensureItemsRemoved(folder_level_2_info, folder_level_1_info);
+  observer.reset();
+
+  // Redo the creation of both folders
+  await PT.redo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    1
+  );
+  await ensureItemsAdded(folder_level_1_info, folder_level_2_info);
+  await ensureBookmarksTreeRestoredCorrectlyExceptDates(
+    original_folder_level_1_tree
+  );
+  observer.reset();
+
+  // Redo Remove "Folder Level 2"
+  await PT.redo();
+  ensureUndoState([
+    [remove_folder_2_txn],
+    [folder_level_2_txn_result, folder_level_1_txn_result],
+  ]);
+  await ensureItemsRemoved(folder_level_2_info);
+  observer.reset();
+
+  // Undo everything one last time
+  await PT.undo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    1
+  );
+  await ensureItemsAdded(folder_level_2_info);
+  observer.reset();
+
+  await PT.undo();
+  ensureUndoState(
+    [
+      [remove_folder_2_txn],
+      [folder_level_2_txn_result, folder_level_1_txn_result],
+    ],
+    2
+  );
+  await ensureItemsRemoved(folder_level_2_info, folder_level_2_info);
+  observer.reset();
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_add_and_remove_bookmarks_with_additional_info() {
+  const testURI = "http://add.remove.tag";
+  const TAG_1 = "TestTag1";
+  const TAG_2 = "TestTag2";
+
+  let folder_info = createTestFolderInfo();
+  folder_info.guid = await PT.NewFolder(folder_info).transact();
+  let ensureTags = ensureTagsForURI.bind(null, testURI);
+
+  // Check that the NewBookmark transaction preserves tags.
+  observer.reset();
+  let b1_info = { parentGuid: folder_info.guid, url: testURI, tags: [TAG_1] };
+  b1_info.guid = await PT.NewBookmark(b1_info).transact();
+  let b1_originalInfo = await PlacesUtils.promiseBookmarksTree(b1_info.guid);
+  ensureTags([TAG_1]);
+  await PT.undo();
+  ensureTags([]);
+
+  observer.reset();
+  await PT.redo();
+  await ensureBookmarksTreeRestoredCorrectly(b1_originalInfo);
+  ensureTags([TAG_1]);
+
+  // Check if the Remove transaction removes and restores tags of children
+  // correctly.
+  await PT.Remove(folder_info.guid).transact();
+  ensureTags([]);
+
+  observer.reset();
+  await PT.undo();
+  await ensureBookmarksTreeRestoredCorrectly(b1_originalInfo);
+  ensureTags([TAG_1]);
+
+  await PT.redo();
+  ensureTags([]);
+
+  observer.reset();
+  await PT.undo();
+  await ensureBookmarksTreeRestoredCorrectly(b1_originalInfo);
+  ensureTags([TAG_1]);
+
+  // * Check that no-op tagging (the uri is already tagged with TAG_1) is
+  //   also a no-op on undo.
+  observer.reset();
+  let b2_info = {
+    parentGuid: folder_info.guid,
+    url: testURI,
+    tags: [TAG_1, TAG_2],
+  };
+  b2_info.guid = await PT.NewBookmark(b2_info).transact();
+  ensureTags([TAG_1, TAG_2]);
+
+  observer.reset();
+  await PT.undo();
+  await ensureItemsRemoved(b2_info);
+  ensureTags([TAG_1]);
+
+  // Check if Remove correctly restores tags.
+  observer.reset();
+  await PT.redo();
+  ensureTags([TAG_1, TAG_2]);
+
+  // Test Remove for multiple items.
+  observer.reset();
+  await PT.Remove(b1_info.guid).transact();
+  await PT.Remove(b2_info.guid).transact();
+  await PT.Remove(folder_info.guid).transact();
+  await ensureItemsRemoved(b1_info, b2_info, folder_info);
+  ensureTags([]);
+
+  observer.reset();
+  await PT.undo();
+  await ensureItemsAdded(folder_info);
+  ensureTags([]);
+
+  observer.reset();
+  await PT.undo();
+  ensureTags([TAG_1, TAG_2]);
+
+  observer.reset();
+  await PT.undo();
+  await ensureItemsAdded(b1_info);
+  ensureTags([TAG_1, TAG_2]);
+
+  // The redo calls below cleanup everything we did.
+  observer.reset();
+  await PT.redo();
+  await ensureItemsRemoved(b1_info);
+  ensureTags([TAG_1, TAG_2]);
+
+  observer.reset();
+  await PT.redo();
+  // The tag containers are removed in async and take some time
+  let oldCountTag1 = 0;
+  let oldCountTag2 = 0;
+  let allTags = await bmsvc.fetchTags();
+  for (let i of allTags) {
+    if (i.name == TAG_1) {
+      oldCountTag1 = i.count;
+    }
+    if (i.name == TAG_2) {
+      oldCountTag2 = i.count;
+    }
+  }
+  await TestUtils.waitForCondition(async () => {
+    allTags = await bmsvc.fetchTags();
+    let newCountTag1 = 0;
+    let newCountTag2 = 0;
+    for (let i of allTags) {
+      if (i.name == TAG_1) {
+        newCountTag1 = i.count;
+      }
+      if (i.name == TAG_2) {
+        newCountTag2 = i.count;
+      }
+    }
+    return newCountTag1 == oldCountTag1 - 1 && newCountTag2 == oldCountTag2 - 1;
+  });
+  await ensureItemsRemoved(b2_info);
+
+  ensureTags([]);
+
+  observer.reset();
+  await PT.redo();
+  await ensureItemsRemoved(folder_info);
+  ensureTags([]);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_creating_and_removing_a_separator() {
+  let folder_info = createTestFolderInfo();
+  let separator_info = {};
+  let undoEntries = [];
+
+  observer.reset();
+  let create_txns = await PT.batch(async function () {
+    let folder_txn = PT.NewFolder(folder_info);
+    folder_info.guid = separator_info.parentGuid = await folder_txn.transact();
+    let separator_txn = PT.NewSeparator(separator_info);
+    separator_info.guid = await separator_txn.transact();
+    return [separator_txn, folder_txn];
+  });
+  undoEntries.unshift(create_txns);
+  ensureUndoState(undoEntries, 0);
+  ensureItemsAdded(folder_info, separator_info);
+
+  observer.reset();
+  await PT.undo();
+  ensureUndoState(undoEntries, 1);
+  ensureItemsRemoved(folder_info, separator_info);
+
+  observer.reset();
+  await PT.redo();
+  ensureUndoState(undoEntries, 0);
+  ensureItemsAdded(folder_info, separator_info);
+
+  observer.reset();
+  let remove_sep_txn = PT.Remove(separator_info);
+  await remove_sep_txn.transact();
+  undoEntries.unshift([remove_sep_txn]);
+  ensureUndoState(undoEntries, 0);
+  ensureItemsRemoved(separator_info);
+
+  observer.reset();
+  await PT.undo();
+  ensureUndoState(undoEntries, 1);
+  ensureItemsAdded(separator_info);
+
+  observer.reset();
+  await PT.undo();
+  ensureUndoState(undoEntries, 2);
+  ensureItemsRemoved(folder_info, separator_info);
+
+  observer.reset();
+  await PT.redo();
+  ensureUndoState(undoEntries, 1);
+  ensureItemsAdded(folder_info, separator_info);
+
+  // Clear redo entries and check that |redo| does nothing
+  observer.reset();
+  await PT.clearTransactionsHistory(false, true);
+  undoEntries.shift();
+  ensureUndoState(undoEntries, 0);
+  await PT.redo();
+  ensureItemsAdded();
+  ensureItemsRemoved();
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureUndoState(undoEntries, 1);
+  ensureItemsRemoved(folder_info, separator_info);
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_edit_title() {
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test_create_item.com",
+    title: "Original Title",
+  };
+
+  function ensureTitleChange(aCurrentTitle) {
+    ensureItemsTitleChanged({
+      guid: bm_info.guid,
+      title: aCurrentTitle,
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+  }
+
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+
+  observer.reset();
+  await PT.EditTitle({ guid: bm_info.guid, title: "New Title" }).transact();
+  ensureTitleChange("New Title");
+
+  observer.reset();
+  await PT.undo();
+  ensureTitleChange("Original Title");
+
+  observer.reset();
+  await PT.redo();
+  ensureTitleChange("New Title");
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureTitleChange("Original Title");
+  await PT.undo();
+  ensureItemsRemoved(bm_info);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_edit_url() {
+  let oldURI = "http://old.test_editing_item_uri.com/";
+  let newURI = "http://new.test_editing_item_uri.com/";
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: oldURI,
+    tags: ["TestTag"],
+  };
+  function ensureURIAndTags(
+    aPreChangeURI,
+    aPostChangeURI,
+    aOLdURITagsPreserved
+  ) {
+    ensureItemsUrlChanged({
+      guid: bm_info.guid,
+      url: aPostChangeURI,
+    });
+    ensureTagsForURI(aPostChangeURI, bm_info.tags);
+    ensureTagsForURI(aPreChangeURI, aOLdURITagsPreserved ? bm_info.tags : []);
+  }
+
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+  ensureTagsForURI(oldURI, bm_info.tags);
+
+  // When there's a single bookmark for the same url, tags should be moved.
+  observer.reset();
+  await PT.EditUrl({ guid: bm_info.guid, url: newURI }).transact();
+  ensureURIAndTags(oldURI, newURI, false);
+
+  observer.reset();
+  await PT.undo();
+  ensureURIAndTags(newURI, oldURI, false);
+
+  observer.reset();
+  await PT.redo();
+  ensureURIAndTags(oldURI, newURI, false);
+
+  observer.reset();
+  await PT.undo();
+  ensureURIAndTags(newURI, oldURI, false);
+
+  // When there're multiple bookmarks for the same url, tags should be copied.
+  let bm2_info = Object.create(bm_info);
+  bm2_info.guid = await PT.NewBookmark(bm2_info).transact();
+  let bm3_info = Object.create(bm_info);
+  bm3_info.url = newURI;
+  bm3_info.guid = await PT.NewBookmark(bm3_info).transact();
+
+  observer.reset();
+  await PT.EditUrl({ guid: bm_info.guid, url: newURI }).transact();
+  ensureURIAndTags(oldURI, newURI, true);
+
+  observer.reset();
+  await PT.undo();
+  ensureURIAndTags(newURI, oldURI, true);
+
+  observer.reset();
+  await PT.redo();
+  ensureURIAndTags(oldURI, newURI, true);
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureURIAndTags(newURI, oldURI, true);
+  await PT.undo();
+  await PT.undo();
+  await PT.undo();
+  ensureItemsRemoved(bm3_info, bm2_info, bm_info);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_edit_keyword() {
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test.edit.keyword/",
+  };
+  const KEYWORD = "test_keyword";
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+  function ensureKeywordChange(aCurrentKeyword = "") {
+    ensureItemsKeywordChanged({
+      guid: bm_info.guid,
+      keyword: aCurrentKeyword,
+    });
+  }
+
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+
+  observer.reset();
+  await PT.EditKeyword({
+    guid: bm_info.guid,
+    keyword: KEYWORD,
+    postData: "postData",
+  }).transact();
+  ensureKeywordChange(KEYWORD);
+  let entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData");
+
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange();
+  entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry, null);
+
+  observer.reset();
+  await PT.redo();
+  ensureKeywordChange(KEYWORD);
+  entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData");
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange();
+  await PT.undo();
+  ensureItemsRemoved(bm_info);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_edit_keyword_null_postData() {
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test.edit.keyword/",
+  };
+  const KEYWORD = "test_keyword";
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+  function ensureKeywordChange(aCurrentKeyword = "") {
+    ensureItemsKeywordChanged({
+      guid: bm_info.guid,
+      keyword: aCurrentKeyword,
+    });
+  }
+
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+
+  observer.reset();
+  await PT.EditKeyword({
+    guid: bm_info.guid,
+    keyword: KEYWORD,
+    postData: null,
+  }).transact();
+  ensureKeywordChange(KEYWORD);
+  let entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, null);
+
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange();
+  entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry, null);
+
+  observer.reset();
+  await PT.redo();
+  ensureKeywordChange(KEYWORD);
+  entry = await PlacesUtils.keywords.fetch(KEYWORD);
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, null);
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange();
+  await PT.undo();
+  ensureItemsRemoved(bm_info);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_edit_specific_keyword() {
+  let bm_info = {
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test.edit.keyword/",
+  };
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+  function ensureKeywordChange(aCurrentKeyword = "", aPreviousKeyword = "") {
+    ensureItemsKeywordChanged({
+      guid: bm_info.guid,
+      keyword: aCurrentKeyword,
+    });
+  }
+
+  await PlacesUtils.keywords.insert({
+    keyword: "kw1",
+    url: bm_info.url,
+    postData: "postData1",
+  });
+  await PlacesUtils.keywords.insert({
+    keyword: "kw2",
+    url: bm_info.url,
+    postData: "postData2",
+  });
+  bm_info.guid = await PT.NewBookmark(bm_info).transact();
+
+  observer.reset();
+  await PT.EditKeyword({
+    guid: bm_info.guid,
+    keyword: "keyword",
+    oldKeyword: "kw2",
+  }).transact();
+  ensureKeywordChange("keyword", "kw2");
+  let entry = await PlacesUtils.keywords.fetch("kw1");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData1");
+  entry = await PlacesUtils.keywords.fetch("keyword");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData2");
+  entry = await PlacesUtils.keywords.fetch("kw2");
+  Assert.equal(entry, null);
+
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange("kw2", "keyword");
+  entry = await PlacesUtils.keywords.fetch("kw1");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData1");
+  entry = await PlacesUtils.keywords.fetch("kw2");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData2");
+  entry = await PlacesUtils.keywords.fetch("keyword");
+  Assert.equal(entry, null);
+
+  observer.reset();
+  await PT.redo();
+  ensureKeywordChange("keyword", "kw2");
+  entry = await PlacesUtils.keywords.fetch("kw1");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData1");
+  entry = await PlacesUtils.keywords.fetch("keyword");
+  Assert.equal(entry.url.href, bm_info.url);
+  Assert.equal(entry.postData, "postData2");
+  entry = await PlacesUtils.keywords.fetch("kw2");
+  Assert.equal(entry, null);
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureKeywordChange("kw2");
+  await PT.undo();
+  ensureItemsRemoved(bm_info);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_tag_uri() {
+  // This also tests passing uri specs.
+  let bm_info_a = {
+    url: "http://bookmarked.uri",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  };
+  let bm_info_b = {
+    url: "http://bookmarked2.uri",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  };
+  let unbookmarked_uri = "http://un.bookmarked.uri";
+
+  await PT.batch(async function () {
+    bm_info_a.guid = await PT.NewBookmark(bm_info_a).transact();
+    bm_info_b.guid = await PT.NewBookmark(bm_info_b).transact();
+  });
+
+  async function doTest(aInfo) {
+    let urls = "url" in aInfo ? [aInfo.url] : aInfo.urls;
+    let tags = "tag" in aInfo ? [aInfo.tag] : aInfo.tags;
+
+    let tagWillAlsoBookmark = new Set();
+    for (let url of urls) {
+      if (!(await bmsvc.fetch({ url }))) {
+        tagWillAlsoBookmark.add(url);
+      }
+    }
+
+    async function ensureTagsSet() {
+      for (let url of urls) {
+        ensureTagsForURI(url, tags);
+        Assert.ok(await bmsvc.fetch({ url }));
+      }
+    }
+    async function ensureTagsUnset() {
+      for (let url of urls) {
+        ensureTagsForURI(url, []);
+        if (tagWillAlsoBookmark.has(url)) {
+          Assert.ok(!(await bmsvc.fetch({ url })));
+        } else {
+          Assert.ok(await bmsvc.fetch({ url }));
+        }
+      }
+    }
+
+    await PT.Tag(aInfo).transact();
+    await ensureTagsSet();
+    await PT.undo();
+    await ensureTagsUnset();
+    await PT.redo();
+    await ensureTagsSet();
+    await PT.undo();
+    await ensureTagsUnset();
+  }
+
+  await doTest({ url: bm_info_a.url, tags: ["MyTag"] });
+  await doTest({ urls: [bm_info_a.url], tag: "MyTag" });
+  await doTest({ urls: [bm_info_a.url, bm_info_b.url], tags: ["A, B"] });
+  await doTest({ urls: [bm_info_a.url, unbookmarked_uri], tag: "C" });
+  // Duplicate URLs listed.
+  await doTest({
+    urls: [bm_info_a.url, bm_info_b.url, bm_info_a.url],
+    tag: "D",
+  });
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureItemsRemoved(bm_info_a, bm_info_b);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_untag_uri() {
+  let bm_info_a = {
+    url: "http://bookmarked.uri",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    tags: ["A", "B"],
+  };
+  let bm_info_b = {
+    url: "http://bookmarked2.uri",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    tag: "B",
+  };
+
+  await PT.batch(async function () {
+    bm_info_a.guid = await PT.NewBookmark(bm_info_a).transact();
+    ensureTagsForURI(bm_info_a.url, bm_info_a.tags);
+    bm_info_b.guid = await PT.NewBookmark(bm_info_b).transact();
+    ensureTagsForURI(bm_info_b.url, [bm_info_b.tag]);
+  });
+
+  async function doTest(aInfo) {
+    let urls, tagsRemoved;
+    if (typeof aInfo == "string") {
+      urls = [aInfo];
+      tagsRemoved = [];
+    } else if (Array.isArray(aInfo)) {
+      urls = aInfo;
+      tagsRemoved = [];
+    } else {
+      urls = "url" in aInfo ? [aInfo.url] : aInfo.urls;
+      tagsRemoved = "tag" in aInfo ? [aInfo.tag] : aInfo.tags;
+    }
+
+    let preRemovalTags = new Map();
+    for (let url of urls) {
+      preRemovalTags.set(url, tagssvc.getTagsForURI(Services.io.newURI(url)));
+    }
+
+    function ensureTagsSet() {
+      for (let url of urls) {
+        ensureTagsForURI(url, preRemovalTags.get(url));
+      }
+    }
+    function ensureTagsUnset() {
+      for (let url of urls) {
+        let expectedTags = !tagsRemoved.length
+          ? []
+          : preRemovalTags.get(url).filter(tag => !tagsRemoved.includes(tag));
+        ensureTagsForURI(url, expectedTags);
+      }
+    }
+
+    await PT.Untag(aInfo).transact();
+    await ensureTagsUnset();
+    await PT.undo();
+    await ensureTagsSet();
+    await PT.redo();
+    await ensureTagsUnset();
+    await PT.undo();
+    await ensureTagsSet();
+  }
+
+  await doTest(bm_info_a);
+  await doTest(bm_info_b);
+  await doTest(bm_info_a.url);
+  await doTest(bm_info_b.url);
+  await doTest([bm_info_a.url, bm_info_b.url]);
+  await doTest({ urls: [bm_info_a.url, bm_info_b.url], tags: ["A", "B"] });
+  await doTest({ urls: [bm_info_a.url, bm_info_b.url], tag: "B" });
+  await doTest({ urls: [bm_info_a.url, bm_info_b.url], tag: "C" });
+  await doTest({ urls: [bm_info_a.url, bm_info_b.url], tags: ["C"] });
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureItemsRemoved(bm_info_a, bm_info_b);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+});
+
+add_task(async function test_sort_folder_by_name() {
+  let folder_info = createTestFolderInfo();
+
+  let url = "http://sort.by.name/";
+  let preSep = ["3", "2", "1"].map(i => ({ title: i, url }));
+  let sep = {};
+  let postSep = ["c", "b", "a"].map(l => ({ title: l, url }));
+  let originalOrder = [...preSep, sep, ...postSep];
+  let sortedOrder = [
+    ...preSep.slice(0).reverse(),
+    sep,
+    ...postSep.slice(0).reverse(),
+  ];
+  await PT.batch(async function () {
+    folder_info.guid = await PT.NewFolder(folder_info).transact();
+    for (let info of originalOrder) {
+      info.parentGuid = folder_info.guid;
+      info.guid = await (info == sep
+        ? PT.NewSeparator(info).transact()
+        : PT.NewBookmark(info).transact());
+    }
+  });
+
+  let folderContainer = PlacesUtils.getFolderContents(folder_info.guid).root;
+  function ensureOrder(aOrder) {
+    for (let i = 0; i < folderContainer.childCount; i++) {
+      Assert.equal(folderContainer.getChild(i).bookmarkGuid, aOrder[i].guid);
+    }
+  }
+
+  ensureOrder(originalOrder);
+  await PT.SortByName(folder_info.guid).transact();
+  ensureOrder(sortedOrder);
+  await PT.undo();
+  ensureOrder(originalOrder);
+  await PT.redo();
+  ensureOrder(sortedOrder);
+
+  // Cleanup
+  observer.reset();
+  await PT.undo();
+  ensureOrder(originalOrder);
+  await PT.undo();
+  ensureItemsRemoved(...originalOrder, folder_info);
+});
+
+add_task(async function test_copy() {
+  async function duplicate_and_test(aOriginalGuid) {
+    let txn = PT.Copy({
+      guid: aOriginalGuid,
+      newParentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+    let duplicateGuid = await txn.transact();
+    let originalInfo = await PlacesUtils.promiseBookmarksTree(aOriginalGuid);
+    let duplicateInfo = await PlacesUtils.promiseBookmarksTree(duplicateGuid);
+    await ensureEqualBookmarksTrees(originalInfo, duplicateInfo, false);
+
+    async function redo() {
+      await PT.redo();
+      await ensureBookmarksTreeRestoredCorrectlyExceptDates(originalInfo);
+      await PT.redo();
+      await ensureBookmarksTreeRestoredCorrectlyExceptDates(duplicateInfo);
+    }
+    async function undo() {
+      await PT.undo();
+      // also undo the original item addition.
+      await PT.undo();
+      await ensureNonExistent(aOriginalGuid, duplicateGuid);
+    }
+
+    await undo();
+    await redo();
+    await undo();
+    await redo();
+
+    // Cleanup. This also remove the original item.
+    await PT.undo();
+    observer.reset();
+    await PT.clearTransactionsHistory();
+  }
+
+  let timerPrecision = Preferences.get("privacy.reduceTimerPrecision");
+  Preferences.set("privacy.reduceTimerPrecision", false);
+
+  registerCleanupFunction(function () {
+    Preferences.set("privacy.reduceTimerPrecision", timerPrecision);
+  });
+
+  // Test duplicating leafs (bookmark, separator, empty folder)
+  PT.NewBookmark({
+    url: "http://test.item.duplicate",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    annos: [{ name: "Anno", value: "AnnoValue" }],
+  });
+  let sepTxn = PT.NewSeparator({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: 1,
+  });
+
+  let emptyFolderTxn = PT.NewFolder(createTestFolderInfo());
+  for (let txn of [sepTxn, emptyFolderTxn]) {
+    let guid = await txn.transact();
+    await duplicate_and_test(guid);
+  }
+
+  // Test duplicating a folder having some contents.
+  let filledFolderGuid = await PT.batch(async function () {
+    let folderGuid = await PT.NewFolder(createTestFolderInfo()).transact();
+    let nestedFolderGuid = await PT.NewFolder({
+      parentGuid: folderGuid,
+      title: "Nested Folder",
+    }).transact();
+    // Insert a bookmark under the nested folder.
+    await PT.NewBookmark({
+      url: "http://nested.nested.bookmark",
+      parentGuid: nestedFolderGuid,
+    }).transact();
+    // Insert a separator below the nested folder
+    await PT.NewSeparator({ parentGuid: folderGuid }).transact();
+    // And another bookmark.
+    await PT.NewBookmark({
+      url: "http://nested.bookmark",
+      parentGuid: folderGuid,
+    }).transact();
+    return folderGuid;
+  });
+
+  await duplicate_and_test(filledFolderGuid);
+
+  // Cleanup
+  await PT.clearTransactionsHistory();
+});
+
+add_task(async function test_array_input_for_batch() {
+  let folderTxn = PT.NewFolder(createTestFolderInfo());
+  let folderGuid = await folderTxn.transact();
+
+  let sep1_txn = PT.NewSeparator({ parentGuid: folderGuid });
+  let sep2_txn = PT.NewSeparator({ parentGuid: folderGuid });
+  await PT.batch([sep1_txn, sep2_txn]);
+  ensureUndoState([[sep2_txn, sep1_txn], [folderTxn]], 0);
+
+  let ensureChildCount = async function (count) {
+    let tree = await PlacesUtils.promiseBookmarksTree(folderGuid);
+    if (count == 0) {
+      Assert.ok(!("children" in tree));
+    } else {
+      Assert.equal(tree.children.length, count);
+    }
+  };
+
+  await ensureChildCount(2);
+  await PT.undo();
+  await ensureChildCount(0);
+  await PT.redo();
+  await ensureChildCount(2);
+  await PT.undo();
+  await ensureChildCount(0);
+
+  await PT.undo();
+  Assert.equal(await PlacesUtils.promiseBookmarksTree(folderGuid), null);
+
+  // Cleanup
+  await PT.clearTransactionsHistory();
+});
+
+add_task(async function test_invalid_uri_spec_throws() {
+  Assert.throws(
+    () =>
+      PT.NewBookmark({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: "invalid uri spec",
+        title: "test bookmark",
+      }),
+    /invalid uri spec is not a valid URL/
+  );
+  Assert.throws(
+    () => PT.Tag({ tag: "TheTag", urls: ["invalid uri spec"] }),
+    /TypeError: URL constructor: invalid uri spec is not a valid URL/
+  );
+  Assert.throws(
+    () => PT.Tag({ tag: "TheTag", urls: ["about:blank", "invalid uri spec"] }),
+    /TypeError: URL constructor: invalid uri spec is not a valid URL/
+  );
+});
+
+add_task(async function test_remove_multiple() {
+  let guids = [];
+  await PT.batch(async function () {
+    let folderGuid = await PT.NewFolder({
+      title: "Test Folder",
+      parentGuid: menuGuid,
+    }).transact();
+    let nestedFolderGuid = await PT.NewFolder({
+      title: "Nested Test Folder",
+      parentGuid: folderGuid,
+    }).transact();
+    await PT.NewSeparator(nestedFolderGuid).transact();
+
+    guids.push(folderGuid);
+
+    let bmGuid = await PT.NewBookmark({
+      url: "http://test.bookmark.removed",
+      parentGuid: menuGuid,
+    }).transact();
+    guids.push(bmGuid);
+  });
+
+  let originalInfos = [];
+  for (let guid of guids) {
+    originalInfos.push(await PlacesUtils.promiseBookmarksTree(guid));
+  }
+
+  await PT.Remove(guids).transact();
+  await ensureNonExistent(...guids);
+  await PT.undo();
+  await ensureBookmarksTreeRestoredCorrectly(...originalInfos);
+  await PT.redo();
+  await ensureNonExistent(...guids);
+  await PT.undo();
+  await ensureBookmarksTreeRestoredCorrectly(...originalInfos);
+
+  // Undo the New* transactions batch.
+  await PT.undo();
+  await ensureNonExistent(...guids);
+
+  // Redo it.
+  await PT.redo();
+  await ensureBookmarksTreeRestoredCorrectlyExceptDates(...originalInfos);
+
+  // Redo remove.
+  await PT.redo();
+  await ensureNonExistent(...guids);
+
+  // Cleanup
+  await PT.clearTransactionsHistory();
+  observer.reset();
+});
+
+add_task(async function test_renameTag() {
+  let url = "http://test.edit.keyword/";
+  await PT.Tag({ url, tags: ["t1", "t2"] }).transact();
+  ensureTagsForURI(url, ["t1", "t2"]);
+
+  // Create bookmark queries that point to the modified tag.
+  let bm1 = await PlacesUtils.bookmarks.insert({
+    url: "place:tag=t2",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  let bm2 = await PlacesUtils.bookmarks.insert({
+    url: "place:tag=t2&sort=1",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  // This points to 2 tags, and as such won't be touched.
+  let bm3 = await PlacesUtils.bookmarks.insert({
+    url: "place:tag=t2&tag=t1",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+
+  await PT.RenameTag({ oldTag: "t2", tag: "t3" }).transact();
+  ensureTagsForURI(url, ["t1", "t3"]);
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm1.guid)).url.href,
+    "place:tag=t3",
+    "The fitst bookmark has been updated"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm2.guid)).url.href,
+    "place:tag=t3&sort=1",
+    "The second bookmark has been updated"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm3.guid)).url.href,
+    "place:tag=t3&tag=t1",
+    "The third bookmark has been updated"
+  );
+
+  await PT.undo();
+  ensureTagsForURI(url, ["t1", "t2"]);
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm1.guid)).url.href,
+    "place:tag=t2",
+    "The fitst bookmark has been restored"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm2.guid)).url.href,
+    "place:tag=t2&sort=1",
+    "The second bookmark has been restored"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm3.guid)).url.href,
+    "place:tag=t2&tag=t1",
+    "The third bookmark has been restored"
+  );
+
+  await PT.redo();
+  ensureTagsForURI(url, ["t1", "t3"]);
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm1.guid)).url.href,
+    "place:tag=t3",
+    "The fitst bookmark has been updated"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm2.guid)).url.href,
+    "place:tag=t3&sort=1",
+    "The second bookmark has been updated"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm3.guid)).url.href,
+    "place:tag=t3&tag=t1",
+    "The third bookmark has been updated"
+  );
+
+  await PT.undo();
+  ensureTagsForURI(url, ["t1", "t2"]);
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm1.guid)).url.href,
+    "place:tag=t2",
+    "The fitst bookmark has been restored"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm2.guid)).url.href,
+    "place:tag=t2&sort=1",
+    "The second bookmark has been restored"
+  );
+  Assert.equal(
+    (await PlacesUtils.bookmarks.fetch(bm3.guid)).url.href,
+    "place:tag=t2&tag=t1",
+    "The third bookmark has been restored"
+  );
+
+  await PT.undo();
+  ensureTagsForURI(url, []);
+
+  await PT.clearTransactionsHistory();
+  ensureUndoState();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_remove_invalid_url() {
+  let folderGuid = await PT.NewFolder({
+    title: "Test Folder",
+    parentGuid: menuGuid,
+  }).transact();
+
+  let guid = "invalid_____";
+  let folderedGuid = "invalid____2";
+  let url = "invalid-uri";
+  await PlacesUtils.withConnectionWrapper("test_bookmarks_remove", async db => {
+    await db.execute(
+      `
+      INSERT INTO moz_places(url, url_hash, title, rev_host, guid)
+      VALUES (:url, hash(:url), 'Invalid URI', '.', GENERATE_GUID())
+    `,
+      { url }
+    );
+    await db.execute(
+      `INSERT INTO moz_bookmarks (type, fk, parent, position, guid)
+        VALUES (:type,
+                (SELECT id FROM moz_places WHERE url = :url),
+                (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid),
+                (SELECT MAX(position) + 1 FROM moz_bookmarks WHERE parent = (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid)),
+                :guid)
+      `,
+      {
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        url,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        guid,
+      }
+    );
+    await db.execute(
+      `INSERT INTO moz_bookmarks (type, fk, parent, position, guid)
+        VALUES (:type,
+                (SELECT id FROM moz_places WHERE url = :url),
+                (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid),
+                (SELECT MAX(position) + 1 FROM moz_bookmarks WHERE parent = (SELECT id FROM moz_bookmarks WHERE guid = :parentGuid)),
+                :guid)
+      `,
+      {
+        type: PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        url,
+        parentGuid: folderGuid,
+        guid: folderedGuid,
+      }
+    );
+  });
+
+  let guids = [folderGuid, guid];
+  await PT.Remove(guids).transact();
+  await ensureNonExistent(...guids, folderedGuid);
+  // Shouldn't throw, should restore the folder but not the bookmarks.
+  await PT.undo();
+  await ensureNonExistent(guid, folderedGuid);
+  Assert.ok(
+    await PlacesUtils.bookmarks.fetch(folderGuid),
+    "The folder should have been re-created"
+  );
+  await PT.redo();
+  await ensureNonExistent(guids, folderedGuid);
+  // Cleanup
+  await PT.clearTransactionsHistory();
+  observer.reset();
+});
diff --git a/toolkit/components/places/tests/unit/test_autocomplete_match_fallbackTitle.js b/toolkit/components/places/tests/unit/test_autocomplete_match_fallbackTitle.js
new file mode 100644
index 0000000000..109ca1edce
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_autocomplete_match_fallbackTitle.js
@@ -0,0 +1,27 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+// This is a test for the fallbackTitle argument of autocomplete_match.
+
+add_task(async function test_match() {
+  async function search(text) {
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(
+      `
+      SELECT AUTOCOMPLETE_MATCH(:text, 'http://mozilla.org/', 'Main title',
+                                NULL, NULL, 1, 1, NULL,
+                                :matchBehavior, :searchBehavior,
+                                'Fallback title')
+      `,
+      {
+        text,
+        matchBehavior: Ci.mozIPlacesAutoComplete.MATCH_ANYWHERE,
+        searchBehavior: 643,
+      }
+    );
+    return !!rows[0].getResultByIndex(0);
+  }
+  Assert.ok(await search("mai"), "Match on main title");
+  Assert.ok(await search("fall"), "Match on fallback title");
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmark-tags-changed_frequency.js b/toolkit/components/places/tests/unit/test_bookmark-tags-changed_frequency.js
new file mode 100644
index 0000000000..568ad43841
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmark-tags-changed_frequency.js
@@ -0,0 +1,56 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test checks that changing a tag for a bookmark with multiple tags
+// notifies bookmark-tags-changed event only once, and not once per tag.
+
+add_task(async function run_test() {
+  let tags = ["a", "b", "c"];
+  let uri = Services.io.newURI("http://1.moz.org/");
+
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: uri,
+    title: "Bookmark 1",
+  });
+  PlacesUtils.tagging.tagURI(uri, tags);
+
+  let promise = PromiseUtils.defer();
+
+  let bookmarksObserver = {
+    _changedCount: 0,
+    handlePlacesEvents(events) {
+      for (let event of events) {
+        switch (event.type) {
+          case "bookmark-removed":
+            if (event.guid == bookmark.guid) {
+              PlacesUtils.observers.removeListener(
+                ["bookmark-removed"],
+                this.handlePlacesEvents
+              );
+              Assert.equal(this._changedCount, 2);
+              promise.resolve();
+            }
+            break;
+          case "bookmark-tags-changed":
+            Assert.equal(event.guid, bookmark.guid);
+            this._changedCount++;
+            break;
+        }
+      }
+    },
+  };
+  bookmarksObserver.handlePlacesEvents =
+    bookmarksObserver.handlePlacesEvents.bind(bookmarksObserver);
+  PlacesUtils.observers.addListener(
+    ["bookmark-removed", "bookmark-tags-changed"],
+    bookmarksObserver.handlePlacesEvents
+  );
+
+  PlacesUtils.tagging.tagURI(uri, ["d"]);
+  PlacesUtils.tagging.tagURI(uri, ["e"]);
+
+  await promise;
+
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html.js b/toolkit/components/places/tests/unit/test_bookmarks_html.js
new file mode 100644
index 0000000000..4b3f04b444
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html.js
@@ -0,0 +1,417 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// An object representing the contents of bookmarks.preplaces.html.
+var test_bookmarks = {
+  menu: [
+    {
+      title: "Mozilla Firefox",
+      children: [
+        {
+          title: "Help and Tutorials",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/help/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          title: "Customize Firefox",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/customize/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          title: "Get Involved",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/community/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          title: "About Us",
+          url: "http://en-us.www.mozilla.com/en-US/about/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+      ],
+    },
+    {
+      type: Ci.nsINavHistoryResultNode.RESULT_TYPE_SEPARATOR,
+    },
+    {
+      title: "test",
+      dateAdded: 1177541020000000,
+      lastModified: 1177541050000000,
+      children: [
+        {
+          title: "test post keyword",
+          dateAdded: 1177375336000000,
+          lastModified: 1177375423000000,
+          keyword: "test",
+          postData: "hidden1%3Dbar&text1%3D%25s",
+          charset: "ISO-8859-1",
+          url: "http://test/post",
+        },
+      ],
+    },
+  ],
+  toolbar: [
+    {
+      title: "Getting Started",
+      url: "http://en-us.www.mozilla.com/en-US/firefox/central/",
+      icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+    },
+    {
+      title: "Latest Headlines",
+      url: "http://en-us.fxfeeds.mozilla.com/en-US/firefox/livebookmarks/",
+      feedUrl: "http://en-us.fxfeeds.mozilla.com/en-US/firefox/headlines.xml",
+    },
+    // This will be ignored, because it has no url.
+    {
+      title: "Latest Headlines No Site",
+      feedUrl: "http://en-us.fxfeeds.mozilla.com/en-US/firefox/headlines.xml",
+      ignore: true,
+    },
+  ],
+  unfiled: [{ title: "Example.tld", url: "http://example.tld/" }],
+};
+
+// Pre-Places bookmarks.html file pointer.
+var gBookmarksFileOld;
+// Places bookmarks.html file pointer.
+var gBookmarksFileNew;
+
+add_task(async function setup() {
+  // File pointer to legacy bookmarks file.
+  gBookmarksFileOld = PathUtils.join(
+    do_get_cwd().path,
+    "bookmarks.preplaces.html"
+  );
+
+  // File pointer to a new Places-exported bookmarks file.
+  gBookmarksFileNew = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.html"
+  );
+  await IOUtils.remove(gBookmarksFileNew, { ignoreAbsent: true });
+
+  // This test must be the first one, since it setups the new bookmarks.html.
+  // Test importing a pre-Places canonical bookmarks file.
+  // 1. import bookmarks.preplaces.html
+  // 2. run the test-suite
+  // Note: we do not empty the db before this import to catch bugs like 380999
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileOld, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_count() {
+  // Ensure the bookmarks count is correct when importing in various cases
+  let count = await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, {
+    replace: true,
+  });
+  Assert.equal(
+    count,
+    8,
+    "There should be 8 imported bookmarks when importing from an empty database"
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  count = -1;
+  count = await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, {
+    replace: true,
+  });
+  Assert.equal(
+    count,
+    8,
+    "There should be 8 imported bookmarks when replacing existing bookmarks"
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  count = -1;
+  count = await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew);
+  Assert.equal(
+    count,
+    8,
+    "There should be 8 imported bookmarks even when we are not replacing existing bookmarks"
+  );
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_new() {
+  // Test importing a Places bookmarks.html file.
+  // 1. import bookmarks.exported.html
+  // 2. run the test-suite
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  await testImportedBookmarks();
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_emptytitle_export() {
+  // Test exporting and importing with an empty-titled bookmark.
+  // 1. import bookmarks
+  // 2. create an empty-titled bookmark.
+  // 3. export to bookmarks.exported.html
+  // 4. empty bookmarks db
+  // 5. import bookmarks.exported.html
+  // 6. run the test-suite
+  // 7. remove the empty-titled bookmark
+  // 8. export to bookmarks.exported.html
+  // 9. empty bookmarks db and continue
+
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  const NOTITLE_URL = "http://notitle.mozilla.org/";
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: NOTITLE_URL,
+  });
+  test_bookmarks.unfiled.push({ title: "", url: NOTITLE_URL });
+
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+
+  // Cleanup.
+  test_bookmarks.unfiled.pop();
+  // HTML imports don't restore GUIDs yet.
+  let reimportedBookmark = await PlacesUtils.bookmarks.fetch({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+  });
+  Assert.equal(reimportedBookmark.url.href, bookmark.url.href);
+  await PlacesUtils.bookmarks.remove(reimportedBookmark);
+
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_chromefavicon() {
+  // Test exporting and importing with a bookmark pointing to a chrome favicon.
+  // 1. import bookmarks
+  // 2. create a bookmark pointing to a chrome favicon.
+  // 3. export to bookmarks.exported.html
+  // 4. empty bookmarks db
+  // 5. import bookmarks.exported.html
+  // 6. run the test-suite
+  // 7. remove the bookmark pointing to a chrome favicon.
+  // 8. export to bookmarks.exported.html
+  // 9. empty bookmarks db and continue
+
+  const PAGE_URI = NetUtil.newURI("http://example.com/chromefavicon_page");
+  const CHROME_FAVICON_URI = NetUtil.newURI(
+    "chrome://global/skin/icons/delete.svg"
+  );
+  const CHROME_FAVICON_URI_2 = NetUtil.newURI(
+    "chrome://global/skin/icons/error.svg"
+  );
+
+  info("Importing from html");
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  info("Insert bookmark");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: PAGE_URI,
+    title: "Test",
+  });
+
+  info("Set favicon");
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      PAGE_URI,
+      CHROME_FAVICON_URI,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      resolve,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  let data = await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      PAGE_URI,
+      (uri, dataLen, faviconData, mimeType) => resolve(faviconData)
+    );
+  });
+
+  let base64Icon =
+    "data:image/png;base64," +
+    base64EncodeString(String.fromCharCode.apply(String, data));
+
+  test_bookmarks.unfiled.push({
+    title: "Test",
+    url: PAGE_URI.spec,
+    icon: base64Icon,
+  });
+
+  info("Export to html");
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  info("Set favicon");
+  // Change the favicon to check it's really imported again later.
+  await new Promise(resolve => {
+    PlacesUtils.favicons.setAndFetchFaviconForPage(
+      PAGE_URI,
+      CHROME_FAVICON_URI_2,
+      true,
+      PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+      resolve,
+      Services.scriptSecurityManager.getSystemPrincipal()
+    );
+  });
+
+  info("import from html");
+  await PlacesUtils.bookmarks.eraseEverything();
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  info("Test imported bookmarks");
+  await testImportedBookmarks();
+
+  // Cleanup.
+  test_bookmarks.unfiled.pop();
+  // HTML imports don't restore GUIDs yet.
+  let reimportedBookmark = await PlacesUtils.bookmarks.fetch({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+  });
+  await PlacesUtils.bookmarks.remove(reimportedBookmark);
+
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_ontop() {
+  // Test importing the exported bookmarks.html file *on top of* the existing
+  // bookmarks.
+  // 1. empty bookmarks db
+  // 2. import the exported bookmarks file
+  // 3. export to file
+  // 3. import the exported bookmarks file
+  // 4. run the test-suite
+
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await BookmarkHTMLUtils.exportToFile(gBookmarksFileNew);
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  await BookmarkHTMLUtils.importFromFile(gBookmarksFileNew, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+async function testImportedBookmarks() {
+  for (let group in test_bookmarks) {
+    info("[testImportedBookmarks()] Checking group '" + group + "'");
+
+    let root = PlacesUtils.getFolderContents(
+      PlacesUtils.bookmarks[`${group}Guid`]
+    ).root;
+
+    let items = test_bookmarks[group].filter(b => !b.ignore);
+    Assert.equal(root.childCount, items.length);
+
+    for (let key in items) {
+      await checkItem(items[key], root.getChild(key));
+    }
+
+    root.containerOpen = false;
+  }
+}
+
+function checkItem(aExpected, aNode) {
+  return (async function () {
+    let bookmark = await PlacesUtils.bookmarks.fetch(aNode.bookmarkGuid);
+
+    for (let prop in aExpected) {
+      switch (prop) {
+        case "type":
+          Assert.equal(aNode.type, aExpected.type);
+          break;
+        case "title":
+          Assert.equal(aNode.title, aExpected.title);
+          break;
+        case "dateAdded":
+          Assert.equal(
+            PlacesUtils.toPRTime(bookmark.dateAdded),
+            aExpected.dateAdded
+          );
+          break;
+        case "lastModified":
+          Assert.equal(
+            PlacesUtils.toPRTime(bookmark.lastModified),
+            aExpected.lastModified
+          );
+          break;
+        case "url":
+          Assert.equal(aNode.uri, aExpected.url);
+          break;
+        case "icon":
+          let { data } = await getFaviconDataForPage(aExpected.url);
+          let base64Icon =
+            "data:image/png;base64," +
+            base64EncodeString(String.fromCharCode.apply(String, data));
+          Assert.ok(base64Icon == aExpected.icon);
+          break;
+        case "keyword": {
+          let entry = await PlacesUtils.keywords.fetch({ url: aNode.uri });
+          Assert.equal(entry.keyword, aExpected.keyword);
+          break;
+        }
+        case "postData": {
+          let entry = await PlacesUtils.keywords.fetch({ url: aNode.uri });
+          Assert.equal(entry.postData, aExpected.postData);
+          break;
+        }
+        case "charset":
+          let pageInfo = await PlacesUtils.history.fetch(aNode.uri, {
+            includeAnnotations: true,
+          });
+          Assert.equal(
+            pageInfo.annotations.get(PlacesUtils.CHARSET_ANNO),
+            aExpected.charset
+          );
+          break;
+        case "feedUrl":
+          // No more supported.
+          break;
+        case "children":
+          let folder = aNode.QueryInterface(
+            Ci.nsINavHistoryContainerResultNode
+          );
+          Assert.equal(folder.hasChildren, !!aExpected.children.length);
+          folder.containerOpen = true;
+          Assert.equal(folder.childCount, aExpected.children.length);
+
+          for (let index = 0; index < aExpected.children.length; index++) {
+            await checkItem(aExpected.children[index], folder.getChild(index));
+          }
+
+          folder.containerOpen = false;
+          break;
+        default:
+          throw new Error("Unknown property");
+      }
+    }
+  })();
+}
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html_corrupt.js b/toolkit/components/places/tests/unit/test_bookmarks_html_corrupt.js
new file mode 100644
index 0000000000..061c8c0c5f
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html_corrupt.js
@@ -0,0 +1,126 @@
+/*
+ * This test ensures that importing/exporting to HTML does not stop
+ * if a malformed uri is found.
+ */
+
+const TEST_FAVICON_PAGE_URL =
+  "http://en-US.www.mozilla.com/en-US/firefox/central/";
+const TEST_FAVICON_DATA_SIZE = 580;
+
+add_task(async function test_corrupt_file() {
+  // Import bookmarks from the corrupt file.
+  let corruptHtml = PathUtils.join(do_get_cwd().path, "bookmarks.corrupt.html");
+  await BookmarkHTMLUtils.importFromFile(corruptHtml, { replace: true });
+
+  // Check that bookmarks that are not corrupt have been imported.
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await database_check();
+});
+
+add_task(async function test_corrupt_database() {
+  // Create corruption in the database, then export.
+  let corruptBookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    url: "http://test.mozilla.org",
+    title: "We love belugas",
+  });
+  await PlacesUtils.withConnectionWrapper("test", async function (db) {
+    await db.execute("UPDATE moz_bookmarks SET fk = NULL WHERE guid = :guid", {
+      guid: corruptBookmark.guid,
+    });
+  });
+
+  let bookmarksFile = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.html"
+  );
+  await IOUtils.remove(bookmarksFile, { ignoreAbsent: true });
+  await BookmarkHTMLUtils.exportToFile(bookmarksFile);
+
+  // Import again and check for correctness.
+  await PlacesUtils.bookmarks.eraseEverything();
+  await BookmarkHTMLUtils.importFromFile(bookmarksFile, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await database_check();
+});
+
+/*
+ * Check for imported bookmarks correctness
+ *
+ * @return {Promise}
+ * @resolves When the checks are finished.
+ * @rejects Never.
+ */
+var database_check = async function () {
+  // BOOKMARKS MENU
+  let root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.menuGuid).root;
+  Assert.equal(root.childCount, 2);
+
+  let folderNode = root.getChild(1);
+  Assert.equal(folderNode.type, folderNode.RESULT_TYPE_FOLDER);
+  Assert.equal(folderNode.title, "test");
+
+  let bookmark = await PlacesUtils.bookmarks.fetch({
+    guid: folderNode.bookmarkGuid,
+  });
+  Assert.equal(PlacesUtils.toPRTime(bookmark.dateAdded), 1177541020000000);
+  Assert.equal(PlacesUtils.toPRTime(bookmark.lastModified), 1177541050000000);
+
+  // open test folder, and test the children
+  PlacesUtils.asQuery(folderNode);
+  Assert.equal(folderNode.hasChildren, true);
+  folderNode.containerOpen = true;
+  Assert.equal(folderNode.childCount, 1);
+
+  let bookmarkNode = folderNode.getChild(0);
+  Assert.equal("http://test/post", bookmarkNode.uri);
+  Assert.equal("test post keyword", bookmarkNode.title);
+
+  let entry = await PlacesUtils.keywords.fetch({ url: bookmarkNode.uri });
+  Assert.equal("test", entry.keyword);
+  Assert.equal("hidden1%3Dbar&text1%3D%25s", entry.postData);
+
+  Assert.equal(bookmarkNode.dateAdded, 1177375336000000);
+  Assert.equal(bookmarkNode.lastModified, 1177375423000000);
+
+  let pageInfo = await PlacesUtils.history.fetch(bookmarkNode.uri, {
+    includeAnnotations: true,
+  });
+  Assert.equal(
+    pageInfo.annotations.get(PlacesUtils.CHARSET_ANNO),
+    "ISO-8859-1",
+    "Should have the correct charset"
+  );
+
+  // clean up
+  folderNode.containerOpen = false;
+  root.containerOpen = false;
+
+  // BOOKMARKS TOOLBAR
+  root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.toolbarGuid).root;
+  Assert.equal(root.childCount, 3);
+
+  // cleanup
+  root.containerOpen = false;
+
+  // UNFILED BOOKMARKS
+  root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.unfiledGuid).root;
+  Assert.equal(root.childCount, 1);
+  root.containerOpen = false;
+
+  // favicons
+  await new Promise(resolve => {
+    PlacesUtils.favicons.getFaviconDataForPage(
+      uri(TEST_FAVICON_PAGE_URL),
+      (aURI, aDataLen, aData, aMimeType) => {
+        // aURI should never be null when aDataLen > 0.
+        Assert.notEqual(aURI, null);
+        // Favicon data is stored in the bookmarks file as a "data:" URI.  For
+        // simplicity, instead of converting the data we receive to a "data:" URI
+        // and comparing it, we just check the data size.
+        Assert.equal(TEST_FAVICON_DATA_SIZE, aDataLen);
+        resolve();
+      }
+    );
+  });
+};
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html_escape_entities.js b/toolkit/components/places/tests/unit/test_bookmarks_html_escape_entities.js
new file mode 100644
index 0000000000..5349d3948c
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html_escape_entities.js
@@ -0,0 +1,98 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Checks that html entities are escaped in bookmarks.html files.
+
+add_task(async function () {
+  // Removes bookmarks.html if the file already exists.
+  let HTMLFile = PathUtils.join(PathUtils.profileDir, "bookmarks.html");
+  await IOUtils.remove(HTMLFile, { ignoreAbsent: true });
+
+  let unescaped = '';
+  // Adds bookmarks and tags to the database.
+  const url = 'http://www.google.it/"/';
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url,
+    title: unescaped,
+  });
+  await PlacesUtils.keywords.insert({
+    url,
+    keyword: unescaped,
+    postData: unescaped,
+  });
+  PlacesUtils.tagging.tagURI(Services.io.newURI(url), [unescaped]);
+  await PlacesUtils.history.update({
+    url,
+    annotations: new Map([[PlacesUtils.CHARSET_ANNO, unescaped]]),
+  });
+
+  // Exports the bookmarks as a HTML file.
+  await BookmarkHTMLUtils.exportToFile(HTMLFile);
+  await PlacesUtils.bookmarks.remove(bm);
+
+  // Check there are no unescaped entities in the html file.
+  let xml = await new Promise((resolve, reject) => {
+    let xhr = new XMLHttpRequest();
+    xhr.onload = () => {
+      try {
+        resolve(xhr.responseXML);
+      } catch (e) {
+        reject(e);
+      }
+    };
+    xhr.onabort =
+      xhr.onerror =
+      xhr.ontimeout =
+        () => {
+          reject(new Error("xmlhttprequest failed"));
+        };
+    xhr.open("GET", PathUtils.toFileURI(HTMLFile));
+    xhr.responseType = "document";
+    xhr.overrideMimeType("text/html");
+    xhr.send();
+  });
+
+  let checksCount = 5;
+  for (
+    let current = xml;
+    current;
+    current =
+      current.firstChild ||
+      current.nextSibling ||
+      current.parentNode.nextSibling
+  ) {
+    switch (current.nodeType) {
+      case current.ELEMENT_NODE:
+        for (let { name, value } of current.attributes) {
+          info("Found attribute: " + name);
+          // Check tags, keyword, postData and charSet.
+          if (
+            ["tags", "last_charset", "shortcuturl", "post_data"].includes(name)
+          ) {
+            Assert.equal(
+              value,
+              unescaped,
+              `Attribute ${name} should be complete`
+            );
+            checksCount--;
+          }
+        }
+        break;
+      case current.TEXT_NODE:
+        // Check Title.
+        if (!current.data.startsWith("\n") && current.data.includes("test")) {
+          Assert.equal(
+            current.data.trim(),
+            unescaped,
+            "Text node should be complete"
+          );
+          checksCount--;
+        }
+        break;
+    }
+  }
+  Assert.equal(checksCount, 0, "All the checks ran");
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html_import_tags.js b/toolkit/components/places/tests/unit/test_bookmarks_html_import_tags.js
new file mode 100644
index 0000000000..f01048e1a5
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html_import_tags.js
@@ -0,0 +1,64 @@
+var bookmarkData = [
+  {
+    uri: uri("http://www.toastytech.com"),
+    title: "Nathan's Toasty Technology Page",
+    tags: ["technology", "personal", "retro"],
+  },
+  {
+    uri: uri("http://www.reddit.com"),
+    title: "reddit: the front page of the internet",
+    tags: ["social media", "news", "humour"],
+  },
+  {
+    uri: uri("http://www.4chan.org"),
+    title: "4chan",
+    tags: ["discussion", "imageboard", "anime"],
+  },
+];
+
+/*
+  TEST SUMMARY
+  - Add bookmarks with tags
+  - Export tagged bookmarks as HTML file
+  - Delete bookmarks
+  - Import bookmarks from HTML file
+  - Check that all bookmarks are successfully imported with tags
+*/
+
+add_task(async function test_import_tags() {
+  // Removes bookmarks.html if the file already exists.
+  let HTMLFile = PathUtils.join(PathUtils.profileDir, "bookmarks.html");
+  await IOUtils.remove(HTMLFile, { ignoreAbsent: true });
+
+  // Adds bookmarks and tags to the database.
+  let bookmarkList = new Set();
+  for (let { uri, title, tags } of bookmarkData) {
+    bookmarkList.add(
+      await PlacesUtils.bookmarks.insert({
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+        url: uri,
+        title,
+      })
+    );
+    PlacesUtils.tagging.tagURI(uri, tags);
+  }
+
+  // Exports the bookmarks as a HTML file.
+  await BookmarkHTMLUtils.exportToFile(HTMLFile);
+
+  // Deletes bookmarks and tags from the database.
+  for (let bookmark of bookmarkList) {
+    await PlacesUtils.bookmarks.remove(bookmark.guid);
+  }
+
+  // Re-imports the bookmarks from the HTML file.
+  await BookmarkHTMLUtils.importFromFile(HTMLFile, { replace: true });
+
+  // Tests to ensure that the tags are still present for each bookmark URI.
+  for (let { uri, tags } of bookmarkData) {
+    info("Test tags for " + uri.spec + ": " + tags + "\n");
+    let foundTags = PlacesUtils.tagging.getTagsForURI(uri);
+    Assert.equal(foundTags.length, tags.length);
+    Assert.ok(tags.every(tag => foundTags.includes(tag)));
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html_localized.js b/toolkit/components/places/tests/unit/test_bookmarks_html_localized.js
new file mode 100644
index 0000000000..4ef2393efb
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html_localized.js
@@ -0,0 +1,51 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { BookmarkHTMLUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/BookmarkHTMLUtils.sys.mjs"
+);
+
+add_task(async function setup_l10n() {
+  // A single localized string.
+  const mockSource = L10nFileSource.createMock(
+    "test",
+    "app",
+    ["en-US"],
+    "/localization/{locale}/",
+    [
+      {
+        path: "/localization/en-US/bookmarks_html_localized.ftl",
+        source: `
+bookmarks-html-localized-folder = Localized Folder
+bookmarks-html-localized-bookmark = Localized Bookmark
+`,
+      },
+    ]
+  );
+
+  L10nRegistry.getInstance().registerSources([mockSource]);
+});
+
+add_task(async function test_bookmarks_html_localized() {
+  let bookmarksFile = PathUtils.join(
+    do_get_cwd().path,
+    "bookmarks_html_localized.html"
+  );
+  await BookmarkHTMLUtils.importFromFile(bookmarksFile, { replace: true });
+
+  let root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.menuGuid).root;
+  Assert.equal(root.childCount, 1);
+  let folder = root.getChild(0);
+  PlacesUtils.asContainer(folder).containerOpen = true;
+  // Folder title is localized.
+  Assert.equal(folder.title, "Localized Folder");
+  Assert.equal(folder.childCount, 1);
+  let bookmark = folder.getChild(0);
+  Assert.equal(bookmark.uri, "http://www.mozilla.com/firefox/help/");
+  // Bookmark title is localized.
+  Assert.equal(bookmark.title, "Localized Bookmark");
+  folder.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_html_singleframe.js b/toolkit/components/places/tests/unit/test_bookmarks_html_singleframe.js
new file mode 100644
index 0000000000..07131feafe
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_html_singleframe.js
@@ -0,0 +1,31 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Test for bug #801450
+
+// Get Services
+const { BookmarkHTMLUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/BookmarkHTMLUtils.sys.mjs"
+);
+
+add_task(async function test_bookmarks_html_singleframe() {
+  let bookmarksFile = PathUtils.join(
+    do_get_cwd().path,
+    "bookmarks_html_singleframe.html"
+  );
+  await BookmarkHTMLUtils.importFromFile(bookmarksFile, { replace: true });
+
+  let root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.menuGuid).root;
+  Assert.equal(root.childCount, 1);
+  let folder = root.getChild(0);
+  PlacesUtils.asContainer(folder).containerOpen = true;
+  Assert.equal(folder.title, "Subtitle");
+  Assert.equal(folder.childCount, 1);
+  let bookmark = folder.getChild(0);
+  Assert.equal(bookmark.uri, "http://www.mozilla.org/");
+  Assert.equal(bookmark.title, "Mozilla");
+  folder.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_json.js b/toolkit/components/places/tests/unit/test_bookmarks_json.js
new file mode 100644
index 0000000000..19aaac7f95
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_json.js
@@ -0,0 +1,368 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { BookmarkJSONUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/BookmarkJSONUtils.sys.mjs"
+);
+
+// An object representing the contents of bookmarks.json.
+var test_bookmarks = {
+  menu: [
+    {
+      guid: "OCyeUO5uu9FF",
+      title: "Mozilla Firefox",
+      children: [
+        {
+          guid: "OCyeUO5uu9FG",
+          title: "Help and Tutorials",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/help/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          guid: "OCyeUO5uu9FH",
+          title: "Customize Firefox",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/customize/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          guid: "OCyeUO5uu9FI",
+          title: "Get Involved",
+          url: "http://en-us.www.mozilla.com/en-US/firefox/community/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          guid: "OCyeUO5uu9FJ",
+          title: "About Us",
+          url: "http://en-us.www.mozilla.com/en-US/about/",
+          icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+        },
+        {
+          guid: "QFM-QnE2ZpMz",
+          title: "Test null postData",
+          url: "http://example.com/search?q=%s&suggid=",
+        },
+      ],
+    },
+    {
+      guid: "OCyeUO5uu9FK",
+      type: Ci.nsINavHistoryResultNode.RESULT_TYPE_SEPARATOR,
+    },
+    {
+      guid: "OCyeUO5uu9FL",
+      title: "test",
+      dateAdded: 1177541020000000,
+      lastModified: 1177541050000000,
+      children: [
+        {
+          guid: "OCyeUO5uu9GX",
+          title: "test post keyword",
+          dateAdded: 1177375336000000,
+          lastModified: 1177375423000000,
+          keyword: "test",
+          postData: "hidden1%3Dbar&text1%3D%25s",
+          charset: "ISO-8859-1",
+        },
+      ],
+    },
+  ],
+  toolbar: [
+    {
+      guid: "OCyeUO5uu9FB",
+      title: "Getting Started",
+      url: "http://en-us.www.mozilla.com/en-US/firefox/central/",
+      icon: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAAK/INwWK6QAAABl0RVh0U29mdHdhcmUAQWRvYmUgSW1hZ2VSZWFkeXHJZTwAAAHWSURBVHjaYvz//z8DJQAggJiQOe/fv2fv7Oz8rays/N+VkfG/iYnJfyD/1+rVq7ffu3dPFpsBAAHEAHIBCJ85c8bN2Nj4vwsDw/8zQLwKiO8CcRoQu0DxqlWrdsHUwzBAAIGJmTNnPgYa9j8UqhFElwPxf2MIDeIrKSn9FwSJoRkAEEAM0DD4DzMAyPi/G+QKY4hh5WAXGf8PDQ0FGwJ22d27CjADAAIIrLmjo+MXA9R2kAHvGBA2wwx6B8W7od6CeQcggKCmCEL8bgwxYCbUIGTDVkHDBia+CuotgACCueD3TDQN75D4xmAvCoK9ARMHBzAw0AECiBHkAlC0Mdy7x9ABNA3obAZXIAa6iKEcGlMVQHwWyjYuL2d4v2cPg8vZswx7gHyAAAK7AOif7SAbOqCmn4Ha3AHFsIDtgPq/vLz8P4MSkJ2W9h8ggBjevXvHDo4FQUQg/kdypqCg4H8lUIACnQ/SOBMYI8bAsAJFPcj1AAEEjwVQqLpAbXmH5BJjqI0gi9DTAAgDBBCcAVLkgmQ7yKCZxpCQxqUZhAECCJ4XgMl493ug21ZD+aDAXH0WLM4A9MZPXJkJIIAwTAR5pQMalaCABQUULttBGCCAGCnNzgABBgAMJ5THwGvJLAAAAABJRU5ErkJggg==",
+    },
+    {
+      guid: "OCyeUO5uu9FR",
+      title: "Latest Headlines",
+      // This used to be a livemark, but we don't import them anymore, instead
+      // it will be imported as an empty folder, because the json format stores
+      // it like that: an empty folder with a couple annotations. Since
+      // annotations will go away, there won't be a clean way to import it as a
+      // bookmark instead.
+      // Note: date gets truncated to milliseconds, whereas the value in bookmarks.json
+      // has full microseconds.
+      dateAdded: 1361551979451000,
+      lastModified: 1361551979457000,
+    },
+  ],
+  unfiled: [
+    { guid: "OCyeUO5uu9FW", title: "Example.tld", url: "http://example.tld/" },
+    {
+      guid: "Cfkety492Afk",
+      title: "test tagged bookmark",
+      dateAdded: 1507025843703000,
+      lastModified: 1507025844703000,
+      url: "http://example.tld/tagged",
+      tags: ["foo"],
+    },
+    {
+      guid: "lOZGoFR1eXbl",
+      title: "Bookmarks Toolbar Shortcut",
+      dateAdded: 1507025843703000,
+      lastModified: 1507025844703000,
+      url: `place:parent=${PlacesUtils.bookmarks.toolbarGuid}`,
+    },
+    {
+      guid: "7yJWnBVhjRtP",
+      title: "Folder Shortcut",
+      dateAdded: 1507025843703000,
+      lastModified: 1507025844703000,
+      url: `place:parent=OCyeUO5uu9FF`,
+    },
+    {
+      guid: "vm5QXWuWc12l",
+      title: "Folder Shortcut 2",
+      dateAdded: 1507025843703000,
+      lastModified: 1507025844703000,
+      url: "place:invalidOldParentId=6123443&excludeItems=1",
+    },
+    {
+      guid: "Icg1XlIozA1D",
+      title: "Folder Shortcut 3",
+      dateAdded: 1507025843703000,
+      lastModified: 1507025844703000,
+      url: `place:parent=OCyeUO5uu9FF&parent=${PlacesUtils.bookmarks.menuGuid}`,
+    },
+  ],
+};
+
+// Exported bookmarks file pointer.
+var bookmarksExportedFile;
+
+add_task(async function test_import_bookmarks_disallowed_url() {
+  await Assert.rejects(
+    BookmarkJSONUtils.importFromURL("http://example.com/bookmarks.json"),
+    /importFromURL can only be used with/,
+    "Should reject importing from an http based url"
+  );
+  await Assert.rejects(
+    BookmarkJSONUtils.importFromURL("https://example.com/bookmarks.json"),
+    /importFromURL can only be used with/,
+    "Should reject importing from an https based url"
+  );
+});
+
+add_task(async function test_import_bookmarks_count() {
+  // Ensure the bookmarks count is correct when importing in various cases
+  await PlacesUtils.bookmarks.eraseEverything();
+  let bookmarksFile = PathUtils.join(do_get_cwd().path, "bookmarks.json");
+  bookmarksExportedFile = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.json"
+  );
+
+  let count = await BookmarkJSONUtils.importFromFile(bookmarksFile, {
+    replace: true,
+  });
+  Assert.equal(
+    count,
+    13,
+    "There should be 13 imported bookmarks when importing from an empty database"
+  );
+
+  await BookmarkJSONUtils.exportToFile(bookmarksExportedFile);
+  count = -1;
+  count = await BookmarkJSONUtils.importFromFile(bookmarksExportedFile, {
+    replace: true,
+  });
+  Assert.equal(
+    count,
+    13,
+    "There should be 13 imported bookmarks when replacing existing bookmarks"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  count = -1;
+  let bookmarksUrl = PathUtils.toFileURI(bookmarksFile);
+  count = await BookmarkJSONUtils.importFromURL(bookmarksUrl);
+  Assert.equal(
+    count,
+    13,
+    "There should be 13 imported bookmarks when importing from a URL"
+  );
+
+  // Clean up task
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_bookmarks() {
+  let bookmarksFile = PathUtils.join(do_get_cwd().path, "bookmarks.json");
+
+  await BookmarkJSONUtils.importFromFile(bookmarksFile, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+});
+
+add_task(async function test_export_bookmarks() {
+  bookmarksExportedFile = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.json"
+  );
+  await BookmarkJSONUtils.exportToFile(bookmarksExportedFile);
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_import_exported_bookmarks() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await BookmarkJSONUtils.importFromFile(bookmarksExportedFile, {
+    replace: true,
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+});
+
+add_task(async function test_import_ontop() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await BookmarkJSONUtils.importFromFile(bookmarksExportedFile, {
+    replace: true,
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await BookmarkJSONUtils.exportToFile(bookmarksExportedFile);
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await BookmarkJSONUtils.importFromFile(bookmarksExportedFile, {
+    replace: true,
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+});
+
+add_task(async function test_import_iconuri() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+
+  let bookmarksFile = PathUtils.join(
+    do_get_cwd().path,
+    "bookmarks_iconuri.json"
+  );
+
+  await BookmarkJSONUtils.importFromFile(bookmarksFile, {
+    replace: true,
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+});
+
+add_task(async function test_export_bookmarks_with_iconuri() {
+  bookmarksExportedFile = PathUtils.join(
+    PathUtils.profileDir,
+    "bookmarks.exported.json"
+  );
+  await BookmarkJSONUtils.exportToFile(bookmarksExportedFile);
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function test_import_exported_bookmarks_with_iconuri() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+  await BookmarkJSONUtils.importFromFile(bookmarksExportedFile, {
+    replace: true,
+  });
+  await PlacesTestUtils.promiseAsyncUpdates();
+  await testImportedBookmarks();
+});
+
+add_task(async function test_clean() {
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+async function testImportedBookmarks() {
+  for (let group in test_bookmarks) {
+    info("[testImportedBookmarks()] Checking group '" + group + "'");
+
+    let root = PlacesUtils.getFolderContents(
+      PlacesUtils.bookmarks[`${group}Guid`]
+    ).root;
+
+    let items = test_bookmarks[group];
+    Assert.equal(root.childCount, items.length);
+
+    for (let key in items) {
+      await checkItem(items[key], root.getChild(key));
+    }
+
+    root.containerOpen = false;
+  }
+}
+
+async function checkItem(aExpected, aNode) {
+  let bookmark = await PlacesUtils.bookmarks.fetch(aNode.bookmarkGuid);
+
+  for (let prop in aExpected) {
+    switch (prop) {
+      case "type":
+        Assert.equal(aNode.type, aExpected.type);
+        break;
+      case "title":
+        Assert.equal(aNode.title, aExpected.title);
+        break;
+      case "dateAdded":
+        Assert.equal(
+          PlacesUtils.toPRTime(bookmark.dateAdded),
+          aExpected.dateAdded
+        );
+        break;
+      case "lastModified":
+        Assert.equal(
+          PlacesUtils.toPRTime(bookmark.lastModified),
+          aExpected.lastModified
+        );
+        break;
+      case "url":
+        Assert.equal(aNode.uri, aExpected.url);
+        break;
+      case "icon":
+        let { data } = await getFaviconDataForPage(aExpected.url);
+        let base64Icon =
+          "data:image/png;base64," +
+          base64EncodeString(String.fromCharCode.apply(String, data));
+        Assert.equal(base64Icon, aExpected.icon);
+        break;
+      case "keyword": {
+        let entry = await PlacesUtils.keywords.fetch({ url: aNode.uri });
+        Assert.equal(entry.keyword, aExpected.keyword);
+        break;
+      }
+      case "guid":
+        Assert.equal(bookmark.guid, aExpected.guid);
+        break;
+      case "postData": {
+        let entry = await PlacesUtils.keywords.fetch({ url: aNode.uri });
+        Assert.equal(entry.postData, aExpected.postData);
+        break;
+      }
+      case "charset":
+        let pageInfo = await PlacesUtils.history.fetch(aNode.uri, {
+          includeAnnotations: true,
+        });
+        Assert.equal(
+          pageInfo.annotations.get(PlacesUtils.CHARSET_ANNO),
+          aExpected.charset
+        );
+        break;
+      case "children":
+        let folder = aNode.QueryInterface(Ci.nsINavHistoryContainerResultNode);
+        Assert.equal(folder.hasChildren, !!aExpected.children.length);
+        folder.containerOpen = true;
+        Assert.equal(folder.childCount, aExpected.children.length);
+
+        for (let index = 0; index < aExpected.children.length; index++) {
+          await checkItem(aExpected.children[index], folder.getChild(index));
+        }
+
+        folder.containerOpen = false;
+        break;
+      case "tags":
+        let uri = Services.io.newURI(aNode.uri);
+        Assert.deepEqual(
+          PlacesUtils.tagging.getTagsForURI(uri),
+          aExpected.tags,
+          "should have the expected tags"
+        );
+        break;
+      default:
+        throw new Error("Unknown property");
+    }
+  }
+}
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_json_corrupt.js b/toolkit/components/places/tests/unit/test_bookmarks_json_corrupt.js
new file mode 100644
index 0000000000..b31f1da5bb
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_json_corrupt.js
@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Tests for importing a corrupt json file.
+ *
+ * The corrupt json file attempts to import into:
+ *   - the menu folder:
+ *     - A bookmark with an invalid type.
+ *     - A valid bookmark.
+ *     - A bookmark with an invalid url.
+ *   - the toolbar folder:
+ *     - A bookmark with an invalid url.
+ *
+ * The menu case ensure that we strip out invalid bookmarks, but retain valid
+ * ones.
+ * The toolbar case ensures that if no valid bookmarks remain, then we do not
+ * throw an error.
+ */
+
+const { BookmarkJSONUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/BookmarkJSONUtils.sys.mjs"
+);
+
+add_task(async function test_import_bookmarks() {
+  let bookmarksFile = PathUtils.join(
+    do_get_cwd().path,
+    "bookmarks_corrupt.json"
+  );
+
+  await BookmarkJSONUtils.importFromFile(bookmarksFile, { replace: true });
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  let bookmarks = await PlacesUtils.promiseBookmarksTree(
+    PlacesUtils.bookmarks.menuGuid
+  );
+
+  Assert.equal(
+    bookmarks.children.length,
+    1,
+    "should only be one bookmark in the menu"
+  );
+  let bookmark = bookmarks.children[0];
+  Assert.equal(bookmark.guid, "OCyeUO5uu9FH", "should have correct guid");
+  Assert.equal(
+    bookmark.title,
+    "Customize Firefox",
+    "should have correct title"
+  );
+  Assert.equal(
+    bookmark.uri,
+    "http://en-us.www.mozilla.com/en-US/firefox/customize/",
+    "should have correct uri"
+  );
+
+  bookmarks = await PlacesUtils.promiseBookmarksTree(
+    PlacesUtils.bookmarks.toolbarGuid
+  );
+
+  Assert.ok(
+    !bookmarks.children,
+    "should not have any bookmarks in the toolbar"
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_bookmarks_restore_notification.js b/toolkit/components/places/tests/unit/test_bookmarks_restore_notification.js
new file mode 100644
index 0000000000..892b2d1d04
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_bookmarks_restore_notification.js
@@ -0,0 +1,319 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { BookmarkHTMLUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/BookmarkHTMLUtils.sys.mjs"
+);
+
+/**
+ * Tests the bookmarks-restore-* nsIObserver notifications after restoring
+ * bookmarks from JSON and HTML.  See bug 470314.
+ */
+
+// The topics and data passed to nsIObserver.observe() on bookmarks restore
+const NSIOBSERVER_TOPIC_BEGIN = "bookmarks-restore-begin";
+const NSIOBSERVER_TOPIC_SUCCESS = "bookmarks-restore-success";
+const NSIOBSERVER_TOPIC_FAILED = "bookmarks-restore-failed";
+const NSIOBSERVER_DATA_JSON = "json";
+const NSIOBSERVER_DATA_HTML = "html";
+const NSIOBSERVER_DATA_HTML_INIT = "html-initial";
+
+// Bookmarks are added for these URIs
+var uris = [
+  "http://example.com/1",
+  "http://example.com/2",
+  "http://example.com/3",
+  "http://example.com/4",
+  "http://example.com/5",
+];
+
+/**
+ * Adds some bookmarks for the URIs in |uris|.
+ */
+async function addBookmarks() {
+  for (let url of uris) {
+    await PlacesUtils.bookmarks.insert({
+      url,
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+    });
+    Assert.ok(await PlacesUtils.bookmarks.fetch({ url }), "Url is bookmarked");
+  }
+}
+
+/**
+ * Creates an file in the profile directory.
+ *
+ * @param  aBasename
+ *         e.g., "foo.txt" in the path /some/long/path/foo.txt
+ * @return {Promise}
+ * @resolves to an OS.File path
+ */
+async function promiseFile(aBasename) {
+  let path = PathUtils.join(PathUtils.profileDir, aBasename);
+  info("opening " + path);
+
+  await IOUtils.writeUTF8(path, "");
+  return path;
+}
+
+/**
+ * Register observers via promiseTopicObserved helper.
+ *
+ * @param  {boolean} expectSuccess pass true when expect a success notification
+ * @return {Promise[]}
+ */
+function registerObservers(expectSuccess) {
+  let promiseBegin = promiseTopicObserved(NSIOBSERVER_TOPIC_BEGIN);
+  let promiseResult;
+  if (expectSuccess) {
+    promiseResult = promiseTopicObserved(NSIOBSERVER_TOPIC_SUCCESS);
+  } else {
+    promiseResult = promiseTopicObserved(NSIOBSERVER_TOPIC_FAILED);
+  }
+
+  return [promiseBegin, promiseResult];
+}
+
+/**
+ * Check notification results.
+ *
+ * @param  {Promise[]} expectPromises array contain promiseBegin and promiseResult
+ * @param  {object} expectedData contain data and folderId
+ */
+async function checkObservers(expectPromises, expectedData) {
+  let [promiseBegin, promiseResult] = expectPromises;
+
+  let beginData = (await promiseBegin)[1];
+  Assert.equal(
+    beginData,
+    expectedData.data,
+    "Data for current test should be what is expected"
+  );
+
+  let [resultSubject, resultData] = await promiseResult;
+  Assert.equal(
+    resultData,
+    expectedData.data,
+    "Data for current test should be what is expected"
+  );
+
+  // Make sure folder ID is what is expected.  For importing HTML into a
+  // folder, this will be an integer, otherwise null.
+  if (resultSubject) {
+    Assert.equal(
+      resultSubject.QueryInterface(Ci.nsISupportsPRInt64).data,
+      expectedData.folderId
+    );
+  } else {
+    Assert.equal(expectedData.folderId, null);
+  }
+}
+
+/**
+ * Run after every test cases.
+ */
+async function teardown(file, begin, success, fail) {
+  // On restore failed, file may not exist, so wrap in try-catch.
+  await IOUtils.remove(file, { ignoreAbsent: true });
+
+  // clean up bookmarks
+  await PlacesUtils.bookmarks.eraseEverything();
+}
+
+add_task(async function test_json_restore_normal() {
+  // data: the data passed to nsIObserver.observe() corresponding to the test
+  // folderId: for HTML restore into a folder, the folder ID to restore into;
+  //           otherwise, set it to null
+  let expectedData = {
+    data: NSIOBSERVER_DATA_JSON,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(true);
+
+  info("JSON restore: normal restore should succeed");
+  let file = await promiseFile("bookmarks-test_restoreNotification.json");
+  await addBookmarks();
+
+  await BookmarkJSONUtils.exportToFile(file);
+  await PlacesUtils.bookmarks.eraseEverything();
+  try {
+    await BookmarkJSONUtils.importFromFile(file, { replace: true });
+  } catch (e) {
+    do_throw("  Restore should not have failed " + e);
+  }
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_json_restore_empty() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_JSON,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(false);
+
+  info("JSON restore: empty file should fail");
+  let file = await promiseFile("bookmarks-test_restoreNotification.json");
+  await Assert.rejects(
+    BookmarkJSONUtils.importFromFile(file, { replace: true }),
+    /SyntaxError/,
+    "Restore should reject for an empty file."
+  );
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_json_restore_nonexist() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_JSON,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(false);
+
+  info("JSON restore: nonexistent file should fail");
+  let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  file.append("this file doesn't exist because nobody created it 1");
+  await Assert.rejects(
+    BookmarkJSONUtils.importFromFile(file.path, { replace: true }),
+    /Cannot restore from nonexisting json file/,
+    "Restore should reject for a non-existent file."
+  );
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file.path);
+});
+
+add_task(async function test_html_restore_normal() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(true);
+
+  info("HTML restore: normal restore should succeed");
+  let file = await promiseFile("bookmarks-test_restoreNotification.html");
+  await addBookmarks();
+  await BookmarkHTMLUtils.exportToFile(file);
+  await PlacesUtils.bookmarks.eraseEverything();
+  try {
+    BookmarkHTMLUtils.importFromFile(file).catch(
+      do_report_unexpected_exception
+    );
+  } catch (e) {
+    do_throw("  Restore should not have failed");
+  }
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_html_restore_empty() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(true);
+
+  info("HTML restore: empty file should succeed");
+  let file = await promiseFile("bookmarks-test_restoreNotification.init.html");
+  try {
+    BookmarkHTMLUtils.importFromFile(file).catch(
+      do_report_unexpected_exception
+    );
+  } catch (e) {
+    do_throw("  Restore should not have failed");
+  }
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_html_restore_nonexist() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(false);
+
+  info("HTML restore: nonexistent file should fail");
+  let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  file.append("this file doesn't exist because nobody created it 2");
+  await Assert.rejects(
+    BookmarkHTMLUtils.importFromFile(file.path),
+    /Cannot import from nonexisting html file/,
+    "Restore should reject for a non-existent file."
+  );
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file.path);
+});
+
+add_task(async function test_html_init_restore_normal() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML_INIT,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(true);
+
+  info("HTML initial restore: normal restore should succeed");
+  let file = await promiseFile("bookmarks-test_restoreNotification.init.html");
+  await addBookmarks();
+  await BookmarkHTMLUtils.exportToFile(file);
+  await PlacesUtils.bookmarks.eraseEverything();
+  try {
+    BookmarkHTMLUtils.importFromFile(file, { replace: true }).catch(
+      do_report_unexpected_exception
+    );
+  } catch (e) {
+    do_throw("  Restore should not have failed");
+  }
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_html_init_restore_empty() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML_INIT,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(true);
+
+  info("HTML initial restore: empty file should succeed");
+  let file = await promiseFile("bookmarks-test_restoreNotification.init.html");
+  try {
+    BookmarkHTMLUtils.importFromFile(file, { replace: true }).catch(
+      do_report_unexpected_exception
+    );
+  } catch (e) {
+    do_throw("  Restore should not have failed");
+  }
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file);
+});
+
+add_task(async function test_html_init_restore_nonexist() {
+  let expectedData = {
+    data: NSIOBSERVER_DATA_HTML_INIT,
+    folderId: null,
+  };
+  let expectPromises = registerObservers(false);
+
+  info("HTML initial restore: nonexistent file should fail");
+  let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  file.append("this file doesn't exist because nobody created it 3");
+  await Assert.rejects(
+    BookmarkHTMLUtils.importFromFile(file.path, { replace: true }),
+    /Cannot import from nonexisting html file/,
+    "Restore should reject for a non-existent file."
+  );
+
+  await checkObservers(expectPromises, expectedData);
+  await teardown(file.path);
+});
diff --git a/toolkit/components/places/tests/unit/test_broken_folderShortcut_result.js b/toolkit/components/places/tests/unit/test_broken_folderShortcut_result.js
new file mode 100644
index 0000000000..1e1f7bc9c7
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_broken_folderShortcut_result.js
@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_brokenFolderShortcut() {
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        url: "http://1.moz.org/",
+        title: "Bookmark 1",
+      },
+      {
+        url: "place:parent=1234",
+        title: "Shortcut 1",
+      },
+      {
+        url: "place:parent=-1",
+        title: "Shortcut 2",
+      },
+      {
+        url: "http://2.moz.org/",
+        title: "Bookmark 2",
+      },
+    ],
+  });
+
+  // Add also a simple visit.
+  await PlacesTestUtils.addVisits(uri("http://3.moz.org/"));
+
+  // Query containing a broken folder shortcuts among results.
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.unfiledGuid]);
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  Assert.equal(root.childCount, 4);
+
+  let shortcut = root.getChild(1);
+  Assert.equal(shortcut.uri, "place:parent=1234");
+  PlacesUtils.asContainer(shortcut);
+  shortcut.containerOpen = true;
+  Assert.equal(shortcut.childCount, 0);
+  shortcut.containerOpen = false;
+  // Remove the broken shortcut while the containing result is open.
+  await PlacesUtils.bookmarks.remove(bookmarks[1]);
+  Assert.equal(root.childCount, 3);
+
+  shortcut = root.getChild(1);
+  Assert.equal(shortcut.uri, "place:parent=-1");
+  PlacesUtils.asContainer(shortcut);
+  shortcut.containerOpen = true;
+  Assert.equal(shortcut.childCount, 0);
+  shortcut.containerOpen = false;
+  // Remove the broken shortcut while the containing result is open.
+  await PlacesUtils.bookmarks.remove(bookmarks[2]);
+  Assert.equal(root.childCount, 2);
+
+  root.containerOpen = false;
+
+  // Broken folder shortcut as root node.
+  query = PlacesUtils.history.getNewQuery();
+  query.setParents([1234]);
+  options = PlacesUtils.history.getNewQueryOptions();
+  root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+
+  // Broken folder shortcut as root node with folder=-1.
+  query = PlacesUtils.history.getNewQuery();
+  query.setParents([-1]);
+  options = PlacesUtils.history.getNewQueryOptions();
+  root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 0);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_browserhistory.js b/toolkit/components/places/tests/unit/test_browserhistory.js
new file mode 100644
index 0000000000..f737262ae7
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_browserhistory.js
@@ -0,0 +1,122 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const TEST_URI = "http://mozilla.com/";
+const TEST_SUBDOMAIN_URI = "http://foobar.mozilla.com/";
+
+async function checkEmptyHistory() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.executeCached("SELECT count(*) FROM moz_historyvisits");
+  return !rows[0].getResultByIndex(0);
+}
+
+add_task(async function test_addPage() {
+  await PlacesTestUtils.addVisits(TEST_URI);
+  Assert.ok(!(await checkEmptyHistory()), "History has entries");
+});
+
+add_task(async function test_removePage() {
+  await PlacesUtils.history.remove(TEST_URI);
+  Assert.ok(await checkEmptyHistory(), "History is empty");
+});
+
+add_task(async function test_removePages() {
+  let pages = [];
+  for (let i = 0; i < 8; i++) {
+    pages.push(TEST_URI + i);
+  }
+
+  await PlacesTestUtils.addVisits(pages.map(uri => ({ uri })));
+  // Bookmarked item should not be removed from moz_places.
+  const ANNO_INDEX = 1;
+  const ANNO_NAME = "testAnno";
+  const ANNO_VALUE = "foo";
+  const BOOKMARK_INDEX = 2;
+  await PlacesUtils.history.update({
+    url: pages[ANNO_INDEX],
+    annotations: new Map([[ANNO_NAME, ANNO_VALUE]]),
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: pages[BOOKMARK_INDEX],
+    title: "test bookmark",
+  });
+  await PlacesUtils.history.update({
+    url: pages[BOOKMARK_INDEX],
+    annotations: new Map([[ANNO_NAME, ANNO_VALUE]]),
+  });
+
+  await PlacesUtils.history.remove(pages);
+  Assert.ok(await checkEmptyHistory(), "History is empty");
+
+  // Check that the bookmark and its annotation still exist.
+  let folder = await PlacesUtils.getFolderContents(
+    PlacesUtils.bookmarks.unfiledGuid
+  );
+  Assert.equal(folder.root.childCount, 1);
+  let pageInfo = await PlacesUtils.history.fetch(pages[BOOKMARK_INDEX], {
+    includeAnnotations: true,
+  });
+  Assert.equal(pageInfo.annotations.get(ANNO_NAME), ANNO_VALUE);
+
+  // Check the annotation on the non-bookmarked page does not exist anymore.
+  await assertNoOrphanPageAnnotations();
+
+  // Cleanup.
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_removePagesByTimeframe() {
+  let visits = [];
+  let startDate = (Date.now() - 10000) * 1000;
+  for (let i = 0; i < 10; i++) {
+    visits.push({
+      uri: TEST_URI + i,
+      visitDate: startDate + i * 1000,
+    });
+  }
+
+  await PlacesTestUtils.addVisits(visits);
+
+  // Delete all pages except the first and the last.
+  await PlacesUtils.history.removeByFilter({
+    beginDate: PlacesUtils.toDate(startDate + 1000),
+    endDate: PlacesUtils.toDate(startDate + 8000),
+  });
+
+  // Check that we have removed the correct pages.
+  for (let i = 0; i < 10; i++) {
+    Assert.equal(page_in_database(TEST_URI + i) == 0, i > 0 && i < 9);
+  }
+
+  // Clear remaining items and check that all pages have been removed.
+  await PlacesUtils.history.removeByFilter({
+    beginDate: PlacesUtils.toDate(startDate),
+    endDate: PlacesUtils.toDate(startDate + 9000),
+  });
+  Assert.ok(await checkEmptyHistory(), "History is empty");
+});
+
+add_task(async function test_removePagesFromHost() {
+  await PlacesTestUtils.addVisits(TEST_URI);
+  await PlacesUtils.history.removeByFilter({ host: ".mozilla.com" });
+  Assert.ok(await checkEmptyHistory(), "History is empty");
+});
+
+add_task(async function test_removePagesFromHost_keepSubdomains() {
+  await PlacesTestUtils.addVisits([
+    { uri: TEST_URI },
+    { uri: TEST_SUBDOMAIN_URI },
+  ]);
+  await PlacesUtils.history.removeByFilter({ host: "mozilla.com" });
+  Assert.ok(!(await checkEmptyHistory()), "History has entries");
+});
+
+add_task(async function test_history_clear() {
+  await PlacesUtils.history.clear();
+  Assert.ok(await checkEmptyHistory(), "History is empty");
+});
diff --git a/toolkit/components/places/tests/unit/test_childlessTags.js b/toolkit/components/places/tests/unit/test_childlessTags.js
new file mode 100644
index 0000000000..9f489a266b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_childlessTags.js
@@ -0,0 +1,140 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Ensures that removal of a bookmark untags the bookmark if it's no longer
+ * contained in any regular, non-tag folders.  See bug 444849.
+ */
+
+var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+  Ci.nsINavHistoryService
+);
+
+var tagssvc = Cc["@mozilla.org/browser/tagging-service;1"].getService(
+  Ci.nsITaggingService
+);
+
+const BOOKMARK_URI = uri("http://example.com/");
+
+add_task(async function test_removing_tagged_bookmark_removes_tag() {
+  print("  Make a bookmark.");
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: BOOKMARK_URI,
+    title: "test bookmark",
+  });
+
+  print("  Tag it up.");
+  let tags = ["foo", "bar"];
+  tagssvc.tagURI(BOOKMARK_URI, tags);
+  ensureTagsExist(tags);
+  let root = getTagRoot();
+  root.containerOpen = true;
+  let oldCount = root.childCount;
+  root.containerOpen = false;
+
+  print("  Remove the bookmark.  The tags should no longer exist.");
+  let wait = TestUtils.waitForCondition(() => {
+    root = getTagRoot();
+    root.containerOpen = true;
+    let val = root.childCount == oldCount - 2;
+    root.containerOpen = false;
+    return val;
+  });
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+  await wait;
+  ensureTagsExist([]);
+});
+
+add_task(
+  async function test_removing_folder_containing_tagged_bookmark_removes_tag() {
+    print("  Make a folder.");
+    let folder = await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      title: "test folder",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    });
+
+    print("  Stick a bookmark in the folder.");
+    var bookmark = await PlacesUtils.bookmarks.insert({
+      parentGuid: folder.guid,
+      url: BOOKMARK_URI,
+      title: "test bookmark",
+    });
+
+    print("  Tag the bookmark.");
+    var tags = ["foo", "bar"];
+    tagssvc.tagURI(BOOKMARK_URI, tags);
+    ensureTagsExist(tags);
+
+    // The tag containers are removed in async and take some time
+    let oldCountFoo = await tagCount("foo");
+    let oldCountBar = await tagCount("bar");
+
+    print("  Remove the folder.  The tags should no longer exist.");
+
+    let wait = TestUtils.waitForCondition(async () => {
+      let newCountFoo = await tagCount("foo");
+      let newCountBar = await tagCount("bar");
+      return newCountFoo == oldCountFoo - 1 && newCountBar == oldCountBar - 1;
+    });
+    await PlacesUtils.bookmarks.remove(bookmark.guid);
+    await wait;
+    ensureTagsExist([]);
+  }
+);
+
+async function tagCount(aTag) {
+  let allTags = await PlacesUtils.bookmarks.fetchTags();
+  for (let i of allTags) {
+    if (i.name == aTag) {
+      return i.count;
+    }
+  }
+  return 0;
+}
+
+function getTagRoot() {
+  var query = histsvc.getNewQuery();
+  var opts = histsvc.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_TAGS_ROOT;
+  var resultRoot = histsvc.executeQuery(query, opts).root;
+  return resultRoot;
+}
+/**
+ * Runs a tag query and ensures that the tags returned are those and only those
+ * in aTags.  aTags may be empty, in which case this function ensures that no
+ * tags exist.
+ *
+ * @param aTags
+ *        An array of tags (strings)
+ */
+function ensureTagsExist(aTags) {
+  var query = histsvc.getNewQuery();
+  var opts = histsvc.getNewQueryOptions();
+  opts.resultType = opts.RESULTS_AS_TAGS_ROOT;
+  var resultRoot = histsvc.executeQuery(query, opts).root;
+
+  // Dupe aTags.
+  var tags = aTags.slice(0);
+
+  resultRoot.containerOpen = true;
+
+  // Ensure that the number of tags returned from the query is the same as the
+  // number in |tags|.
+  Assert.equal(resultRoot.childCount, tags.length);
+
+  // For each tag result from the query, ensure that it's contained in |tags|.
+  // Remove the tag from |tags| so that we ensure the sets are equal.
+  for (let i = 0; i < resultRoot.childCount; i++) {
+    var tag = resultRoot.getChild(i).title;
+    var indexOfTag = tags.indexOf(tag);
+    Assert.ok(indexOfTag >= 0);
+    tags.splice(indexOfTag, 1);
+  }
+
+  resultRoot.containerOpen = false;
+}
diff --git a/toolkit/components/places/tests/unit/test_frecency_decay.js b/toolkit/components/places/tests/unit/test_frecency_decay.js
new file mode 100644
index 0000000000..8fbb08aecc
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_decay.js
@@ -0,0 +1,82 @@
+/* Any copyright is dedicated to the Public Domain.
+ * https://creativecommons.org/publicdomain/zero/1.0/ */
+
+const PREF_FREC_DECAY_RATE_DEF = 0.975;
+
+/**
+ * Promises that the pages-rank-changed event has been seen.
+ *
+ * @returns {Promise} A promise which is resolved when the notification is seen.
+ */
+function promiseRankingChanged() {
+  return PlacesTestUtils.waitForNotification("pages-rank-changed");
+}
+
+add_task(async function setup() {
+  Services.prefs.setCharPref(
+    "places.frecency.decayRate",
+    PREF_FREC_DECAY_RATE_DEF
+  );
+});
+
+add_task(async function test_isFrecencyDecaying() {
+  let db = await PlacesUtils.promiseDBConnection();
+  async function queryFrecencyDecaying() {
+    return (
+      await db.executeCached(`SELECT is_frecency_decaying()`)
+    )[0].getResultByIndex(0);
+  }
+  PlacesUtils.history.isFrecencyDecaying = true;
+  Assert.equal(PlacesUtils.history.isFrecencyDecaying, true);
+  Assert.equal(await queryFrecencyDecaying(), true);
+  PlacesUtils.history.isFrecencyDecaying = false;
+  Assert.equal(PlacesUtils.history.isFrecencyDecaying, false);
+  Assert.equal(await queryFrecencyDecaying(), false);
+});
+
+add_task(async function test_frecency_decay() {
+  let unvisitedBookmarkFrecency = Services.prefs.getIntPref(
+    "places.frecency.unvisitedBookmarkBonus"
+  );
+
+  // Add a bookmark and check its frecency.
+  let url = "http://example.com/b";
+  let promiseOne = promiseRankingChanged();
+  await PlacesUtils.bookmarks.insert({
+    url,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await promiseOne;
+
+  let histogram = TelemetryTestUtils.getAndClearHistogram(
+    "PLACES_IDLE_FRECENCY_DECAY_TIME_MS"
+  );
+  info("Trigger frecency decay.");
+  Assert.equal(PlacesUtils.history.isFrecencyDecaying, false);
+  let promiseRanking = promiseRankingChanged();
+
+  PlacesFrecencyRecalculator.observe(null, "idle-daily", "");
+  Assert.equal(PlacesUtils.history.isFrecencyDecaying, true);
+  info("Wait for completion.");
+  await PlacesFrecencyRecalculator.pendingFrecencyDecayPromise;
+
+  await promiseRanking;
+  Assert.equal(PlacesUtils.history.isFrecencyDecaying, false);
+
+  // Now check the new frecency is correct.
+  let newFrecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    { url }
+  );
+
+  Assert.equal(
+    newFrecency,
+    Math.round(unvisitedBookmarkFrecency * PREF_FREC_DECAY_RATE_DEF),
+    "Frecencies should match"
+  );
+
+  let snapshot = histogram.snapshot();
+  Assert.greater(snapshot.sum, 0);
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_observers.js b/toolkit/components/places/tests/unit/test_frecency_observers.js
new file mode 100644
index 0000000000..44747b06f9
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_observers.js
@@ -0,0 +1,99 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Each of these tests a path that triggers a frecency update.  Together they
+// hit all sites that update a frecency.
+
+// InsertVisitedURIs::UpdateFrecency and History::InsertPlace
+add_task(
+  async function test_InsertVisitedURIs_UpdateFrecency_and_History_InsertPlace() {
+    // InsertPlace is at the end of a path that UpdateFrecency is also on, so kill
+    // two birds with one stone and expect two notifications.  Trigger the path by
+    // adding a download.
+    let url = Services.io.newURI("http://example.com/a");
+    let promise = onRankingChanged();
+    await PlacesUtils.history.insert({
+      url,
+      visits: [
+        {
+          transition: PlacesUtils.history.TRANSITIONS.DOWNLOAD,
+        },
+      ],
+    });
+    await promise;
+  }
+);
+
+// nsNavHistory::UpdateFrecency
+add_task(async function test_nsNavHistory_UpdateFrecency() {
+  let url = Services.io.newURI("http://example.com/b");
+  let promise = onRankingChanged();
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url,
+    title: "test",
+  });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await promise;
+});
+
+// History.jsm invalidateFrecencies()
+add_task(async function test_invalidateFrecencies() {
+  let url = Services.io.newURI("http://test-invalidateFrecencies.com/");
+  // Bookmarking the URI is enough to add it to moz_places, and importantly, it
+  // means that removeByFilter doesn't remove it from moz_places, so its
+  // frecency is able to be changed.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url,
+    title: "test",
+  });
+  let promise = onRankingChanged();
+  await PlacesUtils.history.removeByFilter({ host: url.host });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await promise;
+});
+
+// History.jsm clear() should not cause a frecency recalculation since pages
+// are removed.
+add_task(async function test_clear() {
+  let received = [];
+  let listener = events =>
+    (received = received.concat(events.map(e => e.type)));
+  PlacesObservers.addListener(
+    ["history-cleared", "pages-rank-changed"],
+    listener
+  );
+  await PlacesUtils.history.clear();
+  PlacesObservers.removeListener(
+    ["history-cleared", "pages-rank-changed"],
+    listener
+  );
+  Assert.deepEqual(received, ["history-cleared"]);
+});
+
+add_task(async function test_nsNavHistory_idleDaily() {
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "https://test-site1.org",
+    title: "test",
+  });
+  PlacesFrecencyRecalculator.observe(null, "idle-daily", "");
+  await Promise.all([onRankingChanged()]);
+});
+
+add_task(async function test_nsNavHistory_recalculate() {
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "https://test-site1.org",
+    title: "test",
+  });
+  await Promise.all([
+    onRankingChanged(),
+    PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies(),
+  ]);
+});
+
+function onRankingChanged() {
+  return PlacesTestUtils.waitForNotification("pages-rank-changed");
+}
diff --git a/toolkit/components/places/tests/unit/test_frecency_origins_alternative.js b/toolkit/components/places/tests/unit/test_frecency_origins_alternative.js
new file mode 100644
index 0000000000..5519149cac
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_origins_alternative.js
@@ -0,0 +1,301 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests alternative origins frecency.
+// Note: the order of the tests here matters, since we are emulating subsquent
+// starts of the recalculator component with different initial conditions.
+
+const FEATURE_PREF = "places.frecency.origins.alternative.featureGate";
+
+async function restartRecalculator() {
+  let subject = {};
+  PlacesFrecencyRecalculator.observe(
+    subject,
+    "test-alternative-frecency-init",
+    ""
+  );
+  await subject.promise;
+}
+
+async function getAllOrigins() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`SELECT * FROM moz_origins`);
+  Assert.greater(rows.length, 0);
+  return rows.map(r => ({
+    host: r.getResultByName("host"),
+    frecency: r.getResultByName("frecency"),
+    recalc_frecency: r.getResultByName("recalc_frecency"),
+    alt_frecency: r.getResultByName("alt_frecency"),
+    recalc_alt_frecency: r.getResultByName("recalc_alt_frecency"),
+  }));
+}
+
+add_setup(async function () {
+  await PlacesTestUtils.addVisits([
+    "https://testdomain1.moz.org",
+    "https://testdomain2.moz.org",
+    "https://testdomain3.moz.org",
+  ]);
+  registerCleanupFunction(PlacesUtils.history.clear);
+});
+
+add_task(async function test_normal_init() {
+  // Ensure moz_meta doesn't report anything.
+  Assert.ok(
+    !PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.enabled,
+    "Check the pref is disabled by default"
+  );
+  // Avoid hitting the cache, we want to check the actual database value.
+  PlacesUtils.metadata.cache.clear();
+  Assert.ok(
+    ObjectUtils.isEmpty(
+      await PlacesUtils.metadata.get(
+        PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+        Object.create(null)
+      )
+    ),
+    "Check there's no variables stored"
+  );
+});
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_enable_init() {
+    // Set alt_frecency to NULL and recalc_alt_frecency = 0 for the entries in
+    // moz_origins to verify they are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_origins", {
+      alt_frecency: null,
+      recalc_alt_frecency: 0,
+    });
+
+    await restartRecalculator();
+
+    // Ensure moz_meta doesn't report anything.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.equal(
+      (
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins
+            .metadataKey,
+          Object.create(null)
+        )
+      ).version,
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.variables
+        .version,
+      "Check the algorithm version has been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    let origins = await getAllOrigins();
+    Assert.ok(
+      origins.every(o => o.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      origins.every(o => o.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_different_version() {
+    let origins = await getAllOrigins();
+    Assert.ok(
+      origins.every(o => o.recalc_alt_frecency == 0),
+      "All the entries should not need recalculation"
+    );
+
+    // Set alt_frecency to NULL for the entries in moz_origins to verify they
+    // are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_origins", {
+      alt_frecency: null,
+    });
+
+    // It doesn't matter that the version is, it just have to be different.
+    let variables = await PlacesUtils.metadata.get(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+      Object.create(null)
+    );
+    variables.version = 999;
+    await PlacesUtils.metadata.set(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+      variables
+    );
+
+    await restartRecalculator();
+
+    // Check alternative frecency has been marked for recalculation.
+    // Note just after init we reculate a chunk, and this test code is expected
+    // to run before that... though we can't be sure, so if this starts failing
+    // intermittently we'll have to add more synchronization test code.
+    origins = await getAllOrigins();
+    // Ensure moz_meta has been updated.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.deepEqual(
+      (
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins
+            .metadataKey,
+          Object.create(null)
+        )
+      ).version,
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.variables
+        .version,
+      "Check the algorithm version has been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    origins = await getAllOrigins();
+    Assert.ok(
+      origins.every(o => o.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      origins.every(o => o.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_different_variables() {
+    let origins = await getAllOrigins();
+    Assert.ok(
+      origins.every(o => o.recalc_alt_frecency == 0),
+      "All the entries should not need recalculation"
+    );
+
+    // Set alt_frecency to NULL for the entries in moz_origins to verify they
+    // are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_origins", {
+      alt_frecency: null,
+    });
+
+    // Change variables.
+    let variables = await PlacesUtils.metadata.get(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+      Object.create(null)
+    );
+    Assert.greater(Object.keys(variables).length, 1);
+    Assert.ok("version" in variables, "At least the version is always present");
+    await PlacesUtils.metadata.set(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+      {
+        version:
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.variables
+            .version,
+        someVar: 1,
+      }
+    );
+
+    await restartRecalculator();
+
+    // Ensure moz_meta has been updated.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.deepEqual(
+      await PlacesUtils.metadata.get(
+        PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+        Object.create(null)
+      ),
+      variables,
+      "Check the algorithm variables have been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    origins = await getAllOrigins();
+    Assert.ok(
+      origins.every(o => o.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      origins.every(o => o.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(async function test_disable() {
+  let origins = await getAllOrigins();
+  Assert.ok(
+    origins.every(o => o.recalc_alt_frecency == 0),
+    "All the entries should not need recalculation"
+  );
+  // Avoid hitting the cache, we want to check the actual database value.
+  PlacesUtils.metadata.cache.clear();
+  Assert.equal(
+    (
+      await PlacesUtils.metadata.get(
+        PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+        Object.create(null)
+      )
+    ).version,
+    PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.variables
+      .version,
+    "Check the algorithm version has been stored"
+  );
+
+  await restartRecalculator();
+
+  // Check alternative frecency has not been marked for recalculation.
+  origins = await getAllOrigins();
+  Assert.ok(
+    origins.every(o => o.recalc_alt_frecency == 0),
+    "The entries not have been marked for recalc"
+  );
+  Assert.ok(
+    origins.every(o => o.alt_frecency === null),
+    "All the alt_frecency values should have been nullified"
+  );
+
+  // Ensure moz_meta has been updated.
+  // Avoid hitting the cache, we want to check the actual database value.
+  PlacesUtils.metadata.cache.clear();
+  Assert.ok(
+    ObjectUtils.isEmpty(
+      await PlacesUtils.metadata.get(
+        PlacesFrecencyRecalculator.alternativeFrecencyInfo.origins.metadataKey,
+        Object.create(null)
+      )
+    ),
+    "Check the algorithm variables has been removed"
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_origins_recalc.js b/toolkit/components/places/tests/unit/test_frecency_origins_recalc.js
new file mode 100644
index 0000000000..b96f9ecb58
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_origins_recalc.js
@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that recalc_frecency in the moz_origins table works is consistent.
+
+// This test does not completely cover origins frecency recalculation because
+// the current system uses temp tables and triggers to make the recalculation,
+// but it's likely that will change in the future and then we can add to this.
+
+add_task(async function test() {
+  // test recalc_frecency is set to 1 when frecency of a page changes.
+  // Add a couple visits, then remove one of them.
+  const now = new Date();
+  const url = "https://mozilla.org/test/";
+  await PlacesTestUtils.addVisits([
+    {
+      url,
+      visitDate: now,
+    },
+    {
+      url,
+      visitDate: new Date(new Date().setDate(now.getDate() - 30)),
+    },
+  ]);
+  // TODO: use PlacesTestUtils.getDatabaseValue once available.
+  let db = await PlacesUtils.promiseDBConnection();
+  Assert.equal(
+    (
+      await db.execute(`SELECT recalc_frecency FROM moz_origins`)
+    )[0].getResultByIndex(0),
+    0,
+    "Should have been calculated already"
+  );
+  Assert.equal(
+    (
+      await db.execute(`SELECT recalc_alt_frecency FROM moz_origins`)
+    )[0].getResultByIndex(0),
+    1,
+    "Should not have been calculated"
+  );
+
+  // Remove only one visit (otherwise the page would be orphaned).
+  await PlacesUtils.history.removeVisitsByFilter({
+    beginDate: new Date(now.valueOf() - 10000),
+    endDate: new Date(now.valueOf() + 10000),
+  });
+  Assert.equal(
+    (
+      await db.execute(`SELECT recalc_frecency FROM moz_origins`)
+    )[0].getResultByIndex(0),
+    0,
+    "Should have been calculated already"
+  );
+  Assert.equal(
+    (
+      await db.execute(`SELECT recalc_alt_frecency FROM moz_origins`)
+    )[0].getResultByIndex(0),
+    1,
+    "Should not have been calculated yet"
+  );
+  // test recalc_frecency is set back to 0 when frecency of the origin is set
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_pages_alternative.js b/toolkit/components/places/tests/unit/test_frecency_pages_alternative.js
new file mode 100644
index 0000000000..7fc21d9eb3
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_pages_alternative.js
@@ -0,0 +1,352 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests alternative pages frecency.
+// Note: the order of the tests here matters, since we are emulating subsquent
+// starts of the recalculator component with different initial conditions.
+
+const FEATURE_PREF = "places.frecency.pages.alternative.featureGate";
+
+async function restartRecalculator() {
+  let subject = {};
+  PlacesFrecencyRecalculator.observe(
+    subject,
+    "test-alternative-frecency-init",
+    ""
+  );
+  await subject.promise;
+}
+
+async function getAllPages() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`SELECT * FROM moz_places`);
+  Assert.greater(rows.length, 0);
+  return rows.map(r => ({
+    url: r.getResultByName("url"),
+    frecency: r.getResultByName("frecency"),
+    recalc_frecency: r.getResultByName("recalc_frecency"),
+    alt_frecency: r.getResultByName("alt_frecency"),
+    recalc_alt_frecency: r.getResultByName("recalc_alt_frecency"),
+  }));
+}
+
+add_setup(async function () {
+  await PlacesTestUtils.addVisits([
+    "https://testdomain1.moz.org",
+    "https://testdomain2.moz.org",
+    "https://testdomain3.moz.org",
+  ]);
+  registerCleanupFunction(PlacesUtils.history.clear);
+});
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, false]],
+  },
+  async function test_normal_init() {
+    // The test starts with the pref enabled, otherwise we'd not have the SQL
+    // function defined. So here we disable it, then enable again later.
+    await restartRecalculator();
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.ok(
+      ObjectUtils.isEmpty(
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+          Object.create(null)
+        )
+      ),
+      "Check there's no variables stored"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_enable_init() {
+    // Set alt_frecency to NULL and recalc_alt_frecency = 0 for the entries in
+    // moz_places to verify they are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_places", {
+      alt_frecency: null,
+      recalc_alt_frecency: 0,
+    });
+
+    await restartRecalculator();
+
+    // Ensure moz_meta doesn't report anything.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.equal(
+      (
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+          Object.create(null)
+        )
+      ).version,
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.variables
+        .version,
+      "Check the algorithm version has been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    let pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      pages.every(p => p.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_different_version() {
+    let pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries should not need recalculation"
+    );
+
+    // Set alt_frecency to NULL to verify all the entries are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_places", {
+      alt_frecency: null,
+    });
+
+    // It doesn't matter that the version is, it just have to be different.
+    let variables = await PlacesUtils.metadata.get(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+      Object.create(null)
+    );
+    variables.version = 999;
+    await PlacesUtils.metadata.set(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+      variables
+    );
+
+    await restartRecalculator();
+
+    // Check alternative frecency has been marked for recalculation.
+    // Note just after init we reculate a chunk, and this test code is expected
+    // to run before that... though we can't be sure, so if this starts failing
+    // intermittently we'll have to add more synchronization test code.
+    pages = await getAllPages();
+    // Ensure moz_meta has been updated.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.deepEqual(
+      (
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+          Object.create(null)
+        )
+      ).version,
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.variables
+        .version,
+      "Check the algorithm version has been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      pages.every(p => p.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_different_variables() {
+    let pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries should not need recalculation"
+    );
+
+    // Set alt_frecency to NULL to verify all the entries are recalculated.
+    await PlacesTestUtils.updateDatabaseValues("moz_places", {
+      alt_frecency: null,
+    });
+
+    // Change variables.
+    let variables = await PlacesUtils.metadata.get(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+      Object.create(null)
+    );
+    Assert.greater(Object.keys(variables).length, 1);
+    Assert.ok("version" in variables, "At least the version is always present");
+    await PlacesUtils.metadata.set(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+      {
+        version:
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.variables
+            .version,
+        someVar: 1,
+      }
+    );
+
+    await restartRecalculator();
+
+    // Ensure moz_meta has been updated.
+    Assert.ok(
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.enabled,
+      "Check the pref is enabled"
+    );
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.deepEqual(
+      await PlacesUtils.metadata.get(
+        PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+        Object.create(null)
+      ),
+      variables,
+      "Check the algorithm variables have been stored"
+    );
+
+    // Check all alternative frecencies have been calculated, since we just have
+    // a few.
+    pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      pages.every(p => p.alt_frecency > 0),
+      "All the entries have been recalculated"
+    );
+    Assert.ok(
+      PlacesFrecencyRecalculator.isRecalculationPending,
+      "Recalculation should be pending"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, false]],
+  },
+  async function test_disable() {
+    let pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "All the entries should not need recalculation"
+    );
+
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.equal(
+      (
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+          Object.create(null)
+        )
+      ).version,
+      PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.variables
+        .version,
+      "Check the algorithm version has been stored"
+    );
+
+    await restartRecalculator();
+
+    // Check alternative frecency has not been marked for recalculation.
+    pages = await getAllPages();
+    Assert.ok(
+      pages.every(p => p.recalc_alt_frecency == 0),
+      "The entries not have been marked for recalc"
+    );
+    Assert.ok(
+      pages.every(p => p.alt_frecency === null),
+      "All the alt_frecency values should have been nullified"
+    );
+
+    // Ensure moz_meta has been updated.
+    // Avoid hitting the cache, we want to check the actual database value.
+    PlacesUtils.metadata.cache.clear();
+    Assert.ok(
+      ObjectUtils.isEmpty(
+        await PlacesUtils.metadata.get(
+          PlacesFrecencyRecalculator.alternativeFrecencyInfo.pages.metadataKey,
+          Object.create(null)
+        )
+      ),
+      "Check the algorithm variables has been removed"
+    );
+  }
+);
+
+add_task(
+  {
+    pref_set: [[FEATURE_PREF, true]],
+  },
+  async function test_score() {
+    await restartRecalculator();
+
+    // This is not intended to cover the algorithm as a whole, but just as a
+    // sanity check for scores.
+
+    await PlacesTestUtils.addVisits([
+      {
+        url: "https://low.moz.org",
+        transition: PlacesUtils.history.TRANSITIONS.FRAMED_LINK,
+      },
+      {
+        url: "https://old.moz.org",
+        visitDate: (Date.now() - 2 * 86400000) * 1000,
+      },
+      { url: "https://base.moz.org" },
+      { url: "https://manyvisits.moz.org" },
+      { url: "https://manyvisits.moz.org" },
+      { url: "https://manyvisits.moz.org" },
+      { url: "https://manyvisits.moz.org" },
+    ]);
+    await PlacesUtils.bookmarks.insert({
+      url: "https://unvisitedbookmark.moz.org",
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    });
+    await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+    let getFrecency = url =>
+      PlacesTestUtils.getDatabaseValue("moz_places", "alt_frecency", {
+        url,
+      });
+    let low = await getFrecency("https://low.moz.org/");
+    let old = await getFrecency("https://old.moz.org/");
+    Assert.greater(old, low);
+    let base = await getFrecency("https://base.moz.org/");
+    Assert.greater(base, old);
+    let unvisitedBm = await getFrecency("https://unvisitedbookmark.moz.org/");
+    Assert.greater(unvisitedBm, base);
+    let manyVisits = await getFrecency("https://manyvisits.moz.org/");
+    Assert.greater(manyVisits, unvisitedBm);
+  }
+);
diff --git a/toolkit/components/places/tests/unit/test_frecency_pages_recalc_alt.js b/toolkit/components/places/tests/unit/test_frecency_pages_recalc_alt.js
new file mode 100644
index 0000000000..e492099845
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_pages_recalc_alt.js
@@ -0,0 +1,76 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests that recalc_alt_frecency in the moz_places table is updated correctly.
+
+add_task(async function test() {
+  info("test recalc_alt_frecency is set to 1 when a visit is added");
+  const now = new Date();
+  const URL = "https://mozilla.org/test/";
+  let getRecalc = url =>
+    PlacesTestUtils.getDatabaseValue("moz_places", "recalc_alt_frecency", {
+      url,
+    });
+  let setRecalc = (url, val) =>
+    PlacesTestUtils.updateDatabaseValues(
+      "moz_places",
+      { recalc_alt_frecency: val },
+      { url }
+    );
+  let getFrecency = url =>
+    PlacesTestUtils.getDatabaseValue("moz_places", "alt_frecency", {
+      url,
+    });
+  await PlacesTestUtils.addVisits([
+    {
+      url: URL,
+      visitDate: now,
+    },
+    {
+      url: URL,
+      visitDate: new Date(new Date().setDate(now.getDate() - 30)),
+    },
+  ]);
+  Assert.equal(await getRecalc(URL), 1);
+  Assert.greater(await getFrecency(URL), 0);
+
+  info("Remove just one visit (otherwise the page would be orphaned).");
+  await PlacesUtils.history.removeVisitsByFilter({
+    beginDate: new Date(now.valueOf() - 10000),
+    endDate: new Date(now.valueOf() + 10000),
+  });
+  Assert.equal(await getRecalc(URL), 1);
+  await setRecalc(URL, 0);
+
+  info("Add a bookmark to the page");
+  let bm = await PlacesUtils.bookmarks.insert({
+    url: URL,
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  Assert.equal(await getRecalc(URL), 1);
+  await setRecalc(URL, 0);
+
+  info("Clear history");
+  await PlacesUtils.history.clear();
+  Assert.equal(await getRecalc(URL), 1);
+  await setRecalc(URL, 0);
+
+  // Add back a visit so the page is not an orphan once we remove the bookmark.
+  await PlacesTestUtils.addVisits(URL);
+  Assert.equal(await getRecalc(URL), 0);
+  Assert.greater(await getFrecency(URL), 0);
+
+  info("change the bookmark URL");
+  const URL2 = "https://editedbookmark.org/";
+  bm.url = URL2;
+  await PlacesUtils.bookmarks.update(bm);
+  Assert.equal(await getRecalc(URL), 1);
+  Assert.equal(await getRecalc(URL2), 1);
+  await setRecalc(URL, 0);
+  await setRecalc(URL2, 0);
+
+  info("Remove the bookmark from the page");
+  await PlacesUtils.bookmarks.remove(bm);
+  Assert.equal(await getRecalc(URL2), 1);
+  await setRecalc(URL2, 0);
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_recalc_triggers.js b/toolkit/components/places/tests/unit/test_frecency_recalc_triggers.js
new file mode 100644
index 0000000000..2e75a6d459
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_recalc_triggers.js
@@ -0,0 +1,281 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Tests that all the operations affecting frecency are either recalculating
+ * immediately or triggering a recalculation.
+ * Operations that should recalculate immediately:
+ *  - adding visits
+ * Operations that should just trigger a recalculation:
+ *  - removing visits
+ *  - adding a bookmark
+ *  - removing a bookmark
+ *  - changing url of a bookmark
+ *
+ * Also check setting a frecency resets recalc_frecency to 0.
+ **/
+
+const TEST_URL = "https://example.com/";
+const TEST_URL_2 = "https://example2.com/";
+
+// NOTE: Until we fix Bug 1806666 this test has to run queries manually because
+// the official APIs recalculate frecency immediately. After the fix, these
+// helpers can be removed and the test can be much simpler.
+function insertVisit(url) {
+  return PlacesUtils.withConnectionWrapper("insertVisit", async db => {
+    await db.execute(
+      `INSERT INTO moz_historyvisits(place_id, visit_date, visit_type)
+       VALUES ((SELECT id FROM moz_places WHERE url = :url), 1648226608386000, 1)`,
+      { url }
+    );
+  });
+}
+function removeVisit(url) {
+  return PlacesUtils.withConnectionWrapper("insertVisit", async db => {
+    await db.execute(
+      `DELETE FROM moz_historyvisits WHERE place_id
+        = (SELECT id FROM moz_places WHERE url = :url)`,
+      { url }
+    );
+  });
+}
+function resetFrecency(url) {
+  return PlacesUtils.withConnectionWrapper("insertVisit", async db => {
+    await db.execute(`UPDATE moz_places SET frecency = -1 WHERE url = :url`, {
+      url,
+    });
+  });
+}
+
+add_task(async function test_visit() {
+  // First add a bookmark so the page is not orphaned.
+  let bm = await PlacesUtils.bookmarks.insert({
+    url: TEST_URL,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+
+  info("Add a visit check frecency is calculated immediately");
+  await PlacesTestUtils.addVisits(TEST_URL);
+  let originalFrecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.greater(originalFrecency, 0, "frecency was recalculated immediately");
+  let recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 0, "frecency doesn't need a recalc");
+
+  info("Add a visit (raw query) check frecency is not calculated immediately");
+  await insertVisit(TEST_URL);
+  let frecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(frecency, originalFrecency, "frecency is unchanged");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+
+  info("Check setting frecency resets recalc_frecency");
+  await resetFrecency(TEST_URL);
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 0, "frecency doesn't need a recalc");
+
+  info("Removing a visit sets recalc_frecency");
+  await removeVisit(TEST_URL);
+  frecency = await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+    url: TEST_URL,
+  });
+  Assert.equal(frecency, -1, "frecency is unchanged");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.remove(bm);
+});
+
+add_task(async function test_bookmark() {
+  // First add a visit so the page is not orphaned.
+  await PlacesTestUtils.addVisits([TEST_URL, TEST_URL_2]);
+
+  let originalFrecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.greater(originalFrecency, 0);
+
+  info("Check adding a bookmark sets recalc_frecency");
+  let bm = await PlacesUtils.bookmarks.insert({
+    url: TEST_URL,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+
+  let frecency = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(frecency, originalFrecency, "frecency is unchanged");
+  let recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+
+  info("Check changing a bookmark url sets recalc_frecency on both urls");
+  await PlacesUtils.bookmarks.update({
+    guid: bm.guid,
+    url: TEST_URL_2,
+  });
+  frecency = await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+    url: TEST_URL,
+  });
+  Assert.equal(frecency, originalFrecency, "frecency is unchanged");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+  frecency = await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+    url: TEST_URL_2,
+  });
+  Assert.ok(frecency > 0, "frecency is valid");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL_2,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+
+  info("Check setting frecency resets recalc_frecency");
+  await resetFrecency(TEST_URL);
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 0, "frecency doesn't need a recalc");
+  await resetFrecency(TEST_URL_2);
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL_2,
+    }
+  );
+  Assert.equal(recalc, 0, "frecency doesn't need a recalc");
+
+  info("Removing a bookmark sets recalc_frecency");
+  await PlacesUtils.bookmarks.remove(bm.guid);
+  frecency = await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+    url: TEST_URL,
+  });
+  Assert.equal(frecency, -1, "frecency is unchanged");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL,
+    }
+  );
+  Assert.equal(recalc, 0, "frecency doesn't need a recalc");
+  frecency = await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+    url: TEST_URL_2,
+  });
+  Assert.equal(frecency, -1, "frecency is unchanged");
+  recalc = await PlacesTestUtils.getDatabaseValue(
+    "moz_places",
+    "recalc_frecency",
+    {
+      url: TEST_URL_2,
+    }
+  );
+  Assert.equal(recalc, 1, "frecency needs a recalc");
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_bookmark_frecency_zero() {
+  info("A url with frecency 0 should be recalculated if bookmarked");
+  let url = "https://zerofrecency.org/";
+  await PlacesTestUtils.addVisits({ url, transition: TRANSITION_FRAMED_LINK });
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { url }),
+    0
+  );
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "recalc_frecency", {
+      url,
+    }),
+    0
+  );
+  await PlacesUtils.bookmarks.insert({
+    url,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "recalc_frecency", {
+      url,
+    }),
+    1
+  );
+  info("place: uris should not be recalculated");
+  url = "place:test";
+  await PlacesUtils.bookmarks.insert({
+    url,
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { url }),
+    0
+  );
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "recalc_frecency", {
+      url,
+    }),
+    0
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_recalculator.js b/toolkit/components/places/tests/unit/test_frecency_recalculator.js
new file mode 100644
index 0000000000..91ed743f79
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_recalculator.js
@@ -0,0 +1,176 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Test PlacesFrecencyRecalculator scheduling.
+
+// Enable the collection (during test) for all products so even products
+// that don't collect the data will be able to run the test without failure.
+Services.prefs.setBoolPref(
+  "toolkit.telemetry.testing.overrideProductsCheck",
+  true
+);
+
+async function getOriginFrecency(origin) {
+  let db = await PlacesUtils.promiseDBConnection();
+  return (
+    await db.execute(
+      `SELECT frecency
+       FROM moz_origins
+       WHERE host = :origin`,
+      { origin }
+    )
+  )[0].getResultByIndex(0);
+}
+
+async function resetOriginFrecency(origin) {
+  await PlacesUtils.withConnectionWrapper(
+    "test_frecency_recalculator reset origin",
+    async db => {
+      await db.executeCached(
+        `UPDATE moz_origins
+         SET frecency = -1
+         WHERE host = :origin`,
+        { origin }
+      );
+    }
+  );
+}
+
+async function addVisitsAndSetRecalc(urls) {
+  await PlacesTestUtils.addVisits(urls);
+  await PlacesUtils.withConnectionWrapper(
+    "test_frecency_recalculator set recalc",
+    async db => {
+      await db.executeCached(
+        `UPDATE moz_places
+         SET frecency = -1
+         WHERE url in (
+           ${PlacesUtils.sqlBindPlaceholders(urls)}
+         )`,
+        urls
+      );
+      await db.executeCached(`DELETE FROM moz_updateoriginsupdate_temp`);
+      await db.executeCached(
+        `UPDATE moz_places
+         SET recalc_frecency = (CASE WHEN url in (
+               ${PlacesUtils.sqlBindPlaceholders(urls)}
+             ) THEN 1 ELSE 0 END)`,
+        urls
+      );
+    }
+  );
+}
+
+add_task(async function test() {
+  info("On startup a recalculation is always pending.");
+  Assert.ok(
+    PlacesFrecencyRecalculator.isRecalculationPending,
+    "Recalculation should be pending"
+  );
+  // If everything gets recalculated, then it should not be pending anymore.
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  Assert.ok(
+    !PlacesFrecencyRecalculator.isRecalculationPending,
+    "Recalculation should not be pending"
+  );
+
+  // If after a recalculation there's outdated entries left, a new recalculation
+  // should be pending.
+  info("Insert outdated frecencies");
+  const url1 = new URL("https://test1.moz.org/");
+  const url2 = new URL("https://test2.moz.org/");
+  await addVisitsAndSetRecalc([url1.href, url2.href]);
+  await resetOriginFrecency(url1.host);
+  await resetOriginFrecency(url2.host);
+
+  Assert.ok(
+    PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have set shouldStartFrecencyRecalculation"
+  );
+  await PlacesFrecencyRecalculator.recalculateSomeFrecencies({ chunkSize: 1 });
+  Assert.ok(
+    PlacesFrecencyRecalculator.isRecalculationPending,
+    "Recalculation should be pending"
+  );
+  await PlacesFrecencyRecalculator.recalculateSomeFrecencies({ chunkSize: 2 });
+  Assert.ok(
+    !PlacesFrecencyRecalculator.isRecalculationPending,
+    "Recalculation should not be pending"
+  );
+  Assert.ok(
+    !PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have unset shouldStartFrecencyRecalculation"
+  );
+
+  Assert.greater(await getOriginFrecency(url1.host), 0);
+  Assert.greater(await getOriginFrecency(url2.host), 0);
+
+  info("Changing recalc_frecency of an entry adds a pending recalculation.");
+  PlacesUtils.history.shouldStartFrecencyRecalculation = false;
+  let promiseNotify = TestUtils.topicObserved("frecency-recalculation-needed");
+  await PlacesUtils.withConnectionWrapper(
+    "test_frecency_recalculator",
+    async db => {
+      await db.executeCached(
+        `UPDATE moz_places SET recalc_frecency = 1 WHERE url = :url`,
+        { url: url1.href }
+      );
+    }
+  );
+  Assert.ok(
+    PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have set shouldStartFrecencyRecalculation"
+  );
+
+  await promiseNotify;
+  Assert.ok(
+    PlacesFrecencyRecalculator.isRecalculationPending,
+    "Recalculation should be pending"
+  );
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  Assert.ok(
+    !PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have unset shouldStartFrecencyRecalculation"
+  );
+});
+
+add_task(async function test_chunk_time_telemetry() {
+  await PlacesUtils.bookmarks.insert({
+    url: "https://test-bookmark.com",
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+  });
+  Assert.ok(
+    PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have set shouldStartFrecencyRecalculation"
+  );
+  let histogram = TelemetryTestUtils.getAndClearHistogram(
+    "PLACES_FRECENCY_RECALC_CHUNK_TIME_MS"
+  );
+  let subject = {};
+  PlacesFrecencyRecalculator.observe(subject, "test-execute-taskFn", "");
+  await subject.promise;
+  let snapshot = histogram.snapshot();
+  Assert.equal(
+    Object.values(snapshot.values).reduce((a, b) => a + b, 0),
+    1
+  );
+  Assert.greater(snapshot.sum, 0);
+  Assert.ok(
+    !PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should have unset shouldStartFrecencyRecalculation"
+  );
+
+  // It should now not report any new time, since there's nothing to recalculate.
+  histogram.clear();
+  PlacesFrecencyRecalculator.observe(subject, "test-execute-taskFn", "");
+  await subject.promise;
+  snapshot = histogram.snapshot();
+  Assert.equal(
+    Object.values(snapshot.values).reduce((a, b) => a + b, 0),
+    0
+  );
+  Assert.ok(
+    !PlacesUtils.history.shouldStartFrecencyRecalculation,
+    "Should still not have set shouldStartFrecencyRecalculation"
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_unvisited_bookmark.js b/toolkit/components/places/tests/unit/test_frecency_unvisited_bookmark.js
new file mode 100644
index 0000000000..ae7bd3d813
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_unvisited_bookmark.js
@@ -0,0 +1,54 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests frecency of unvisited bookmarks.
+
+add_task(async function () {
+  // Randomly sorted by date.
+  const now = new Date();
+  const bookmarks = [
+    {
+      url: "https://example.com/1",
+      date: new Date(new Date().setDate(now.getDate() - 30)),
+    },
+    {
+      url: "https://example.com/2",
+      date: new Date(new Date().setDate(now.getDate() - 1)),
+    },
+    {
+      url: "https://example.com/3",
+      date: new Date(new Date().setDate(now.getDate() - 100)),
+    },
+    {
+      url: "https://example.com/1", // Same url but much older.
+      date: new Date(new Date().setDate(now.getDate() - 120)),
+    },
+  ];
+
+  for (let bookmark of bookmarks) {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      url: bookmark.url,
+      dateAdded: bookmark.date,
+    });
+  }
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+
+  // The newest bookmark should have an higher frecency.
+  Assert.greater(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: bookmarks[1].url,
+    }),
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: bookmarks[0].url,
+    })
+  );
+  Assert.greater(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: bookmarks[0].url,
+    }),
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: bookmarks[2].url,
+    })
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_frecency_zero_updated.js b/toolkit/components/places/tests/unit/test_frecency_zero_updated.js
new file mode 100644
index 0000000000..44c329635e
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_frecency_zero_updated.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests a zero frecency is correctly updated when inserting new valid visits.
+
+add_task(async function () {
+  const TEST_URI = NetUtil.newURI("http://example.com/");
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: TEST_URI,
+    title: "A title",
+  });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+
+  // Removing the bookmark should leave an orphan page with zero frecency.
+  // Note this would usually be expired later by expiration.
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  // Now add a valid visit to the page, frecency should increase.
+  await PlacesTestUtils.addVisits({ uri: TEST_URI });
+  Assert.ok(
+    (await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    })) > 0
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_getChildIndex.js b/toolkit/components/places/tests/unit/test_getChildIndex.js
new file mode 100644
index 0000000000..35eb6fd22b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_getChildIndex.js
@@ -0,0 +1,73 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim:set ts=2 sw=2 sts=2 et:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Tests nsNavHistoryContainerResultNode::GetChildIndex(aNode) functionality.
+ */
+
+add_task(async function test_get_child_index() {
+  // Add a bookmark to the menu.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://test.mozilla.org/bookmark/",
+    title: "Test bookmark",
+  });
+
+  // Add a bookmark to unfiled folder.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://test.mozilla.org/unfiled/",
+    title: "Unfiled bookmark",
+  });
+
+  // Get the unfiled bookmark node.
+  let unfiledNode = getNodeAt(PlacesUtils.bookmarks.unfiledGuid, 0);
+  if (!unfiledNode) {
+    do_throw("Unable to find bookmark in hierarchy!");
+  }
+  Assert.equal(unfiledNode.title, "Unfiled bookmark");
+
+  let hs = PlacesUtils.history;
+  let query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.menuGuid]);
+  let options = hs.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let root = hs.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  // Check functionality for proper nodes.
+  for (let i = 0; i < root.childCount; i++) {
+    let node = root.getChild(i);
+    print("Now testing: " + node.title);
+    Assert.equal(root.getChildIndex(node), i);
+  }
+
+  // Now search for an invalid node and expect an exception.
+  try {
+    root.getChildIndex(unfiledNode);
+    do_throw("Searching for an invalid node should have thrown.");
+  } catch (ex) {
+    print("We correctly got an exception.");
+  }
+
+  root.containerOpen = false;
+});
+
+function getNodeAt(aFolderGuid, aIndex) {
+  let hs = PlacesUtils.history;
+  let query = hs.getNewQuery();
+  query.setParents([aFolderGuid]);
+  let options = hs.getNewQueryOptions();
+  options.queryType = options.QUERY_TYPE_BOOKMARKS;
+  let root = hs.executeQuery(query, options).root;
+  root.containerOpen = true;
+  if (root.childCount < aIndex) {
+    do_throw("Not enough children to find bookmark!");
+  }
+  let node = root.getChild(aIndex);
+  root.containerOpen = false;
+  return node;
+}
diff --git a/toolkit/components/places/tests/unit/test_get_query_param_sql_function.js b/toolkit/components/places/tests/unit/test_get_query_param_sql_function.js
new file mode 100644
index 0000000000..072f53ceac
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_get_query_param_sql_function.js
@@ -0,0 +1,21 @@
+add_task(async function test_get_query_param_sql_function() {
+  let db = await PlacesUtils.promiseDBConnection();
+  await Assert.rejects(
+    db.execute(`SELECT get_query_param()`),
+    /wrong number of arguments/
+  );
+  let rows = await db.execute(`SELECT
+    get_query_param('a=b&c=d', 'a'),
+    get_query_param('a=b&c=d', 'c'),
+    get_query_param('a=b&a=c', 'a'),
+    get_query_param('a=b&c=d', 'e'),
+    get_query_param('a', 'a'),
+    get_query_param(NULL, NULL),
+    get_query_param('a=b&c=d', NULL),
+    get_query_param(NULL, 'a')`);
+  let results = ["b", "d", "b", null, "", null, null, null];
+  equal(rows[0].numEntries, results.length);
+  for (let i = 0; i < results.length; ++i) {
+    equal(rows[0].getResultByIndex(i), results[i]);
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_hash.js b/toolkit/components/places/tests/unit/test_hash.js
new file mode 100644
index 0000000000..701cb3d151
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_hash.js
@@ -0,0 +1,50 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function () {
+  // Check particular unicode urls with insertion and selection APIs to ensure
+  // url hashes match properly.
+  const URLS = [
+    "http://президент.президент/президент/",
+    "https://www.аррӏе.com/аррӏе/",
+    "http://名がドメイン/",
+  ];
+
+  for (let url of URLS) {
+    await PlacesTestUtils.addVisits(url);
+    Assert.ok(await PlacesUtils.history.fetch(url), "Found the added visit");
+    await PlacesUtils.bookmarks.insert({
+      url,
+      parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    });
+    Assert.ok(
+      await PlacesUtils.bookmarks.fetch({ url }),
+      "Found the added bookmark"
+    );
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.execute(
+      "SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url",
+      { url: new URL(url).href }
+    );
+    Assert.equal(rows.length, 1, "Matched the place from the database");
+    let id = rows[0].getResultByName("id");
+
+    // Now, suppose the urls has been inserted without proper parsing and retry.
+    // This should normally not happen through the API, but we have evidence
+    // it somehow happened.
+    await PlacesUtils.withConnectionWrapper("test_hash.js", async wdb => {
+      await wdb.execute(
+        `
+        UPDATE moz_places SET url_hash = hash(:url), url = :url
+        WHERE id = :id
+      `,
+        { url, id }
+      );
+      rows = await wdb.execute(
+        "SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url",
+        { url }
+      );
+      Assert.equal(rows.length, 1, "Matched the place from the database");
+    });
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_history.js b/toolkit/components/places/tests/unit/test_history.js
new file mode 100644
index 0000000000..a96d1fa5f0
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_history.js
@@ -0,0 +1,178 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Get history services
+var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+  Ci.nsINavHistoryService
+);
+
+/**
+ * Checks to see that a URI is in the database.
+ *
+ * @param aURI
+ *        The URI to check.
+ * @returns true if the URI is in the DB, false otherwise.
+ */
+function uri_in_db(aURI) {
+  var options = histsvc.getNewQueryOptions();
+  options.maxResults = 1;
+  options.resultType = options.RESULTS_AS_URI;
+  var query = histsvc.getNewQuery();
+  query.uri = aURI;
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var cc = root.childCount;
+  root.containerOpen = false;
+  return cc == 1;
+}
+
+// main
+
+add_task(async function test_execute() {
+  // we have a new profile, so we should have imported bookmarks
+  Assert.equal(histsvc.databaseStatus, histsvc.DATABASE_STATUS_CREATE);
+
+  // add a visit
+  var testURI = uri("http://mozilla.com");
+  await PlacesTestUtils.addVisits(testURI);
+
+  // now query for the visit, setting sorting and limit such that
+  // we should retrieve only the visit we just added
+  var options = histsvc.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.maxResults = 1;
+  options.resultType = options.RESULTS_AS_VISIT;
+  var query = histsvc.getNewQuery();
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  var cc = root.childCount;
+  for (var i = 0; i < cc; ++i) {
+    var node = root.getChild(i);
+    // test node properties in RESULTS_AS_VISIT
+    Assert.equal(node.uri, testURI.spec);
+    Assert.equal(node.type, Ci.nsINavHistoryResultNode.RESULT_TYPE_URI);
+  }
+  root.containerOpen = false;
+
+  // add another visit for the same URI, and a third visit for a different URI
+  var testURI2 = uri("http://google.com/");
+  await PlacesTestUtils.addVisits(testURI);
+  await PlacesTestUtils.addVisits(testURI2);
+
+  options.maxResults = 5;
+  options.resultType = options.RESULTS_AS_URI;
+
+  // test minVisits
+  query.minVisits = 0;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 2);
+  result.root.containerOpen = false;
+  query.minVisits = 1;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 2);
+  result.root.containerOpen = false;
+  query.minVisits = 2;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 1);
+  query.minVisits = 3;
+  result.root.containerOpen = false;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 0);
+  result.root.containerOpen = false;
+
+  // test maxVisits
+  query.minVisits = -1;
+  query.maxVisits = -1;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 2);
+  result.root.containerOpen = false;
+  query.maxVisits = 0;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 0);
+  result.root.containerOpen = false;
+  query.maxVisits = 1;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 1);
+  result.root.containerOpen = false;
+  query.maxVisits = 2;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 2);
+  result.root.containerOpen = false;
+  query.maxVisits = 3;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 2);
+  result.root.containerOpen = false;
+
+  // test annotation-based queries
+  await PlacesUtils.history.update({
+    url: "http://mozilla.com/",
+    annotations: new Map([["testAnno", 123]]),
+  });
+  query.annotation = "testAnno";
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 1);
+  Assert.equal(result.root.getChild(0).uri, "http://mozilla.com/");
+  result.root.containerOpen = false;
+
+  // test annotationIsNot
+  query.annotationIsNot = true;
+  result = histsvc.executeQuery(query, options);
+  result.root.containerOpen = true;
+  Assert.equal(result.root.childCount, 1);
+  Assert.equal(result.root.getChild(0).uri, "http://google.com/");
+  result.root.containerOpen = false;
+
+  // By default history is enabled.
+  Assert.ok(!histsvc.historyDisabled);
+
+  // test getPageTitle
+  await PlacesTestUtils.addVisits({
+    uri: uri("http://example.com"),
+    title: "title",
+  });
+  let placeInfo = await PlacesUtils.history.fetch("http://example.com");
+  Assert.equal(placeInfo.title, "title");
+
+  // query for the visit
+  Assert.ok(uri_in_db(testURI));
+
+  // test for schema changes in bug 373239
+  // get direct db connection
+  var db = histsvc.DBConnection;
+  var q = "SELECT id FROM moz_bookmarks";
+  var statement;
+  try {
+    statement = db.createStatement(q);
+  } catch (ex) {
+    do_throw("bookmarks table does not have id field, schema is too old!");
+  } finally {
+    statement.finalize();
+  }
+
+  // bug 394741 - regressed history text searches
+  await PlacesTestUtils.addVisits(uri("http://mozilla.com"));
+  options = histsvc.getNewQueryOptions();
+  // options.resultType = options.RESULTS_AS_VISIT;
+  query = histsvc.getNewQuery();
+  query.searchTerms = "moz";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.ok(root.childCount > 0);
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_history_clear.js b/toolkit/components/places/tests/unit/test_history_clear.js
new file mode 100644
index 0000000000..7c121a5b84
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_history_clear.js
@@ -0,0 +1,146 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var mDBConn = DBConn();
+
+add_task(async function test_history_clear() {
+  await PlacesTestUtils.addVisits([
+    { uri: uri("http://typed.mozilla.org/"), transition: TRANSITION_TYPED },
+    { uri: uri("http://link.mozilla.org/"), transition: TRANSITION_LINK },
+    {
+      uri: uri("http://download.mozilla.org/"),
+      transition: TRANSITION_DOWNLOAD,
+    },
+    {
+      uri: uri("http://redir_temp.mozilla.org/"),
+      transition: TRANSITION_REDIRECT_TEMPORARY,
+      referrer: "http://link.mozilla.org/",
+    },
+    {
+      uri: uri("http://redir_perm.mozilla.org/"),
+      transition: TRANSITION_REDIRECT_PERMANENT,
+      referrer: "http://link.mozilla.org/",
+    },
+  ]);
+
+  // add a place: bookmark
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: `place:parent=${PlacesUtils.bookmarks.tagsGuid}`,
+    title: "shortcut",
+  });
+
+  // Add an expire never annotation
+  // Actually expire never annotations are removed as soon as a page is removed
+  // from the database, so this should act as a normal visit.
+  await PlacesUtils.history.update({
+    url: "http://download.mozilla.org/",
+    annotations: new Map([["never", "never"]]),
+  });
+
+  // Add a bookmark
+  // Bookmarked page should have history cleared and frecency to be recalculated
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    url: "http://typed.mozilla.org/",
+    title: "bookmark",
+  });
+
+  await PlacesTestUtils.addVisits([
+    { uri: uri("http://typed.mozilla.org/"), transition: TRANSITION_BOOKMARK },
+    { uri: uri("http://frecency.mozilla.org/"), transition: TRANSITION_LINK },
+  ]);
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  // Clear history and wait for the history-cleared event notification.
+  let promiseClearHistory =
+    PlacesTestUtils.waitForNotification("history-cleared");
+  await PlacesUtils.history.clear();
+  await promiseClearHistory;
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  // Check that frecency for not cleared items (bookmarks) has been marked
+  // as to be recalculated.
+  let stmt = mDBConn.createStatement(
+    "SELECT h.id FROM moz_places h WHERE frecency <> 0 AND h.recalc_frecency = 0 "
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  stmt = mDBConn.createStatement(
+    `SELECT h.id FROM moz_places h WHERE h.recalc_frecency = 1
+       AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1`
+  );
+  Assert.ok(stmt.executeStep());
+  stmt.finalize();
+
+  // Check that all visit_counts have been brought to 0
+  stmt = mDBConn.createStatement(
+    "SELECT id FROM moz_places WHERE visit_count <> 0 LIMIT 1"
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that history tables are empty
+  stmt = mDBConn.createStatement(
+    "SELECT * FROM (SELECT id FROM moz_historyvisits LIMIT 1)"
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that all moz_places entries except bookmarks and place: have been removed
+  stmt = mDBConn.createStatement(
+    `SELECT h.id FROM moz_places h WHERE
+       url_hash NOT BETWEEN hash('place', 'prefix_lo') AND hash('place', 'prefix_hi')
+       AND NOT EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that we only have favicons for retained places
+  stmt = mDBConn.createStatement(
+    `SELECT 1
+     FROM moz_pages_w_icons
+     LEFT JOIN moz_places h ON url_hash = page_url_hash AND url = page_url
+     WHERE h.id ISNULL`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+  stmt = mDBConn.createStatement(
+    `SELECT 1
+     FROM moz_icons WHERE id NOT IN (
+       SELECT icon_id FROM moz_icons_to_pages
+     )`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that we only have annotations for retained places
+  stmt = mDBConn.createStatement(
+    `SELECT a.id FROM moz_annos a WHERE NOT EXISTS
+       (SELECT id FROM moz_places WHERE id = a.place_id) LIMIT 1`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that we only have inputhistory for retained places
+  stmt = mDBConn.createStatement(
+    `SELECT i.place_id FROM moz_inputhistory i WHERE NOT EXISTS
+       (SELECT id FROM moz_places WHERE id = i.place_id) LIMIT 1`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+
+  // Check that place:uris have frecency 0
+  stmt = mDBConn.createStatement(
+    `SELECT h.id FROM moz_places h
+     WHERE url_hash BETWEEN hash('place', 'prefix_lo')
+                        AND hash('place', 'prefix_hi')
+       AND h.frecency <> 0 LIMIT 1`
+  );
+  Assert.ok(!stmt.executeStep());
+  stmt.finalize();
+});
diff --git a/toolkit/components/places/tests/unit/test_history_notifications.js b/toolkit/components/places/tests/unit/test_history_notifications.js
new file mode 100644
index 0000000000..339080b042
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_history_notifications.js
@@ -0,0 +1,50 @@
+const NS_PLACES_INIT_COMPLETE_TOPIC = "places-init-complete";
+let gLockedConn;
+
+add_task(async function setup() {
+  // Create a dummy places.sqlite and open an unshared connection on it
+  let db = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  db.append("places.sqlite");
+  gLockedConn = Services.storage.openUnsharedDatabase(db);
+  Assert.ok(db.exists(), "The database should have been created");
+
+  // We need an exclusive lock on the db
+  gLockedConn.executeSimpleSQL("PRAGMA locking_mode = EXCLUSIVE");
+  // Exclusive locking is lazy applied, we need to make a write to activate it
+  gLockedConn.executeSimpleSQL("PRAGMA USER_VERSION = 1");
+});
+
+add_task(async function locked() {
+  // Try to create history service while the db is locked.
+  // It should be possible to create the service, but any method using the
+  // database will fail.
+  let resolved = false;
+  let promiseComplete = promiseTopicObserved(
+    NS_PLACES_INIT_COMPLETE_TOPIC
+  ).then(() => (resolved = true));
+  let history = Cc["@mozilla.org/browser/nav-history-service;1"].createInstance(
+    Ci.nsINavHistoryService
+  );
+  // The notification shouldn't happen until something tries to use the database.
+  await new Promise(resolve => do_timeout(100, resolve));
+  Assert.equal(
+    resolved,
+    false,
+    "The notification should not have been fired yet"
+  );
+  // This will initialize the database.
+  Assert.equal(history.databaseStatus, history.DATABASE_STATUS_LOCKED);
+  await promiseComplete;
+
+  // Close our connection and try to cleanup the file (could fail on Windows)
+  gLockedConn.close();
+  let db = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  db.append("places.sqlite");
+  if (db.exists()) {
+    try {
+      db.remove(false);
+    } catch (e) {
+      info("Unable to remove dummy places.sqlite");
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_history_observer.js b/toolkit/components/places/tests/unit/test_history_observer.js
new file mode 100644
index 0000000000..1a9323f890
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_history_observer.js
@@ -0,0 +1,186 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Registers a one-time places observer for 'page-visited',
+ * which resolves a promise on being called.
+ */
+function promiseVisitAdded(callback) {
+  return new Promise(resolve => {
+    async function listener(events) {
+      PlacesObservers.removeListener(["page-visited"], listener);
+      Assert.equal(events.length, 1, "Right number of visits notified");
+      Assert.equal(events[0].type, "page-visited");
+      await callback(events[0]);
+      resolve();
+    }
+    PlacesObservers.addListener(["page-visited"], listener);
+  });
+}
+
+/**
+ * Asynchronous task that adds a visit to the history database.
+ */
+async function task_add_visit(uri, timestamp, transition) {
+  uri = uri || NetUtil.newURI("http://firefox.com/");
+  timestamp = timestamp || Date.now() * 1000;
+  await PlacesTestUtils.addVisits({
+    uri,
+    transition: transition || TRANSITION_TYPED,
+    visitDate: timestamp,
+  });
+  return [uri, timestamp];
+}
+
+add_task(async function test_visitAdded() {
+  let promiseNotify = promiseVisitAdded(async function (visit) {
+    Assert.ok(visit.visitId > 0);
+    Assert.equal(visit.url, testuri.spec);
+    Assert.equal(visit.visitTime, testtime / 1000);
+    Assert.equal(visit.referringVisitId, 0);
+    Assert.equal(visit.transitionType, TRANSITION_TYPED);
+    let uri = NetUtil.newURI(visit.url);
+    await check_guid_for_uri(uri, visit.pageGuid);
+    Assert.ok(!visit.hidden);
+    Assert.equal(visit.visitCount, 1);
+    Assert.equal(visit.typedCount, 1);
+  });
+  let testuri = NetUtil.newURI("http://firefox.com/");
+  let testtime = Date.now() * 1000;
+  await task_add_visit(testuri, testtime);
+  await promiseNotify;
+});
+
+add_task(async function test_visitAdded() {
+  let promiseNotify = promiseVisitAdded(async function (visit) {
+    Assert.ok(visit.visitId > 0);
+    Assert.equal(visit.url, testuri.spec);
+    Assert.equal(visit.visitTime, testtime / 1000);
+    Assert.equal(visit.referringVisitId, 0);
+    Assert.equal(visit.transitionType, TRANSITION_FRAMED_LINK);
+    let uri = NetUtil.newURI(visit.url);
+    await check_guid_for_uri(uri, visit.pageGuid);
+    Assert.ok(visit.hidden);
+    Assert.equal(visit.visitCount, 1);
+    Assert.equal(visit.typedCount, 0);
+  });
+  let testuri = NetUtil.newURI("http://hidden.firefox.com/");
+  let testtime = Date.now() * 1000;
+  await task_add_visit(testuri, testtime, TRANSITION_FRAMED_LINK);
+  await promiseNotify;
+});
+
+add_task(async function test_multiple_onVisit() {
+  let testuri = NetUtil.newURI("http://self.firefox.com/");
+  let promiseNotifications = new Promise(resolve => {
+    async function listener(aEvents) {
+      Assert.equal(aEvents.length, 3, "Right number of visits notified");
+      for (let i = 0; i < aEvents.length; i++) {
+        Assert.equal(aEvents[i].type, "page-visited");
+        let visit = aEvents[i];
+        Assert.equal(testuri.spec, visit.url);
+        Assert.ok(visit.visitId > 0);
+        Assert.ok(visit.visitTime > 0);
+        Assert.ok(!visit.hidden);
+        let uri = NetUtil.newURI(visit.url);
+        await check_guid_for_uri(uri, visit.pageGuid);
+        switch (i) {
+          case 0:
+            Assert.equal(visit.referringVisitId, 0);
+            Assert.equal(visit.transitionType, TRANSITION_LINK);
+            Assert.equal(visit.visitCount, 1);
+            Assert.equal(visit.typedCount, 0);
+            break;
+          case 1:
+            Assert.ok(visit.referringVisitId > 0);
+            Assert.equal(visit.transitionType, TRANSITION_LINK);
+            Assert.equal(visit.visitCount, 2);
+            Assert.equal(visit.typedCount, 0);
+            break;
+          case 2:
+            Assert.equal(visit.referringVisitId, 0);
+            Assert.equal(visit.transitionType, TRANSITION_TYPED);
+            Assert.equal(visit.visitCount, 3);
+            Assert.equal(visit.typedCount, 1);
+
+            PlacesObservers.removeListener(["page-visited"], listener);
+            resolve();
+            break;
+        }
+      }
+    }
+    PlacesObservers.addListener(["page-visited"], listener);
+  });
+  await PlacesTestUtils.addVisits([
+    { uri: testuri, transition: TRANSITION_LINK },
+    { uri: testuri, referrer: testuri, transition: TRANSITION_LINK },
+    { uri: testuri, transition: TRANSITION_TYPED },
+  ]);
+  await promiseNotifications;
+});
+
+add_task(async function test_pageRemovedFromStore() {
+  let [testuri] = await task_add_visit();
+  let testguid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: testuri,
+  });
+
+  const promiseNotify = PlacesTestUtils.waitForNotification("page-removed");
+
+  await PlacesUtils.history.remove(testuri);
+
+  const events = await promiseNotify;
+  Assert.equal(events.length, 1, "Right number of page-removed notified");
+  Assert.equal(events[0].type, "page-removed");
+  Assert.ok(events[0].isRemovedFromStore);
+  Assert.equal(events[0].url, testuri.spec);
+  Assert.equal(events[0].pageGuid, testguid);
+  Assert.equal(events[0].reason, PlacesVisitRemoved.REASON_DELETED);
+});
+
+add_task(async function test_pageRemovedAllVisits() {
+  const promiseNotify = PlacesTestUtils.waitForNotification("page-removed");
+
+  let msecs24hrsAgo = Date.now() - 86400 * 1000;
+  let [testuri] = await task_add_visit(undefined, msecs24hrsAgo * 1000);
+  // Add a bookmark so the page is not removed.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "test",
+    url: testuri,
+  });
+  let testguid = await PlacesTestUtils.getDatabaseValue("moz_places", "guid", {
+    url: testuri,
+  });
+  await PlacesUtils.history.remove(testuri);
+
+  const events = await promiseNotify;
+  Assert.equal(events.length, 1, "Right number of page-removed notified");
+  Assert.equal(events[0].type, "page-removed");
+  Assert.ok(!events[0].isRemovedFromStore);
+  Assert.equal(events[0].url, testuri.spec);
+  // Can't use do_check_guid_for_uri() here because the visit is already gone.
+  Assert.equal(events[0].pageGuid, testguid);
+  Assert.equal(events[0].reason, PlacesVisitRemoved.REASON_DELETED);
+  Assert.ok(!events[0].isPartialVisistsRemoval); // All visits have been removed.
+});
+
+add_task(async function test_pageTitleChanged() {
+  const [testuri] = await task_add_visit();
+  const title = "test-title";
+
+  const promiseNotify =
+    PlacesTestUtils.waitForNotification("page-title-changed");
+
+  await PlacesTestUtils.addVisits({
+    uri: testuri,
+    title,
+  });
+
+  const events = await promiseNotify;
+  Assert.equal(events.length, 1, "Right number of title changed notified");
+  Assert.equal(events[0].type, "page-title-changed");
+  Assert.equal(events[0].url, testuri.spec);
+  Assert.equal(events[0].title, title);
+  await check_guid_for_uri(testuri, events[0].pageGuid);
+});
diff --git a/toolkit/components/places/tests/unit/test_history_sidebar.js b/toolkit/components/places/tests/unit/test_history_sidebar.js
new file mode 100644
index 0000000000..868ef79f70
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_history_sidebar.js
@@ -0,0 +1,418 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+let nowObj = new Date();
+
+/**
+ * Normalizes a Date to midnight.
+ *
+ * @param {Date} inputDate
+ * @return normalized Date
+ */
+function toMidnight(inputDate) {
+  let date = new Date(inputDate);
+  date.setHours(0);
+  date.setMinutes(0);
+  date.setSeconds(0);
+  date.setMilliseconds(0);
+  return date;
+}
+
+/**
+ * Adds a test URI visit to the database.
+ *
+ * @param aURI
+ *        The URI to add a visit for.
+ * @param aTime
+ *        Reference "now" time.
+ * @param aDayOffset
+ *        number of days to add, pass a negative value to subtract them.
+ */
+async function addNormalizedVisit(aURI, aTime, aDayOffset) {
+  let dateObj = toMidnight(aTime);
+  // Days where DST changes should be taken into account.
+  let previousDateObj = new Date(dateObj.getTime() + aDayOffset * 86400000);
+  let DSTCorrection =
+    (dateObj.getTimezoneOffset() - previousDateObj.getTimezoneOffset()) *
+    60 *
+    1000;
+  // Substract aDayOffset
+  let PRTimeWithOffset = (previousDateObj.getTime() - DSTCorrection) * 1000;
+  info(
+    "Adding visit to " +
+      aURI.spec +
+      " at " +
+      PlacesUtils.toDate(PRTimeWithOffset)
+  );
+  await PlacesTestUtils.addVisits({
+    uri: aURI,
+    visitDate: PRTimeWithOffset,
+  });
+}
+
+function openRootForResultType(resultType) {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = resultType;
+  let query = PlacesUtils.history.getNewQuery();
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let root = result.root;
+  root.containerOpen = true;
+  return result;
+}
+
+function daysForMonthsAgo(months) {
+  let oldTime = toMidnight(new Date());
+  // Set day before month, otherwise we could try to calculate 30 February, or
+  // other nonexistent days.
+  oldTime.setDate(1);
+  oldTime.setMonth(nowObj.getMonth() - months);
+  // Stay larger for eventual timezone issues, add 2 days.
+  return parseInt((nowObj - oldTime) / (1000 * 60 * 60 * 24)) + 2;
+}
+
+// This test relies on en-US locale
+// Offset is number of days
+let containers = [
+  { label: "Today", offset: 0, visible: true },
+  { label: "Yesterday", offset: -1, visible: true },
+  { label: "Last 7 days", offset: -2, visible: true },
+  { label: "This month", offset: -8, visible: nowObj.getDate() > 8 },
+  { label: "", offset: -daysForMonthsAgo(0), visible: true },
+  { label: "", offset: -daysForMonthsAgo(1), visible: true },
+  { label: "", offset: -daysForMonthsAgo(2), visible: true },
+  { label: "", offset: -daysForMonthsAgo(3), visible: true },
+  { label: "", offset: -daysForMonthsAgo(4), visible: true },
+  { label: "Older than 6 months", offset: -daysForMonthsAgo(5), visible: true },
+];
+
+let visibleContainers = containers.filter(container => container.visible);
+
+/**
+ * Asynchronous task that fills history and checks containers' labels.
+ */
+add_task(async function task_fill_history() {
+  info("*** TEST Fill History");
+  // We can't use "now" because our hardcoded offsets would be invalid for some
+  // date.  So we hardcode a date.
+  for (let i = 0; i < containers.length; i++) {
+    let container = containers[i];
+    let testURI = uri("http://mirror" + i + ".mozilla.com/b");
+    await addNormalizedVisit(testURI, nowObj, container.offset);
+    testURI = uri("http://mirror" + i + ".mozilla.com/a");
+    await addNormalizedVisit(testURI, nowObj, container.offset);
+    testURI = uri("http://mirror" + i + ".google.com/b");
+    await addNormalizedVisit(testURI, nowObj, container.offset);
+    testURI = uri("http://mirror" + i + ".google.com/a");
+    await addNormalizedVisit(testURI, nowObj, container.offset);
+    // Bug 485703 - Hide date containers not containing additional entries
+    //              compared to previous ones.
+    // Check after every new container is added.
+    check_visit(container.offset);
+  }
+
+  let root = openRootForResultType(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  ).root;
+  let cc = root.childCount;
+  info("Found containers:");
+  let previousLabels = [];
+  for (let i = 0; i < cc; i++) {
+    let container = visibleContainers[i];
+    let node = root.getChild(i);
+    info(node.title);
+    if (container.label) {
+      Assert.equal(node.title, container.label);
+    }
+    // Check labels are not repeated.
+    Assert.ok(!previousLabels.includes(node.title));
+    previousLabels.push(node.title);
+  }
+  Assert.equal(cc, visibleContainers.length);
+  root.containerOpen = false;
+});
+
+/**
+ * Bug 485703 - Hide date containers not containing additional entries compared
+ *              to previous ones.
+ */
+function check_visit(aOffset) {
+  let root = openRootForResultType(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  ).root;
+
+  let unexpected = [];
+  switch (aOffset) {
+    case 0:
+      unexpected = ["Yesterday", "Last 7 days", "This month"];
+      break;
+    case -1:
+      unexpected = ["Last 7 days", "This month"];
+      break;
+    case -2:
+      unexpected = ["This month"];
+      break;
+    default:
+    // Other containers are tested later.
+  }
+
+  info("Found containers:");
+  let cc = root.childCount;
+  for (let i = 0; i < cc; i++) {
+    let node = root.getChild(i);
+    info(node.title);
+    Assert.ok(!unexpected.includes(node.title));
+  }
+  root.containerOpen = false;
+}
+
+/**
+ * Queries history grouped by date and site, checking containers' labels and
+ * children.
+ */
+add_task(async function test_RESULTS_AS_DATE_SITE_QUERY() {
+  info("*** TEST RESULTS_AS_DATE_SITE_QUERY");
+  let result = openRootForResultType(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  );
+  let root = result.root;
+
+  // Check one of the days
+  let dayNode = PlacesUtils.asContainer(root.getChild(0));
+  dayNode.containerOpen = true;
+  Assert.equal(dayNode.childCount, 2);
+
+  // Items should be sorted by host
+  let site1 = PlacesUtils.asContainer(dayNode.getChild(0));
+  Assert.equal(site1.title, "mirror0.google.com");
+
+  let site2 = PlacesUtils.asContainer(dayNode.getChild(1));
+  Assert.equal(site2.title, "mirror0.mozilla.com");
+
+  site1.containerOpen = true;
+  Assert.equal(site1.childCount, 2);
+
+  // Inside of host sites are sorted by title
+  let site1visit = site1.getChild(0);
+  Assert.equal(site1visit.uri, "http://mirror0.google.com/a");
+
+  // Bug 473157: changing sorting mode should not affect the containers
+  result.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_DESCENDING;
+
+  // Check one of the days
+  dayNode = PlacesUtils.asContainer(root.getChild(0));
+  dayNode.containerOpen = true;
+  Assert.equal(dayNode.childCount, 2);
+
+  // Hosts are still sorted by title
+  site1 = PlacesUtils.asContainer(dayNode.getChild(0));
+  Assert.equal(site1.title, "mirror0.google.com");
+
+  site2 = PlacesUtils.asContainer(dayNode.getChild(1));
+  Assert.equal(site2.title, "mirror0.mozilla.com");
+
+  site1.containerOpen = true;
+  Assert.equal(site1.childCount, 2);
+
+  // But URLs are now sorted by title descending
+  site1visit = site1.getChild(0);
+  Assert.equal(site1visit.uri, "http://mirror0.google.com/b");
+
+  site1.containerOpen = false;
+  dayNode.containerOpen = false;
+  root.containerOpen = false;
+});
+
+/**
+ * Queries history grouped by date, checking containers' labels and children.
+ */
+add_task(async function test_RESULTS_AS_DATE_QUERY() {
+  info("*** TEST RESULTS_AS_DATE_QUERY");
+  let result = openRootForResultType(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY
+  );
+  let root = result.root;
+  let cc = root.childCount;
+  Assert.equal(cc, visibleContainers.length);
+  info("Found containers:");
+  for (let i = 0; i < cc; i++) {
+    let container = visibleContainers[i];
+    let node = root.getChild(i);
+    info(node.title);
+    if (container.label) {
+      Assert.equal(node.title, container.label);
+    }
+  }
+
+  // Check one of the days
+  let dayNode = PlacesUtils.asContainer(root.getChild(0));
+  dayNode.containerOpen = true;
+  Assert.equal(dayNode.childCount, 4);
+
+  // Items should be sorted by title
+  let visit1 = dayNode.getChild(0);
+  Assert.equal(visit1.uri, "http://mirror0.google.com/a");
+
+  let visit2 = dayNode.getChild(3);
+  Assert.equal(visit2.uri, "http://mirror0.mozilla.com/b");
+
+  // Bug 473157: changing sorting mode should not affect the containers
+  result.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_DESCENDING;
+
+  // Check one of the days
+  dayNode = PlacesUtils.asContainer(root.getChild(0));
+  dayNode.containerOpen = true;
+  Assert.equal(dayNode.childCount, 4);
+
+  // But URLs are now sorted by title descending
+  visit1 = dayNode.getChild(0);
+  Assert.equal(visit1.uri, "http://mirror0.mozilla.com/b");
+
+  visit2 = dayNode.getChild(3);
+  Assert.equal(visit2.uri, "http://mirror0.google.com/a");
+
+  dayNode.containerOpen = false;
+  root.containerOpen = false;
+});
+
+/**
+ * Queries history grouped by site, checking containers' labels and children.
+ */
+add_task(async function test_RESULTS_AS_SITE_QUERY() {
+  info("*** TEST RESULTS_AS_SITE_QUERY");
+  // add a bookmark with a domain not in the set of visits in the db
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    url: "http://foobar",
+    title: "",
+  });
+
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.resultType = options.RESULTS_AS_SITE_QUERY;
+  options.sortingMode = options.SORT_BY_TITLE_ASCENDING;
+  let query = PlacesUtils.history.getNewQuery();
+  let result = PlacesUtils.history.executeQuery(query, options);
+  let root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, containers.length * 2);
+
+  // Expected results:
+  //   "mirror0.google.com",
+  //   "mirror0.mozilla.com",
+  //   "mirror1.google.com",
+  //   "mirror1.mozilla.com",
+  //   "mirror2.google.com",
+  //   "mirror2.mozilla.com",
+  //   "mirror3.google.com",  <== We check for this site (index 6)
+  //   "mirror3.mozilla.com",
+  //   "mirror4.google.com",
+  //   "mirror4.mozilla.com",
+  //   "mirror5.google.com",
+  //   "mirror5.mozilla.com",
+  //   ...
+
+  // Items should be sorted by host
+  let siteNode = PlacesUtils.asContainer(root.getChild(6));
+  Assert.equal(siteNode.title, "mirror3.google.com");
+
+  siteNode.containerOpen = true;
+  Assert.equal(siteNode.childCount, 2);
+
+  // Inside of host sites are sorted by title
+  let visitNode = siteNode.getChild(0);
+  Assert.equal(visitNode.uri, "http://mirror3.google.com/a");
+
+  // Bug 473157: changing sorting mode should not affect the containers
+  result.sortingMode = options.SORT_BY_TITLE_DESCENDING;
+  siteNode = PlacesUtils.asContainer(root.getChild(6));
+  Assert.equal(siteNode.title, "mirror3.google.com");
+
+  siteNode.containerOpen = true;
+  Assert.equal(siteNode.childCount, 2);
+
+  // But URLs are now sorted by title descending
+  let visit = siteNode.getChild(0);
+  Assert.equal(visit.uri, "http://mirror3.google.com/b");
+
+  siteNode.containerOpen = false;
+  root.containerOpen = false;
+
+  // Cleanup.
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+});
+
+/**
+ * Checks that queries grouped by date do liveupdate correctly.
+ */
+async function test_date_liveupdate(aResultType) {
+  let midnight = toMidnight(nowObj);
+
+  // TEST 1. Test that the query correctly updates when it is root.
+  let root = openRootForResultType(aResultType).root;
+  Assert.equal(root.childCount, visibleContainers.length);
+
+  // Remove "Today".
+  await PlacesUtils.history.removeByFilter({
+    beginDate: new Date(midnight.getTime()),
+    endDate: new Date(Date.now()),
+  });
+  Assert.equal(root.childCount, visibleContainers.length - 1);
+
+  // Open "Last 7 days" container, this way we will have a container accepting
+  // the new visit, but we should still add back "Today" container.
+  let last7Days = PlacesUtils.asContainer(root.getChild(1));
+  last7Days.containerOpen = true;
+
+  // Add a visit for "Today".  This should add back the missing "Today"
+  // container.
+  await addNormalizedVisit(uri("http://www.mozilla.org/"), nowObj, 0);
+  Assert.equal(root.childCount, visibleContainers.length);
+
+  last7Days.containerOpen = false;
+  root.containerOpen = false;
+
+  // TEST 2. Test that the query correctly updates even if it is not root.
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    url: "place:type=" + aResultType,
+    title: "",
+  });
+
+  // Query toolbar and open our query container, then check again liveupdate.
+  root = PlacesUtils.getFolderContents(PlacesUtils.bookmarks.toolbarGuid).root;
+  Assert.equal(root.childCount, 1);
+  let dateContainer = PlacesUtils.asContainer(root.getChild(0));
+  dateContainer.containerOpen = true;
+
+  Assert.equal(dateContainer.childCount, visibleContainers.length);
+  // Remove "Today".
+  await PlacesUtils.history.removeByFilter({
+    beginDate: new Date(midnight.getTime()),
+    endDate: new Date(Date.now()),
+  });
+  Assert.equal(dateContainer.childCount, visibleContainers.length - 1);
+  // Add a visit for "Today".
+  await addNormalizedVisit(uri("http://www.mozilla.org/"), nowObj, 0);
+  Assert.equal(dateContainer.childCount, visibleContainers.length);
+
+  dateContainer.containerOpen = false;
+  root.containerOpen = false;
+
+  // Cleanup.
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+}
+
+add_task(async function test_history_sidebar() {
+  await test_date_liveupdate(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_SITE_QUERY
+  );
+  await test_date_liveupdate(
+    Ci.nsINavHistoryQueryOptions.RESULTS_AS_DATE_QUERY
+  );
+
+  // The remaining views are
+  //   RESULTS_AS_URI + SORT_BY_VISITCOUNT_DESCENDING
+  //   ->  test_399266.js
+  //   RESULTS_AS_URI + SORT_BY_DATE_DESCENDING
+  //   ->  test_385397.js
+});
diff --git a/toolkit/components/places/tests/unit/test_import_mobile_bookmarks.js b/toolkit/components/places/tests/unit/test_import_mobile_bookmarks.js
new file mode 100644
index 0000000000..167b8786e4
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_import_mobile_bookmarks.js
@@ -0,0 +1,326 @@
+async function importFromFixture(fixture, replace) {
+  let cwd = do_get_cwd().path;
+  let path = PathUtils.join(cwd, fixture);
+
+  info(`Importing from ${path}`);
+  await BookmarkJSONUtils.importFromFile(path, { replace });
+  await PlacesTestUtils.promiseAsyncUpdates();
+}
+
+async function treeEquals(guid, expected, message) {
+  let root = await PlacesUtils.promiseBookmarksTree(guid);
+  let bookmarks = (function nodeToEntry(node) {
+    let entry = { guid: node.guid, index: node.index };
+    if (node.children) {
+      entry.children = node.children.map(nodeToEntry);
+    }
+    return entry;
+  })(root);
+
+  info(`Checking if ${guid} tree matches ${JSON.stringify(expected)}`);
+  info(`Got bookmarks tree for ${guid}: ${JSON.stringify(bookmarks)}`);
+
+  deepEqual(bookmarks, expected, message);
+}
+
+add_task(async function test_restore_mobile_bookmarks_root() {
+  await importFromFixture(
+    "mobile_bookmarks_root_import.json",
+    /* replace */ true
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [{ guid: "X6lUyOspVYwi", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            { guid: "_o8e1_zxTJFg", index: 0 },
+            { guid: "QCtSqkVYUbXB", index: 1 },
+          ],
+        },
+      ],
+    },
+    "Should restore mobile bookmarks from root"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_mobile_bookmarks_root() {
+  await importFromFixture(
+    "mobile_bookmarks_root_import.json",
+    /* replace */ false
+  );
+  await importFromFixture(
+    "mobile_bookmarks_root_merge.json",
+    /* replace */ false
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [
+            { guid: "X6lUyOspVYwi", index: 0 },
+            { guid: "Utodo9b0oVws", index: 1 },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            // The first two are in ..._import.json, the second two are in
+            // ..._merge.json
+            { guid: "_o8e1_zxTJFg", index: 0 },
+            { guid: "QCtSqkVYUbXB", index: 1 },
+            { guid: "a17yW6-nTxEJ", index: 2 },
+            { guid: "xV10h9Wi3FBM", index: 3 },
+          ],
+        },
+      ],
+    },
+    "Should merge bookmarks root contents"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_restore_mobile_bookmarks_folder() {
+  // This tests importing a mobile bookmarks folder with the annotation,
+  // and the old, random guid.
+  await importFromFixture(
+    "mobile_bookmarks_folder_import.json",
+    /* replace */ true
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [
+            { guid: "X6lUyOspVYwi", index: 0 },
+            { guid: "XF4yRP6bTuil", index: 1 },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+          children: [{ guid: "buy7711R3ZgE", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+          children: [{ guid: "KIa9iKZab2Z5", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            { guid: "_o8e1_zxTJFg", index: 0 },
+            { guid: "QCtSqkVYUbXB", index: 1 },
+          ],
+        },
+      ],
+    },
+    "Should restore mobile bookmark folder contents into mobile root"
+  );
+
+  let queryById = await PlacesUtils.bookmarks.fetch("XF4yRP6bTuil");
+  equal(
+    queryById.url.href,
+    `place:parent=${PlacesUtils.bookmarks.mobileGuid}`,
+    "Should rewrite mobile query to point to root GUID"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_mobile_bookmarks_folder() {
+  await importFromFixture(
+    "mobile_bookmarks_folder_import.json",
+    /* replace */ false
+  );
+  await importFromFixture(
+    "mobile_bookmarks_folder_merge.json",
+    /* replace */ false
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [
+            { guid: "X6lUyOspVYwi", index: 0 },
+            { guid: "XF4yRP6bTuil", index: 1 },
+            { guid: "Utodo9b0oVws", index: 2 },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+          children: [{ guid: "buy7711R3ZgE", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+          children: [{ guid: "KIa9iKZab2Z5", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            { guid: "_o8e1_zxTJFg", index: 0 },
+            { guid: "QCtSqkVYUbXB", index: 1 },
+            { guid: "a17yW6-nTxEJ", index: 2 },
+            { guid: "xV10h9Wi3FBM", index: 3 },
+          ],
+        },
+      ],
+    },
+    "Should merge bookmarks folder contents into mobile root"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_restore_multiple_bookmarks_folders() {
+  await importFromFixture(
+    "mobile_bookmarks_multiple_folders.json",
+    /* replace */ true
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [
+            { guid: "buy7711R3ZgE", index: 0 },
+            { guid: "F_LBgd1fS_uQ", index: 1 },
+            { guid: "oIpmQXMWsXvY", index: 2 },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+          children: [{ guid: "Utodo9b0oVws", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+          children: [{ guid: "xV10h9Wi3FBM", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            { guid: "a17yW6-nTxEJ", index: 0 },
+            { guid: "sSZ86WT9WbN3", index: 1 },
+          ],
+        },
+      ],
+    },
+    "Should restore multiple bookmarks folder contents into root"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_import_multiple_bookmarks_folders() {
+  await importFromFixture(
+    "mobile_bookmarks_root_import.json",
+    /* replace */ false
+  );
+  await importFromFixture(
+    "mobile_bookmarks_multiple_folders.json",
+    /* replace */ false
+  );
+
+  await treeEquals(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+      index: 0,
+      children: [
+        {
+          guid: PlacesUtils.bookmarks.menuGuid,
+          index: 0,
+          children: [
+            { guid: "X6lUyOspVYwi", index: 0 },
+            { guid: "buy7711R3ZgE", index: 1 },
+            { guid: "F_LBgd1fS_uQ", index: 2 },
+            { guid: "oIpmQXMWsXvY", index: 3 },
+          ],
+        },
+        {
+          guid: PlacesUtils.bookmarks.toolbarGuid,
+          index: 1,
+          children: [{ guid: "Utodo9b0oVws", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.unfiledGuid,
+          index: 3,
+          children: [{ guid: "xV10h9Wi3FBM", index: 0 }],
+        },
+        {
+          guid: PlacesUtils.bookmarks.mobileGuid,
+          index: 4,
+          children: [
+            { guid: "_o8e1_zxTJFg", index: 0 },
+            { guid: "QCtSqkVYUbXB", index: 1 },
+            { guid: "a17yW6-nTxEJ", index: 2 },
+            { guid: "sSZ86WT9WbN3", index: 3 },
+          ],
+        },
+      ],
+    },
+    "Should merge multiple mobile folders into root"
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
diff --git a/toolkit/components/places/tests/unit/test_isPageInDB.js b/toolkit/components/places/tests/unit/test_isPageInDB.js
new file mode 100644
index 0000000000..2eda125994
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_isPageInDB.js
@@ -0,0 +1,10 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+add_task(async function test_execute() {
+  var good_uri = uri("http://mozilla.com");
+  var bad_uri = uri("http://google.com");
+  await PlacesTestUtils.addVisits({ uri: good_uri });
+  Assert.ok(await PlacesTestUtils.isPageInDB(good_uri));
+  Assert.equal(false, await PlacesTestUtils.isPageInDB(bad_uri));
+});
diff --git a/toolkit/components/places/tests/unit/test_isURIVisited.js b/toolkit/components/places/tests/unit/test_isURIVisited.js
new file mode 100644
index 0000000000..cf3a8c139f
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_isURIVisited.js
@@ -0,0 +1,73 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests functionality of the isURIVisited API.
+
+const SCHEMES = {
+  "http://": true,
+  "https://": true,
+  "ftp://": true,
+  "file:///": true,
+  "about:": false,
+  // nsIIOService.newURI() can throw if e.g. the app knows about imap://
+  // but the account is not set up and so the URL is invalid for it.
+  //  "imap://": false,
+  "news://": false,
+  "mailbox:": false,
+  "moz-anno:favicon:http://": false,
+  "view-source:http://": false,
+  "chrome://browser/content/browser.xhtml?": false,
+  "resource://": false,
+  "data:,": false,
+  "javascript:": false,
+};
+
+add_task(async function test_isURIVisited() {
+  let history = Cc["@mozilla.org/browser/history;1"].getService(
+    Ci.mozIAsyncHistory
+  );
+
+  function visitsPromise(uri) {
+    return new Promise(resolve => {
+      history.isURIVisited(uri, (receivedURI, visited) => {
+        resolve([receivedURI, visited]);
+      });
+    });
+  }
+
+  for (let scheme in SCHEMES) {
+    info("Testing scheme " + scheme);
+    for (let t in PlacesUtils.history.TRANSITIONS) {
+      if (t == "EMBED") {
+        continue;
+      }
+      info("With transition " + t);
+      let aTransition = PlacesUtils.history.TRANSITIONS[t];
+
+      let aURI = Services.io.newURI(scheme + "mozilla.org/");
+
+      let [receivedURI1, visited1] = await visitsPromise(aURI);
+      Assert.ok(aURI.equals(receivedURI1));
+      Assert.ok(!visited1);
+
+      if (PlacesUtils.history.canAddURI(aURI)) {
+        await PlacesTestUtils.addVisits([
+          {
+            uri: aURI,
+            transition: aTransition,
+          },
+        ]);
+        info("Added visit for " + aURI.spec);
+      }
+
+      let [receivedURI2, visited2] = await visitsPromise(aURI);
+      Assert.ok(aURI.equals(receivedURI2));
+      Assert.equal(SCHEMES[scheme], visited2);
+
+      await PlacesUtils.history.clear();
+      let [receivedURI3, visited3] = await visitsPromise(aURI);
+      Assert.ok(aURI.equals(receivedURI3));
+      Assert.ok(!visited3);
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_isvisited.js b/toolkit/components/places/tests/unit/test_isvisited.js
new file mode 100644
index 0000000000..42bd913f55
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_isvisited.js
@@ -0,0 +1,69 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function test_execute() {
+  var referrer = uri("about:blank");
+
+  // add a http:// uri
+  var uri1 = uri("http://mozilla.com");
+  await PlacesTestUtils.addVisits({ uri: uri1, referrer });
+  await check_guid_for_uri(uri1);
+  Assert.ok(await PlacesUtils.history.hasVisits(uri1));
+
+  // add a https:// uri
+  var uri2 = uri("https://etrade.com");
+  await PlacesTestUtils.addVisits({ uri: uri2, referrer });
+  await check_guid_for_uri(uri2);
+  Assert.ok(await PlacesUtils.history.hasVisits(uri2));
+
+  // add a ftp:// uri
+  var uri3 = uri("ftp://ftp.mozilla.org");
+  await PlacesTestUtils.addVisits({ uri: uri3, referrer });
+  await check_guid_for_uri(uri3);
+  Assert.ok(await PlacesUtils.history.hasVisits(uri3));
+
+  // check if a nonexistent uri is visited
+  var uri4 = uri("http://foobarcheese.com");
+  Assert.equal(false, await PlacesUtils.history.hasVisits(uri4));
+
+  // check that certain schemes never show up as visited
+  // even if we attempt to add them to history
+  // see CanAddURI() in nsNavHistory.cpp
+  const URLS = [
+    "about:config",
+    "imap://cyrus.andrew.cmu.edu/archive.imap",
+    "news://new.mozilla.org/mozilla.dev.apps.firefox",
+    "mailbox:Inbox",
+    "moz-anno:favicon:http://mozilla.org/made-up-favicon",
+    "view-source:http://mozilla.org",
+    "chrome://browser/content/browser.xhtml",
+    "resource://gre-resources/hiddenWindow.html",
+    "data:,Hello%2C%20World!",
+    "javascript:alert('hello wolrd!');",
+    "http://localhost/" + "a".repeat(1984),
+  ];
+  for (let currentURL of URLS) {
+    try {
+      var cantAddUri = uri(currentURL);
+    } catch (e) {
+      // nsIIOService.newURI() can throw if e.g. our app knows about imap://
+      // but the account is not set up and so the URL is invalid for us.
+      // Note this in the log but ignore as it's not the subject of this test.
+      info("Could not construct URI for '" + currentURL + "'; ignoring");
+    }
+    if (cantAddUri) {
+      PlacesTestUtils.addVisits({ uri: cantAddUri, referrer }).then(
+        () => {
+          do_throw("Should not have added history for invalid URI.");
+        },
+        error => {
+          Assert.ok(error.message.includes("No items were added to history"));
+        }
+      );
+      Assert.equal(false, await PlacesUtils.history.hasVisits(cantAddUri));
+    }
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_keywords.js b/toolkit/components/places/tests/unit/test_keywords.js
new file mode 100644
index 0000000000..57e81515aa
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_keywords.js
@@ -0,0 +1,733 @@
+"use strict";
+
+ChromeUtils.defineESModuleGetters(this, {
+  Preferences: "resource://gre/modules/Preferences.sys.mjs",
+});
+
+async function check_keyword(aExpectExists, aHref, aKeyword, aPostData = null) {
+  // Check case-insensitivity.
+  aKeyword = aKeyword.toUpperCase();
+
+  let entry = await PlacesUtils.keywords.fetch(aKeyword);
+
+  Assert.deepEqual(
+    entry,
+    await PlacesUtils.keywords.fetch({ keyword: aKeyword })
+  );
+
+  if (aExpectExists) {
+    Assert.ok(!!entry, "A keyword should exist");
+    Assert.equal(entry.url.href, aHref);
+    Assert.equal(entry.postData, aPostData);
+    Assert.deepEqual(
+      entry,
+      await PlacesUtils.keywords.fetch({ keyword: aKeyword, url: aHref })
+    );
+    let entries = [];
+    await PlacesUtils.keywords.fetch({ url: aHref }, e => entries.push(e));
+    Assert.ok(
+      entries.some(
+        e => e.url.href == aHref && e.keyword == aKeyword.toLowerCase()
+      )
+    );
+  } else {
+    Assert.ok(
+      !entry || entry.url.href != aHref,
+      "The given keyword entry should not exist"
+    );
+    if (aHref) {
+      Assert.equal(
+        null,
+        await PlacesUtils.keywords.fetch({ keyword: aKeyword, url: aHref })
+      );
+    } else {
+      Assert.equal(
+        null,
+        await PlacesUtils.keywords.fetch({ keyword: aKeyword })
+      );
+    }
+  }
+}
+
+/**
+ * Polls the keywords cache waiting for the given keyword entry.
+ */
+async function promiseKeyword(keyword, expectedHref) {
+  let href = null;
+  do {
+    await new Promise(resolve => do_timeout(100, resolve));
+    let entry = await PlacesUtils.keywords.fetch(keyword);
+    if (entry) {
+      href = entry.url.href;
+    }
+  } while (href != expectedHref);
+}
+
+async function check_no_orphans() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.executeCached(
+    `SELECT id FROM moz_keywords k
+     WHERE NOT EXISTS (SELECT 1 FROM moz_places WHERE id = k.place_id)
+    `
+  );
+  Assert.equal(rows.length, 0);
+}
+
+function expectBookmarkNotifications() {
+  const observer = {
+    notifications: [],
+    _start() {
+      this._handle = this._handle.bind(this);
+      PlacesUtils.observers.addListener(
+        ["bookmark-keyword-changed"],
+        this._handle
+      );
+    },
+    _handle(events) {
+      for (const event of events) {
+        this.notifications.push({
+          type: event.type,
+          id: event.id,
+          itemType: event.itemType,
+          url: event.url,
+          guid: event.guid,
+          parentGuid: event.parentGuid,
+          keyword: event.keyword,
+          lastModified: new Date(event.lastModified),
+          source: event.source,
+          isTagging: event.isTagging,
+        });
+      }
+    },
+    check(expected) {
+      PlacesUtils.observers.removeListener(
+        ["bookmark-keyword-changed"],
+        this._handle
+      );
+      Assert.deepEqual(this.notifications, expected);
+    },
+  };
+  observer._start();
+  return observer;
+}
+
+add_task(async function test_invalid_input() {
+  Assert.throws(() => PlacesUtils.keywords.fetch(null), /Invalid keyword/);
+  Assert.throws(() => PlacesUtils.keywords.fetch(5), /Invalid keyword/);
+  Assert.throws(() => PlacesUtils.keywords.fetch(undefined), /Invalid keyword/);
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ keyword: null }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ keyword: {} }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ keyword: 5 }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({}),
+    /At least keyword or url must be provided/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ keyword: "test" }, "test"),
+    /onResult callback must be a valid function/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ url: "test" }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ url: {} }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ url: null }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.fetch({ url: "" }),
+    /is not a valid URL/
+  );
+
+  Assert.throws(
+    () => PlacesUtils.keywords.insert(null),
+    /Input should be a valid object/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert("test"),
+    /Input should be a valid object/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert(undefined),
+    /Input should be a valid object/
+  );
+  Assert.throws(() => PlacesUtils.keywords.insert({}), /Invalid keyword/);
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: null }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: 5 }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "" }),
+    /Invalid keyword/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", postData: 5 }),
+    /Invalid POST data/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", postData: {} }),
+    /Invalid POST data/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test" }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", url: 5 }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", url: "" }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", url: null }),
+    /is not a valid URL/
+  );
+  Assert.throws(
+    () => PlacesUtils.keywords.insert({ keyword: "test", url: "mozilla" }),
+    /is not a valid URL/
+  );
+
+  Assert.throws(() => PlacesUtils.keywords.remove(null), /Invalid keyword/);
+  Assert.throws(() => PlacesUtils.keywords.remove(""), /Invalid keyword/);
+  Assert.throws(() => PlacesUtils.keywords.remove(5), /Invalid keyword/);
+});
+
+add_task(async function test_addKeyword() {
+  await check_keyword(false, "http://example.com/", "keyword");
+  let fc = await foreign_count("http://example.com/");
+  let observer = expectBookmarkNotifications();
+
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+  observer.check([]);
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 1); // +1 keyword
+
+  // Now remove the keyword.
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.remove("keyword");
+  observer.check([]);
+
+  await check_keyword(false, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc); // -1 keyword
+
+  // Check using URL.
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: new URL("http://example.com/"),
+  });
+  await check_keyword(true, "http://example.com/", "keyword");
+  await PlacesUtils.keywords.remove("keyword");
+  await check_keyword(false, "http://example.com/", "keyword");
+
+  await check_no_orphans();
+});
+
+add_task(async function test_addBookmarkAndKeyword() {
+  let timerPrecision = Preferences.get("privacy.reduceTimerPrecision");
+  Preferences.set("privacy.reduceTimerPrecision", false);
+
+  registerCleanupFunction(function () {
+    Preferences.set("privacy.reduceTimerPrecision", timerPrecision);
+  });
+
+  await check_keyword(false, "http://example.com/", "keyword");
+  let fc = await foreign_count("http://example.com/");
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+
+  let observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark.guid),
+      itemType: bookmark.type,
+      url: bookmark.url,
+      guid: bookmark.guid,
+      parentGuid: bookmark.parentGuid,
+      keyword: "keyword",
+      lastModified: new Date(bookmark.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // +1 bookmark +1 keyword
+
+  // Now remove the keyword.
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.remove("keyword");
+
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark.guid),
+      itemType: bookmark.type,
+      url: bookmark.url,
+      guid: bookmark.guid,
+      parentGuid: bookmark.parentGuid,
+      keyword: "",
+      lastModified: new Date(bookmark.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  await check_keyword(false, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 1); // -1 keyword
+
+  // Add again the keyword, then remove the bookmark.
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+  // the notification is synchronous but the removal process is async.
+  // Unfortunately there's nothing explicit we can wait for.
+  // eslint-disable-next-line no-empty
+  while (await foreign_count("http://example.com/")) {}
+  // We don't get any itemChanged notification since the bookmark has been
+  // removed already.
+  observer.check([]);
+
+  await check_keyword(false, "http://example.com/", "keyword");
+
+  await check_no_orphans();
+});
+
+add_task(async function test_addKeywordToURIHavingKeyword() {
+  await check_keyword(false, "http://example.com/", "keyword");
+  let fc = await foreign_count("http://example.com/");
+
+  let observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+  observer.check([]);
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 1); // +1 keyword
+
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword2",
+    url: "http://example.com/",
+    postData: "test=1",
+  });
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  await check_keyword(true, "http://example.com/", "keyword2", "test=1");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // +1 keyword
+  let entries = [];
+  let entry = await PlacesUtils.keywords.fetch(
+    { url: "http://example.com/" },
+    e => entries.push(e)
+  );
+  Assert.equal(entries.length, 2);
+  Assert.deepEqual(entries[0], entry);
+
+  // Now remove the keywords.
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.remove("keyword");
+  await PlacesUtils.keywords.remove("keyword2");
+  observer.check([]);
+
+  await check_keyword(false, "http://example.com/", "keyword");
+  await check_keyword(false, "http://example.com/", "keyword2");
+  Assert.equal(await foreign_count("http://example.com/"), fc); // -1 keyword
+
+  await check_no_orphans();
+});
+
+add_task(async function test_addBookmarkToURIHavingKeyword() {
+  await check_keyword(false, "http://example.com/", "keyword");
+  let fc = await foreign_count("http://example.com/");
+  let observer = expectBookmarkNotifications();
+
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+  observer.check([]);
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 1); // +1 keyword
+
+  observer = expectBookmarkNotifications();
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // +1 bookmark
+  observer.check([]);
+
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+  // the notification is synchronous but the removal process is async.
+  // Unfortunately there's nothing explicit we can wait for.
+  // eslint-disable-next-line no-empty
+  while (await foreign_count("http://example.com/")) {}
+  // We don't get any itemChanged notification since the bookmark has been
+  // removed already.
+  observer.check([]);
+
+  await check_keyword(false, "http://example.com/", "keyword");
+
+  await check_no_orphans();
+});
+
+add_task(async function test_sameKeywordDifferentURL() {
+  let fc1 = await foreign_count("http://example1.com/");
+  let bookmark1 = await PlacesUtils.bookmarks.insert({
+    url: "http://example1.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  let fc2 = await foreign_count("http://example2.com/");
+  let bookmark2 = await PlacesUtils.bookmarks.insert({
+    url: "http://example2.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example1.com/",
+  });
+
+  await check_keyword(true, "http://example1.com/", "keyword");
+  Assert.equal(await foreign_count("http://example1.com/"), fc1 + 2); // +1 bookmark +1 keyword
+  await check_keyword(false, "http://example2.com/", "keyword");
+  Assert.equal(await foreign_count("http://example2.com/"), fc2 + 1); // +1 bookmark
+
+  // Assign the same keyword to another url.
+  let observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example2.com/",
+  });
+
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark1.guid),
+      itemType: bookmark1.type,
+      url: bookmark1.url,
+      guid: bookmark1.guid,
+      parentGuid: bookmark1.parentGuid,
+      keyword: "",
+      lastModified: new Date(bookmark1.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark2.guid),
+      itemType: bookmark2.type,
+      url: bookmark2.url,
+      guid: bookmark2.guid,
+      parentGuid: bookmark2.parentGuid,
+      keyword: "keyword",
+      lastModified: new Date(bookmark2.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  await check_keyword(false, "http://example1.com/", "keyword");
+  Assert.equal(await foreign_count("http://example1.com/"), fc1 + 1); // -1 keyword
+  await check_keyword(true, "http://example2.com/", "keyword");
+  Assert.equal(await foreign_count("http://example2.com/"), fc2 + 2); // +1 keyword
+
+  // Now remove the keyword.
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.remove("keyword");
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark2.guid),
+      itemType: bookmark2.type,
+      url: bookmark2.url,
+      guid: bookmark2.guid,
+      parentGuid: bookmark2.parentGuid,
+      keyword: "",
+      lastModified: new Date(bookmark2.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  await check_keyword(false, "http://example1.com/", "keyword");
+  await check_keyword(false, "http://example2.com/", "keyword");
+  Assert.equal(await foreign_count("http://example1.com/"), fc1 + 1);
+  Assert.equal(await foreign_count("http://example2.com/"), fc2 + 1); // -1 keyword
+
+  await PlacesUtils.bookmarks.remove(bookmark1);
+  await PlacesUtils.bookmarks.remove(bookmark2);
+  Assert.equal(await foreign_count("http://example1.com/"), fc1); // -1 bookmark
+  // eslint-disable-next-line no-empty
+  while (await foreign_count("http://example2.com/")) {} // -1 keyword
+
+  await check_no_orphans();
+});
+
+add_task(async function test_sameURIDifferentKeyword() {
+  let fc = await foreign_count("http://example.com/");
+
+  let observer = expectBookmarkNotifications();
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // +1 bookmark +1 keyword
+
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark.guid),
+      itemType: bookmark.type,
+      url: bookmark.url,
+      guid: bookmark.guid,
+      parentGuid: bookmark.parentGuid,
+      keyword: "keyword",
+      lastModified: new Date(bookmark.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword2",
+    url: "http://example.com/",
+  });
+  await check_keyword(false, "http://example.com/", "keyword");
+  await check_keyword(true, "http://example.com/", "keyword2");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // -1 keyword +1 keyword
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark.guid),
+      itemType: bookmark.type,
+      url: bookmark.url,
+      guid: bookmark.guid,
+      parentGuid: bookmark.parentGuid,
+      keyword: "keyword2",
+      lastModified: new Date(bookmark.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  // Now remove the bookmark.
+  await PlacesUtils.bookmarks.remove(bookmark);
+  // eslint-disable-next-line no-empty
+  while (await foreign_count("http://example.com/")) {}
+  await check_keyword(false, "http://example.com/", "keyword");
+  await check_keyword(false, "http://example.com/", "keyword2");
+
+  await check_no_orphans();
+});
+
+add_task(async function test_deleteKeywordMultipleBookmarks() {
+  let fc = await foreign_count("http://example.com/");
+
+  let observer = expectBookmarkNotifications();
+  let bookmark1 = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  let bookmark2 = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+  });
+
+  await check_keyword(true, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 3); // +2 bookmark +1 keyword
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark2.guid),
+      itemType: bookmark2.type,
+      url: bookmark2.url,
+      guid: bookmark2.guid,
+      parentGuid: bookmark2.parentGuid,
+      keyword: "keyword",
+      lastModified: new Date(bookmark2.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark1.guid),
+      itemType: bookmark1.type,
+      url: bookmark1.url,
+      guid: bookmark1.guid,
+      parentGuid: bookmark1.parentGuid,
+      keyword: "keyword",
+      lastModified: new Date(bookmark1.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  observer = expectBookmarkNotifications();
+  await PlacesUtils.keywords.remove("keyword");
+  await check_keyword(false, "http://example.com/", "keyword");
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // -1 keyword
+  observer.check([
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark2.guid),
+      itemType: bookmark2.type,
+      url: bookmark2.url,
+      guid: bookmark2.guid,
+      parentGuid: bookmark2.parentGuid,
+      keyword: "",
+      lastModified: new Date(bookmark2.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+    {
+      type: "bookmark-keyword-changed",
+      id: await PlacesUtils.promiseItemId(bookmark1.guid),
+      itemType: bookmark1.type,
+      url: bookmark1.url,
+      guid: bookmark1.guid,
+      parentGuid: bookmark1.parentGuid,
+      keyword: "",
+      lastModified: new Date(bookmark1.lastModified),
+      source: Ci.nsINavBookmarksService.SOURCE_DEFAULT,
+      isTagging: false,
+    },
+  ]);
+
+  // Now remove the bookmarks.
+  await PlacesUtils.bookmarks.remove(bookmark1);
+  await PlacesUtils.bookmarks.remove(bookmark2);
+  Assert.equal(await foreign_count("http://example.com/"), fc); // -2 bookmarks
+
+  await check_no_orphans();
+});
+
+add_task(async function test_multipleKeywordsSamePostData() {
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example.com/",
+    postData: "postData1",
+  });
+  await check_keyword(true, "http://example.com/", "keyword", "postData1");
+  // Add another keyword with same postData, should fail.
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword2",
+    url: "http://example.com/",
+    postData: "postData1",
+  });
+  await check_keyword(false, "http://example.com/", "keyword", "postData1");
+  await check_keyword(true, "http://example.com/", "keyword2", "postData1");
+
+  await PlacesUtils.keywords.remove("keyword2");
+
+  await check_no_orphans();
+});
+
+add_task(async function test_bookmarkURLChange() {
+  let fc1 = await foreign_count("http://example1.com/");
+  let fc2 = await foreign_count("http://example2.com/");
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    url: "http://example1.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  await PlacesUtils.keywords.insert({
+    keyword: "keyword",
+    url: "http://example1.com/",
+  });
+
+  await check_keyword(true, "http://example1.com/", "keyword");
+  Assert.equal(await foreign_count("http://example1.com/"), fc1 + 2); // +1 bookmark +1 keyword
+
+  await PlacesUtils.bookmarks.update({
+    guid: bookmark.guid,
+    url: "http://example2.com/",
+  });
+  await promiseKeyword("keyword", "http://example2.com/");
+
+  await check_keyword(false, "http://example1.com/", "keyword");
+  await check_keyword(true, "http://example2.com/", "keyword");
+  Assert.equal(await foreign_count("http://example1.com/"), fc1); // -1 bookmark -1 keyword
+  Assert.equal(await foreign_count("http://example2.com/"), fc2 + 2); // +1 bookmark +1 keyword
+});
+
+add_task(async function test_tagDoesntPreventKeywordRemoval() {
+  await check_keyword(false, "http://example.com/", "example");
+  let fc = await foreign_count("http://example.com/");
+
+  let httpBookmark = await PlacesUtils.bookmarks.insert({
+    url: "http://example.com/",
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+  });
+  Assert.equal(await foreign_count("http://example.com/"), fc + 1); // +1 bookmark
+
+  PlacesUtils.tagging.tagURI(uri("http://example.com/"), ["example_tag"]);
+  Assert.equal(await foreign_count("http://example.com/"), fc + 2); // +1 bookmark +1 tag
+
+  await PlacesUtils.keywords.insert({
+    keyword: "example",
+    url: "http://example.com/",
+  });
+  Assert.equal(await foreign_count("http://example.com/"), fc + 3); // +1 bookmark +1 tag +1 keyword
+
+  await check_keyword(true, "http://example.com/", "example");
+
+  await PlacesUtils.bookmarks.remove(httpBookmark);
+
+  await TestUtils.waitForCondition(
+    async () =>
+      !(await PlacesUtils.bookmarks.fetch({ url: "http://example.com/" })),
+    "Wait for bookmark to be removed"
+  );
+
+  await check_keyword(false, "http://example.com/", "example");
+  Assert.equal(await foreign_count("http://example.com/"), fc); // bookmark, keyword, and tag should all have been removed
+
+  await check_no_orphans();
+});
diff --git a/toolkit/components/places/tests/unit/test_lastModified.js b/toolkit/components/places/tests/unit/test_lastModified.js
new file mode 100644
index 0000000000..dc44814548
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_lastModified.js
@@ -0,0 +1,78 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function assert_date_eq(a, b) {
+  if (typeof a != "number") {
+    a = PlacesUtils.toPRTime(a);
+  }
+  if (typeof b != "number") {
+    b = PlacesUtils.toPRTime(b);
+  }
+  Assert.equal(a, b, "The dates should match");
+}
+
+/**
+ * Test that inserting a new bookmark will set lastModified to the same
+ * values as dateAdded.
+ */
+add_task(async function test_bookmarkLastModified() {
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://www.mozilla.org/",
+    title: "itemTitle",
+  });
+
+  let guid = bookmark.guid;
+
+  // Check the bookmark from the database.
+  bookmark = await PlacesUtils.bookmarks.fetch(guid);
+
+  let dateAdded = PlacesUtils.toPRTime(bookmark.dateAdded);
+  assert_date_eq(dateAdded, bookmark.lastModified);
+
+  // Change lastModified, then change dateAdded.  LastModified should be set
+  // to the new dateAdded.
+  // This could randomly fail on virtual machines due to timing issues, so
+  // we manually increase the time value.  See bug 500640 for details.
+  await PlacesUtils.bookmarks.update({
+    guid,
+    lastModified: PlacesUtils.toDate(dateAdded + 1000),
+  });
+
+  bookmark = await PlacesUtils.bookmarks.fetch(guid);
+
+  assert_date_eq(bookmark.lastModified, dateAdded + 1000);
+  Assert.ok(
+    bookmark.dateAdded < bookmark.lastModified,
+    "Date added should be earlier than last modified."
+  );
+
+  await PlacesUtils.bookmarks.update({
+    guid,
+    dateAdded: PlacesUtils.toDate(dateAdded + 2000),
+  });
+
+  bookmark = await PlacesUtils.bookmarks.fetch(guid);
+
+  assert_date_eq(bookmark.dateAdded, dateAdded + 2000);
+  assert_date_eq(bookmark.dateAdded, bookmark.lastModified);
+
+  // If dateAdded is set to older than lastModified, then we shouldn't
+  // update lastModified to keep sync happy.
+  let origLastModified = bookmark.lastModified;
+
+  await PlacesUtils.bookmarks.update({
+    guid,
+    dateAdded: PlacesUtils.toDate(dateAdded - 10000),
+  });
+
+  bookmark = await PlacesUtils.bookmarks.fetch(guid);
+
+  assert_date_eq(bookmark.dateAdded, dateAdded - 10000);
+  assert_date_eq(bookmark.lastModified, origLastModified);
+
+  await PlacesUtils.bookmarks.remove(guid);
+});
diff --git a/toolkit/components/places/tests/unit/test_markpageas.js b/toolkit/components/places/tests/unit/test_markpageas.js
new file mode 100644
index 0000000000..03080f4af6
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_markpageas.js
@@ -0,0 +1,48 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var gVisits = [
+  { url: "http://www.mozilla.com/", transition: TRANSITION_TYPED },
+  { url: "http://www.google.com/", transition: TRANSITION_BOOKMARK },
+  { url: "http://www.espn.com/", transition: TRANSITION_LINK },
+];
+
+add_task(async function test_execute() {
+  let completionPromise = new Promise(resolveCompletionPromise => {
+    let visitCount = 0;
+    function listener(aEvents) {
+      Assert.equal(aEvents.length, 1, "Right number of visits notified");
+      Assert.equal(aEvents[0].type, "page-visited");
+      let event = aEvents[0];
+      Assert.equal(event.url, gVisits[visitCount].url);
+      Assert.equal(event.transitionType, gVisits[visitCount].transition);
+      visitCount++;
+
+      if (visitCount == gVisits.length) {
+        resolveCompletionPromise();
+        PlacesObservers.removeListener(["page-visited"], listener);
+      }
+    }
+    PlacesObservers.addListener(["page-visited"], listener);
+  });
+
+  for (var visit of gVisits) {
+    if (visit.transition == TRANSITION_TYPED) {
+      PlacesUtils.history.markPageAsTyped(uri(visit.url));
+    } else if (visit.transition == TRANSITION_BOOKMARK) {
+      PlacesUtils.history.markPageAsFollowedBookmark(uri(visit.url));
+    } else {
+      // because it is a top level visit with no referrer,
+      // it will result in TRANSITION_LINK
+    }
+    await PlacesTestUtils.addVisits({
+      uri: uri(visit.url),
+      transition: visit.transition,
+    });
+  }
+
+  await completionPromise;
+});
diff --git a/toolkit/components/places/tests/unit/test_metadata.js b/toolkit/components/places/tests/unit/test_metadata.js
new file mode 100644
index 0000000000..f202b7f405
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_metadata.js
@@ -0,0 +1,253 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+add_task(async function test_metadata() {
+  await PlacesUtils.metadata.set("test/integer", 123);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/integer"),
+    123,
+    "Should store new integer value"
+  );
+
+  await PlacesUtils.metadata.set("test/double", 123.45);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/double"),
+    123.45,
+    "Should store new double value"
+  );
+  await PlacesUtils.metadata.set("test/double", 567.89);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/double"),
+    567.89,
+    "Should update existing double value"
+  );
+
+  await PlacesUtils.metadata.set("test/boolean", false);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/boolean"),
+    false,
+    "Should store new Boolean value"
+  );
+  await PlacesUtils.metadata.set("test/boolean", true);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/boolean"),
+    true,
+    "Should update existing Boolean value"
+  );
+
+  await PlacesUtils.metadata.set("test/string", "hi");
+  Assert.equal(
+    await PlacesUtils.metadata.get("test/string"),
+    "hi",
+    "Should store new string value"
+  );
+  await PlacesUtils.metadata.cache.clear();
+  Assert.equal(
+    await PlacesUtils.metadata.get("test/string"),
+    "hi",
+    "Should return string value after clearing cache"
+  );
+
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/nonexistent"),
+    /No data stored for key test\/nonexistent/,
+    "Should reject for a non-existent key and no default value."
+  );
+  Assert.equal(
+    await PlacesUtils.metadata.get("test/nonexistent", "defaultValue"),
+    "defaultValue",
+    "Should return the default value for a non-existent key."
+  );
+
+  // Values are untyped; it's OK to store a value of a different type for the
+  // same key.
+  await PlacesUtils.metadata.set("test/string", 111);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/string"),
+    111,
+    "Should replace string with integer"
+  );
+  await PlacesUtils.metadata.set("test/string", null);
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/string"),
+    /No data stored for key test\/string/,
+    "Should clear value when setting to NULL"
+  );
+
+  await PlacesUtils.metadata.delete("test/string", "test/boolean");
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/string"),
+    /No data stored for key test\/string/,
+    "Should delete string value"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/boolean"),
+    /No data stored for key test\/boolean/,
+    "Should delete Boolean value"
+  );
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("test/integer"),
+    123,
+    "Should keep undeleted integer value"
+  );
+
+  await PlacesTestUtils.clearMetadata();
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/integer"),
+    /No data stored for key test\/integer/,
+    "Should clear integer value"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/double"),
+    /No data stored for key test\/double/,
+    "Should clear double value"
+  );
+});
+
+add_task(async function test_metadata_canonical_keys() {
+  await PlacesUtils.metadata.set("Test/Integer", 123);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("tEsT/integer"),
+    123,
+    "New keys should be case-insensitive"
+  );
+  await PlacesUtils.metadata.set("test/integer", 456);
+  Assert.strictEqual(
+    await PlacesUtils.metadata.get("TEST/INTEGER"),
+    456,
+    "Existing keys should be case-insensitive"
+  );
+
+  await Assert.rejects(
+    PlacesUtils.metadata.set("", 123),
+    /Invalid metadata key/,
+    "Should reject empty keys"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.get(123),
+    /Invalid metadata key/,
+    "Should reject numeric keys"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.delete(true),
+    /Invalid metadata key/,
+    "Should reject Boolean keys"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.set({}),
+    /Invalid metadata key/,
+    "Should reject object keys"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.get(null),
+    /Invalid metadata key/,
+    "Should reject null keys"
+  );
+  await Assert.rejects(
+    PlacesUtils.metadata.delete("!@#$"),
+    /Invalid metadata key/,
+    "Should reject keys with invalid characters"
+  );
+});
+
+add_task(async function test_metadata_blobs() {
+  let blob = new Uint8Array([1, 2, 3]);
+  await PlacesUtils.metadata.set("test/blob", blob);
+
+  let sameBlob = await PlacesUtils.metadata.get("test/blob");
+  Assert.equal(
+    ChromeUtils.getClassName(sameBlob),
+    "Uint8Array",
+    "Should cache typed array for blob value"
+  );
+  Assert.deepEqual(sameBlob, blob, "Should store new blob value");
+
+  info("Remove blob from cache");
+  await PlacesUtils.metadata.cache.clear();
+
+  let newBlob = await PlacesUtils.metadata.get("test/blob");
+  Assert.equal(
+    ChromeUtils.getClassName(newBlob),
+    "Uint8Array",
+    "Should inflate blob into typed array"
+  );
+  Assert.deepEqual(
+    newBlob,
+    blob,
+    "Should return same blob after clearing cache"
+  );
+
+  await PlacesTestUtils.clearMetadata();
+});
+
+add_task(async function test_metadata_arrays() {
+  let array = [1, 2, 3, "\u2713 \u00E0 la mode"];
+  await PlacesUtils.metadata.set("test/array", array);
+
+  let sameArray = await PlacesUtils.metadata.get("test/array");
+  Assert.ok(Array.isArray(sameArray), "Should cache array for array value");
+  Assert.deepEqual(sameArray, array, "Should store new array value");
+
+  info("Remove array from cache");
+  await PlacesUtils.metadata.cache.clear();
+
+  let newArray = await PlacesUtils.metadata.get("test/array");
+  Assert.ok(Array.isArray(newArray), "Should inflate into array");
+  Assert.deepEqual(
+    newArray,
+    array,
+    "Should return same array after clearing cache"
+  );
+
+  await PlacesTestUtils.clearMetadata();
+});
+
+add_task(async function test_metadata_objects() {
+  let object = { foo: 123, bar: "test", meow: "\u2713 \u00E0 la mode" };
+  await PlacesUtils.metadata.set("test/object", object);
+
+  let sameObject = await PlacesUtils.metadata.get("test/object");
+  Assert.equal(
+    typeof sameObject,
+    "object",
+    "Should cache object for object value"
+  );
+  Assert.deepEqual(sameObject, object, "Should store new object value");
+
+  info("Remove object from cache");
+  await PlacesUtils.metadata.cache.clear();
+
+  let newObject = await PlacesUtils.metadata.get("test/object");
+  Assert.equal(typeof newObject, "object", "Should inflate into object");
+  Assert.deepEqual(
+    newObject,
+    object,
+    "Should return same object after clearing cache"
+  );
+
+  await PlacesTestUtils.clearMetadata();
+});
+
+add_task(async function test_metadata_unparsable() {
+  await PlacesUtils.withConnectionWrapper("test_medata", db => {
+    let data = PlacesUtils.metadata._base64Encode("{hjjkhj}");
+
+    return db.execute(`
+      INSERT INTO moz_meta (key, value)
+      VALUES ("test/unparsable", "data:application/json;base64,${data}")
+    `);
+  });
+
+  await Assert.rejects(
+    PlacesUtils.metadata.get("test/unparsable"),
+    /SyntaxError: JSON.parse/,
+    "Should reject for an unparsable value with no default"
+  );
+  Assert.deepEqual(
+    await PlacesUtils.metadata.get("test/unparsable", { foo: 1 }),
+    { foo: 1 },
+    "Should return the default when encountering an unparsable value."
+  );
+
+  await PlacesTestUtils.clearMetadata();
+});
diff --git a/toolkit/components/places/tests/unit/test_missing_builtin_folders.js b/toolkit/components/places/tests/unit/test_missing_builtin_folders.js
new file mode 100644
index 0000000000..826cc37ff7
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_missing_builtin_folders.js
@@ -0,0 +1,123 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This file tests that a missing built-in folders (child of root) are correctly
+ * fixed when the database is loaded.
+ */
+
+const ALL_ROOT_GUIDS = [
+  PlacesUtils.bookmarks.menuGuid,
+  PlacesUtils.bookmarks.unfiledGuid,
+  PlacesUtils.bookmarks.toolbarGuid,
+  PlacesUtils.bookmarks.tagsGuid,
+  PlacesUtils.bookmarks.mobileGuid,
+];
+
+const INITIAL_ROOT_GUIDS = [
+  PlacesUtils.bookmarks.menuGuid,
+  PlacesUtils.bookmarks.tagsGuid,
+  PlacesUtils.bookmarks.unfiledGuid,
+];
+
+add_task(async function setup() {
+  // This file has the toolbar and mobile folders missing.
+  await setupPlacesDatabase("missingBuiltIn.sqlite");
+
+  // Check database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+
+  let rows = await db.execute(
+    `
+    SELECT guid FROM moz_bookmarks
+    WHERE parent = (SELECT id from moz_bookmarks WHERE guid = :guid)
+  `,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+    }
+  );
+
+  let guids = rows.map(row => row.getResultByName("guid"));
+  Assert.deepEqual(
+    guids,
+    INITIAL_ROOT_GUIDS,
+    "Initial database should have only the expected GUIDs"
+  );
+
+  await db.close();
+});
+
+add_task(async function test_database_recreates_roots() {
+  Assert.ok(
+    PlacesUtils.history.databaseStatus ==
+      PlacesUtils.history.DATABASE_STATUS_OK ||
+      PlacesUtils.history.databaseStatus ==
+        PlacesUtils.history.DATABASE_STATUS_UPGRADED,
+    "Should successfully access the database for the first time"
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rootId = await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.rootGuid);
+  for (let guid of ALL_ROOT_GUIDS) {
+    let rows = await db.execute(
+      `
+      SELECT id, parent FROM moz_bookmarks
+      WHERE guid = :guid
+    `,
+      { guid }
+    );
+
+    Assert.equal(rows.length, 1, "Should have exactly one row for the root");
+
+    Assert.equal(
+      rows[0].getResultByName("parent"),
+      rootId,
+      "Should have been created with the correct parent"
+    );
+
+    let root = await PlacesUtils.bookmarks.fetch(guid);
+
+    Assert.equal(root.guid, guid, "GUIDs should match");
+    Assert.equal(
+      root.parentGuid,
+      PlacesUtils.bookmarks.rootGuid,
+      "Should have the correct parent GUID"
+    );
+    Assert.equal(
+      root.type,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      "Should have the correct type"
+    );
+
+    let id = rows[0].getResultByName("id");
+    Assert.equal(
+      await PlacesUtils.promiseItemId(guid),
+      id,
+      "Should return the correct id from promiseItemId"
+    );
+    Assert.equal(
+      await PlacesUtils.promiseItemGuid(id),
+      guid,
+      "Should return the correct guid from promiseItemGuid"
+    );
+  }
+
+  let rows = await db.execute(
+    `
+    SELECT 1 FROM moz_bookmarks
+    WHERE parent = (SELECT id from moz_bookmarks WHERE guid = :guid)
+  `,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+    }
+  );
+
+  Assert.equal(
+    rows.length,
+    ALL_ROOT_GUIDS.length,
+    "Root folder should have the expected number of children"
+  );
+});
diff --git a/toolkit/components/places/tests/unit/test_missing_root_folder.js b/toolkit/components/places/tests/unit/test_missing_root_folder.js
new file mode 100644
index 0000000000..f6f1bb19ef
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_missing_root_folder.js
@@ -0,0 +1,109 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This file tests that a missing root folder is correctly fixed when the
+ * database is loaded.
+ */
+
+const ALL_ROOT_GUIDS = [
+  PlacesUtils.bookmarks.menuGuid,
+  PlacesUtils.bookmarks.unfiledGuid,
+  PlacesUtils.bookmarks.toolbarGuid,
+  PlacesUtils.bookmarks.tagsGuid,
+  PlacesUtils.bookmarks.mobileGuid,
+];
+
+add_task(async function setup() {
+  // This file has no root folder.
+  await setupPlacesDatabase("noRoot.sqlite");
+
+  // Check database contents to be migrated.
+  let path = PathUtils.join(PathUtils.profileDir, DB_FILENAME);
+  let db = await Sqlite.openConnection({ path });
+
+  let rows = await db.execute(
+    `
+    SELECT guid FROM moz_bookmarks
+    WHERE guid = :guid
+  `,
+    {
+      guid: PlacesUtils.bookmarks.rootGuid,
+    }
+  );
+
+  Assert.equal(rows.length, 0, "Root folder should not exist");
+
+  await db.close();
+});
+
+add_task(async function test_database_recreates_roots() {
+  Assert.ok(
+    PlacesUtils.history.databaseStatus ==
+      PlacesUtils.history.DATABASE_STATUS_OK ||
+      PlacesUtils.history.databaseStatus ==
+        PlacesUtils.history.DATABASE_STATUS_UPGRADED,
+    "Should successfully access the database for the first time"
+  );
+
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let rows = await db.execute(
+    `
+    SELECT id, parent, type FROM moz_bookmarks
+    WHERE guid = :guid
+  `,
+    { guid: PlacesUtils.bookmarks.rootGuid }
+  );
+
+  Assert.equal(rows.length, 1, "Should have added exactly one root");
+  Assert.greaterOrEqual(
+    rows[0].getResultByName("id"),
+    1,
+    "Should have a valid root Id"
+  );
+  Assert.equal(
+    rows[0].getResultByName("parent"),
+    0,
+    "Should have a parent of id 0"
+  );
+  Assert.equal(
+    rows[0].getResultByName("type"),
+    PlacesUtils.bookmarks.TYPE_FOLDER,
+    "Should have a type of folder"
+  );
+
+  let id = rows[0].getResultByName("id");
+  Assert.equal(
+    await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.rootGuid),
+    id,
+    "Should return the correct id from promiseItemId"
+  );
+  Assert.equal(
+    await PlacesUtils.promiseItemGuid(id),
+    PlacesUtils.bookmarks.rootGuid,
+    "Should return the correct guid from promiseItemGuid"
+  );
+
+  // Note: Currently we do not fix the parent of the folders on initial startup.
+  // There is a maintenance task that will do it, hence we don't check the parents
+  // here, just that the built-in folders correctly exist and haven't been
+  // duplicated.
+  for (let guid of ALL_ROOT_GUIDS) {
+    rows = await db.execute(
+      `
+      SELECT id FROM moz_bookmarks
+      WHERE guid = :guid
+    `,
+      { guid }
+    );
+
+    Assert.equal(rows.length, 1, "Should have exactly one row for the root");
+
+    let root = await PlacesUtils.bookmarks.fetch(guid);
+
+    Assert.equal(root.guid, guid, "GUIDs should match");
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_multi_observation.js b/toolkit/components/places/tests/unit/test_multi_observation.js
new file mode 100644
index 0000000000..d56eabf8f1
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_multi_observation.js
@@ -0,0 +1,384 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Test whether registered listener capture proper events.
+// We test the following combinations.
+// * Listener: listen to single/multi event(s)
+// * Event: fire single/multi type of event(s)
+// * Timing: fire event(s) at same time/separately
+// And also test notifying empty events.
+
+add_task(async () => {
+  info("Test for listening to single event and firing single event");
+
+  const observer = startObservation(["page-visited"]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "test",
+      url: "http://example.com/test",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/test",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async () => {
+  info("Test for listening to multi events with firing single event");
+
+  const observer = startObservation(["page-visited", "page-title-changed"]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "test",
+      url: "http://example.com/test",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/test",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async () => {
+  info(
+    "Test for listening to single event with firing multi events at same time"
+  );
+
+  const vistedObserver = startObservation(["page-visited"]);
+  const titleChangedObserver = startObservation(["page-title-changed"]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "will change",
+      url: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+    {
+      title: "changed",
+      url: "http://example.com/title",
+      referrer: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+    {
+      title: "another",
+      url: "http://example.com/another",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedVisitedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+      {
+        type: "page-visited",
+        url: "http://example.com/another",
+      },
+    ],
+  ];
+  assertFiredEvents(vistedObserver.firedEvents, expectedVisitedFiredEvents);
+
+  const expectedTitleChangedFiredEvents = [
+    [
+      {
+        type: "page-title-changed",
+        url: "http://example.com/title",
+        title: "changed",
+      },
+    ],
+  ];
+  assertFiredEvents(
+    titleChangedObserver.firedEvents,
+    expectedTitleChangedFiredEvents
+  );
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async () => {
+  info(
+    "Test for listening to single event with firing multi events separately"
+  );
+
+  const observer = startObservation(["page-visited"]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "test",
+      url: "http://example.com/test",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "a folder",
+  });
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "another",
+      url: "http://example.com/another",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/test",
+      },
+    ],
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/another",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+});
+
+add_task(async () => {
+  info("Test for listening to multi events with firing single event");
+
+  const observer = startObservation([
+    "page-visited",
+    "page-title-changed",
+    "bookmark-added",
+  ]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "test",
+      url: "http://example.com/test",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/test",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async () => {
+  info(
+    "Test for listening to multi events with firing multi events at same time"
+  );
+
+  const observer = startObservation([
+    "page-visited",
+    "page-title-changed",
+    "bookmark-added",
+  ]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "will change",
+      url: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+    {
+      title: "changed",
+      url: "http://example.com/title",
+      referrer: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+    {
+      title: "another",
+      url: "http://example.com/another",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+      {
+        type: "page-title-changed",
+        url: "http://example.com/title",
+        title: "changed",
+      },
+      {
+        type: "page-visited",
+        url: "http://example.com/another",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async () => {
+  info(
+    "Test for listening to multi events with firing multi events separately"
+  );
+
+  const observer = startObservation([
+    "page-visited",
+    "page-title-changed",
+    "bookmark-added",
+  ]);
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "will change",
+      url: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "a folder",
+  });
+
+  await PlacesUtils.history.insertMany([
+    {
+      title: "changed",
+      url: "http://example.com/title",
+      referrer: "http://example.com/title",
+      visits: [{ transition: TRANSITION_LINK }],
+    },
+  ]);
+
+  const expectedFiredEvents = [
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+    ],
+    [
+      {
+        type: "bookmark-added",
+        title: "a folder",
+      },
+    ],
+    [
+      {
+        type: "page-visited",
+        url: "http://example.com/title",
+      },
+      {
+        type: "page-title-changed",
+        url: "http://example.com/title",
+        title: "changed",
+      },
+    ],
+  ];
+  assertFiredEvents(observer.firedEvents, expectedFiredEvents);
+
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.remove(bookmark.guid);
+});
+
+add_task(async function test_empty_notifications_array() {
+  info("Test whether listener does not receive empty events");
+
+  if (AppConstants.DEBUG) {
+    info(
+      "Ignore this test since we added a MOZ_ASSERT for empty events in debug build"
+    );
+    return;
+  }
+
+  const observer = startObservation(["page-visited"]);
+  PlacesObservers.notifyListeners([]);
+  Assert.equal(observer.firedEvents.length, 0, "Listener does not receive any");
+});
+
+function startObservation(targets) {
+  const observer = {
+    firedEvents: [],
+    handle(events) {
+      this.firedEvents.push(events);
+    },
+  };
+
+  PlacesObservers.addListener(targets, observer.handle.bind(observer));
+
+  return observer;
+}
+
+function assertFiredEvents(firedEvents, expectedFiredEvents) {
+  Assert.equal(
+    firedEvents.length,
+    expectedFiredEvents.length,
+    "Number events fired is correct"
+  );
+
+  for (let i = 0; i < firedEvents.length; i++) {
+    info(`Check firedEvents[${i}]`);
+    const events = firedEvents[i];
+    const expectedEvents = expectedFiredEvents[i];
+    assertEvents(events, expectedEvents);
+  }
+}
+
+function assertEvents(events, expectedEvents) {
+  Assert.equal(
+    events.length,
+    expectedEvents.length,
+    "Number events is correct"
+  );
+
+  for (let i = 0; i < events.length; i++) {
+    info(`Check events[${i}]`);
+    const event = events[i];
+    const expectedEvent = expectedEvents[i];
+
+    for (let field in expectedEvent) {
+      Assert.equal(event[field], expectedEvent[field], `${field} is correct`);
+    }
+  }
+}
diff --git a/toolkit/components/places/tests/unit/test_multi_word_tags.js b/toolkit/components/places/tests/unit/test_multi_word_tags.js
new file mode 100644
index 0000000000..17a68cf972
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_multi_word_tags.js
@@ -0,0 +1,147 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Get history service
+try {
+  var histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].getService(
+    Ci.nsINavHistoryService
+  );
+} catch (ex) {
+  do_throw("Could not get history service\n");
+}
+
+// Get tagging service
+try {
+  var tagssvc = Cc["@mozilla.org/browser/tagging-service;1"].getService(
+    Ci.nsITaggingService
+  );
+} catch (ex) {
+  do_throw("Could not get tagging service\n");
+}
+
+add_task(async function run_test() {
+  var uri1 = uri("http://site.tld/1");
+  var uri2 = uri("http://site.tld/2");
+  var uri3 = uri("http://site.tld/3");
+  var uri4 = uri("http://site.tld/4");
+  var uri5 = uri("http://site.tld/5");
+  var uri6 = uri("http://site.tld/6");
+
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      { url: uri1 },
+      { url: uri2 },
+      { url: uri3 },
+      { url: uri4 },
+      { url: uri5 },
+      { url: uri6 },
+    ],
+  });
+
+  tagssvc.tagURI(uri1, ["foo"]);
+  tagssvc.tagURI(uri2, ["bar"]);
+  tagssvc.tagURI(uri3, ["cheese"]);
+  tagssvc.tagURI(uri4, ["foo bar"]);
+  tagssvc.tagURI(uri5, ["bar cheese"]);
+  tagssvc.tagURI(uri6, ["foo bar cheese"]);
+
+  // Search for "item", should get one result
+  var options = histsvc.getNewQueryOptions();
+  options.queryType = Ci.nsINavHistoryQueryOptions.QUERY_TYPE_BOOKMARKS;
+
+  var query = histsvc.getNewQuery();
+  query.searchTerms = "foo";
+  var result = histsvc.executeQuery(query, options);
+  var root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 3);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/1");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/4");
+  Assert.equal(root.getChild(2).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "bar";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 4);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/2");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/4");
+  Assert.equal(root.getChild(2).uri, "http://site.tld/5");
+  Assert.equal(root.getChild(3).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "cheese";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 3);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/3");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/5");
+  Assert.equal(root.getChild(2).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "foo bar";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/4");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "bar foo";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/4");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "bar cheese";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/5");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "cheese bar";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/5");
+  Assert.equal(root.getChild(1).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "foo bar cheese";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "cheese foo bar";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/6");
+  root.containerOpen = false;
+
+  query.searchTerms = "cheese bar foo";
+  result = histsvc.executeQuery(query, options);
+  root = result.root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  Assert.equal(root.getChild(0).uri, "http://site.tld/6");
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_nested_notifications.js b/toolkit/components/places/tests/unit/test_nested_notifications.js
new file mode 100644
index 0000000000..cf1e4b77fa
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_nested_notifications.js
@@ -0,0 +1,178 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Test for nested notifications of Places events.
+ * In this test, we check behavior of listeners of Places event upon firing nested
+ * notification from inside of listener that received notifications.
+ */
+add_task(async function () {
+  // We prepare 6 listeners for the test.
+  // 1. Listener that added before root notification.
+  const addRoot = new Observer();
+  // 2. Listener that added before root notification
+  //    but removed before first nest notification.
+  const addRootRemoveFirst = new Observer();
+  // 3. Listener that added before root notification
+  //    but removed before second nest notification.
+  const addRootRemoveSecond = new Observer();
+  // 4. Listener that added before first nest notification.
+  const addFirst = new Observer();
+  // 5. Listener that added before first nest notification
+  //    but removed before second nest notification.
+  const addFirstRemoveSecond = new Observer();
+  // 6. Listener that added before second nest notification.
+  const addSecond = new Observer();
+
+  // This is a listener listened the root notification
+  // and do what we have to do for test in the first nest.
+  const firstNestOperator = () => {
+    info("Start to operate at first nest");
+
+    // Remove itself to avoid listening more.
+    removePlacesListener(firstNestOperator);
+
+    info("Add/Remove test listeners at first nest");
+    removePlacesListener(addRootRemoveFirst.handle);
+    addPlacesListener(addFirst.handle);
+    addPlacesListener(addFirstRemoveSecond.handle);
+
+    // Add second nest operator.
+    addPlacesListener(secondNestOperator);
+
+    info("Send notification at first nest");
+    notifyPlacesEvent("first");
+  };
+
+  // This is a listener listened the first nest notification
+  // and do what we have to do for test in the second nest.
+  const secondNestOperator = () => {
+    info("Start to operate at second nest");
+
+    // Remove itself to avoid listening more.
+    removePlacesListener(secondNestOperator);
+
+    info("Add/Remove test listeners at second nest");
+    removePlacesListener(addRootRemoveSecond.handle);
+    removePlacesListener(addFirstRemoveSecond.handle);
+    addPlacesListener(addSecond.handle);
+
+    info("Send notification at second nest");
+    notifyPlacesEvent("second");
+  };
+
+  info("Add test listeners that handle notification sent at root");
+  addPlacesListener(addRoot.handle);
+  addPlacesListener(addRootRemoveFirst.handle);
+  addPlacesListener(addRootRemoveSecond.handle);
+
+  // Add first nest operator.
+  addPlacesListener(firstNestOperator);
+
+  info("Send notification at root");
+  notifyPlacesEvent("root");
+
+  info("Check whether or not test listeners could get expected notifications");
+  assertNotifications(addRoot.notifications, [
+    [{ guid: "root" }],
+    [{ guid: "first" }],
+    [{ guid: "second" }],
+  ]);
+  assertNotifications(addRootRemoveFirst.notifications, [[{ guid: "root" }]]);
+  assertNotifications(addRootRemoveSecond.notifications, [
+    [{ guid: "root" }],
+    [{ guid: "first" }],
+  ]);
+  assertNotifications(addFirst.notifications, [
+    [{ guid: "first" }],
+    [{ guid: "second" }],
+  ]);
+  assertNotifications(addFirstRemoveSecond.notifications, [
+    [{ guid: "first" }],
+  ]);
+  assertNotifications(addSecond.notifications, [[{ guid: "second" }]]);
+});
+
+function addPlacesListener(listener) {
+  PlacesObservers.addListener(["bookmark-added"], listener);
+}
+
+function removePlacesListener(listener) {
+  PlacesObservers.removeListener(["bookmark-added"], listener);
+}
+
+function notifyPlacesEvent(guid) {
+  PlacesObservers.notifyListeners([
+    new PlacesBookmarkAddition({
+      dateAdded: 0,
+      guid,
+      id: -1,
+      index: 0,
+      isTagging: false,
+      itemType: 1,
+      parentGuid: "fake",
+      parentId: -2,
+      source: 0,
+      title: guid,
+      url: `http://example.com/${guid}`,
+    }),
+  ]);
+}
+
+class Observer {
+  constructor() {
+    this.notifications = [];
+    this.handle = this.handle.bind(this);
+  }
+
+  handle(events) {
+    this.notifications.push(events);
+  }
+}
+
+/**
+ * Assert notifications the observer received.
+ *
+ * @param Array - notifications
+ * @param Array - expectedNotifications
+ */
+function assertNotifications(notifications, expectedNotifications) {
+  Assert.equal(
+    notifications.length,
+    expectedNotifications.length,
+    "Number of notifications is correct"
+  );
+
+  for (let i = 0; i < notifications.length; i++) {
+    info(`Check notifications[${i}]`);
+    const placesEvents = notifications[i];
+    const expectedPlacesEvents = expectedNotifications[i];
+    assertPlacesEvents(placesEvents, expectedPlacesEvents);
+  }
+}
+
+/**
+ * Assert Places events.
+ * This function checks given expected event field only.
+ *
+ * @param Array - events
+ * @param Array - expectedEvents
+ */
+function assertPlacesEvents(events, expectedEvents) {
+  Assert.equal(
+    events.length,
+    expectedEvents.length,
+    "Number of Places events is correct"
+  );
+
+  for (let i = 0; i < events.length; i++) {
+    info(`Check Places events[${i}]`);
+    const event = events[i];
+    const expectedEvent = expectedEvents[i];
+
+    for (let field in expectedEvent) {
+      Assert.equal(event[field], expectedEvent[field], `${field} is correct`);
+    }
+  }
+}
diff --git a/toolkit/components/places/tests/unit/test_nsINavHistoryViewer.js b/toolkit/components/places/tests/unit/test_nsINavHistoryViewer.js
new file mode 100644
index 0000000000..8c061c60d7
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_nsINavHistoryViewer.js
@@ -0,0 +1,285 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var resultObserver = {
+  insertedNode: null,
+  nodeInserted(parent, node, newIndex) {
+    this.insertedNode = node;
+  },
+  removedNode: null,
+  nodeRemoved(parent, node, oldIndex) {
+    this.removedNode = node;
+  },
+
+  newTitle: "",
+  nodeChangedByTitle: null,
+  nodeTitleChanged(node, newTitle) {
+    this.nodeChangedByTitle = node;
+    this.newTitle = newTitle;
+  },
+
+  newAccessCount: 0,
+  newTime: 0,
+  nodeChangedByHistoryDetails: null,
+  nodeHistoryDetailsChanged(node, oldVisitDate, oldVisitCount) {
+    this.nodeChangedByHistoryDetails = node;
+    this.newTime = node.time;
+    this.newAccessCount = node.accessCount;
+  },
+
+  movedNode: null,
+  nodeMoved(node, oldParent, oldIndex, newParent, newIndex) {
+    this.movedNode = node;
+  },
+  openedContainer: null,
+  closedContainer: null,
+  containerStateChanged(aNode, aOldState, aNewState) {
+    if (aNewState == Ci.nsINavHistoryContainerResultNode.STATE_OPENED) {
+      this.openedContainer = aNode;
+    } else if (aNewState == Ci.nsINavHistoryContainerResultNode.STATE_CLOSED) {
+      this.closedContainer = aNode;
+    }
+  },
+  invalidatedContainer: null,
+  invalidateContainer(node) {
+    this.invalidatedContainer = node;
+  },
+  sortingMode: null,
+  sortingChanged(sortingMode) {
+    this.sortingMode = sortingMode;
+  },
+  inBatchMode: false,
+  batchingCallCount: 0,
+  batching(aToggleMode) {
+    Assert.notEqual(this.inBatchMode, aToggleMode);
+    this.inBatchMode = aToggleMode;
+    this.batchingCallCount++;
+  },
+  result: null,
+  reset() {
+    this.insertedNode = null;
+    this.removedNode = null;
+    this.nodeChangedByTitle = null;
+    this.nodeChangedByHistoryDetails = null;
+    this.replacedNode = null;
+    this.movedNode = null;
+    this.openedContainer = null;
+    this.closedContainer = null;
+    this.invalidatedContainer = null;
+    this.sortingMode = null;
+    this.inBatchMode = false;
+    this.batchingCallCount = 0;
+  },
+};
+
+var testURI = uri("http://mozilla.com");
+
+add_task(async function check_history_query() {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  options.sortingMode = options.SORT_BY_DATE_DESCENDING;
+  options.resultType = options.RESULTS_AS_VISIT;
+  let query = PlacesUtils.history.getNewQuery();
+  let result = PlacesUtils.history.executeQuery(query, options);
+  result.addObserver(resultObserver);
+  let root = result.root;
+  root.containerOpen = true;
+
+  Assert.notEqual(resultObserver.openedContainer, null);
+
+  // nsINavHistoryResultObserver.nodeInserted
+  // add a visit
+  await PlacesTestUtils.addVisits(testURI);
+  Assert.equal(testURI.spec, resultObserver.insertedNode.uri);
+
+  // nsINavHistoryResultObserver.nodeHistoryDetailsChanged
+  // adding a visit causes nodeHistoryDetailsChanged for the folder
+  Assert.equal(root.uri, resultObserver.nodeChangedByHistoryDetails.uri);
+
+  // nsINavHistoryResultObserver.itemTitleChanged for a leaf node
+  await PlacesTestUtils.addVisits({ uri: testURI, title: "baz" });
+  Assert.equal(resultObserver.nodeChangedByTitle.title, "baz");
+
+  // nsINavHistoryResultObserver.nodeRemoved
+  let removedURI = uri("http://google.com");
+  await PlacesTestUtils.addVisits(removedURI);
+  await PlacesUtils.history.remove(removedURI);
+  Assert.equal(removedURI.spec, resultObserver.removedNode.uri);
+
+  // nsINavHistoryResultObserver.invalidateContainer
+  await PlacesUtils.history.removeByFilter({ host: "mozilla.com" });
+  // This test is disabled for bug 1089691. It is failing bcause the new API
+  // doesn't send batching notifications and thus the result doesn't invalidate
+  // the whole container.
+  // Assert.equal(root.uri, resultObserver.invalidatedContainer.uri);
+
+  // nsINavHistoryResultObserver.sortingChanged
+  resultObserver.invalidatedContainer = null;
+  result.sortingMode = options.SORT_BY_TITLE_ASCENDING;
+  Assert.equal(resultObserver.sortingMode, options.SORT_BY_TITLE_ASCENDING);
+  Assert.equal(resultObserver.invalidatedContainer, result.root);
+
+  // nsINavHistoryResultObserver.invalidateContainer
+  await PlacesUtils.history.clear();
+  Assert.equal(root.uri, resultObserver.invalidatedContainer.uri);
+
+  root.containerOpen = false;
+  Assert.equal(resultObserver.closedContainer, resultObserver.openedContainer);
+  result.removeObserver(resultObserver);
+  resultObserver.reset();
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function check_bookmarks_query() {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let query = PlacesUtils.history.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.menuGuid]);
+  let result = PlacesUtils.history.executeQuery(query, options);
+  result.addObserver(resultObserver);
+  let root = result.root;
+  root.containerOpen = true;
+
+  Assert.notEqual(resultObserver.openedContainer, null);
+
+  // nsINavHistoryResultObserver.nodeInserted
+  // add a bookmark
+  let testBookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: testURI,
+    title: "foo",
+  });
+  Assert.equal("foo", resultObserver.insertedNode.title);
+  Assert.equal(testURI.spec, resultObserver.insertedNode.uri);
+
+  // nsINavHistoryResultObserver.nodeHistoryDetailsChanged
+  // adding a visit causes nodeHistoryDetailsChanged for the folder
+  Assert.equal(root.uri, resultObserver.nodeChangedByHistoryDetails.uri);
+
+  // nsINavHistoryResultObserver.nodeTitleChanged for a leaf node
+  await PlacesUtils.bookmarks.update({
+    guid: testBookmark.guid,
+    title: "baz",
+  });
+  Assert.equal(resultObserver.nodeChangedByTitle.title, "baz");
+  Assert.equal(resultObserver.newTitle, "baz");
+
+  let testBookmark2 = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://google.com",
+    title: "foo",
+  });
+
+  await PlacesUtils.bookmarks.update({
+    guid: testBookmark2.guid,
+    index: 0,
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+  });
+  Assert.equal(resultObserver.movedNode.bookmarkGuid, testBookmark2.guid);
+
+  // nsINavHistoryResultObserver.nodeRemoved
+  await PlacesUtils.bookmarks.remove(testBookmark2.guid);
+  Assert.equal(testBookmark2.guid, resultObserver.removedNode.bookmarkGuid);
+
+  // XXX nsINavHistoryResultObserver.invalidateContainer
+
+  // nsINavHistoryResultObserver.sortingChanged
+  resultObserver.invalidatedContainer = null;
+  result.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING;
+  Assert.equal(
+    resultObserver.sortingMode,
+    Ci.nsINavHistoryQueryOptions.SORT_BY_TITLE_ASCENDING
+  );
+  Assert.equal(resultObserver.invalidatedContainer, result.root);
+
+  root.containerOpen = false;
+  Assert.equal(resultObserver.closedContainer, resultObserver.openedContainer);
+  result.removeObserver(resultObserver);
+  resultObserver.reset();
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function check_mixed_query() {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let query = PlacesUtils.history.getNewQuery();
+  query.onlyBookmarked = true;
+  let result = PlacesUtils.history.executeQuery(query, options);
+  result.addObserver(resultObserver);
+  let root = result.root;
+  root.containerOpen = true;
+
+  Assert.notEqual(resultObserver.openedContainer, null);
+
+  root.containerOpen = false;
+  Assert.equal(resultObserver.closedContainer, resultObserver.openedContainer);
+  result.removeObserver(resultObserver);
+  resultObserver.reset();
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function check_a_batch_process() {
+  const options = PlacesUtils.history.getNewQueryOptions();
+  const query = PlacesUtils.history.getNewQuery();
+  const result = PlacesUtils.history.executeQuery(query, options);
+  result.addObserver(resultObserver);
+
+  info("Check initial state");
+  Assert.equal(resultObserver.inBatchMode, false);
+  Assert.equal(resultObserver.batchingCallCount, 0);
+
+  info("Check whether batching is called when call onBeginUpdateBatch");
+  result.onBeginUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, true);
+  Assert.equal(resultObserver.batchingCallCount, 1);
+
+  info("Check whether batching is called when call onEndUpdateBatch");
+  result.onEndUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, false);
+  Assert.equal(resultObserver.batchingCallCount, 2);
+
+  result.removeObserver(resultObserver);
+  resultObserver.reset();
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
+
+add_task(async function check_multi_batch_processes() {
+  const options = PlacesUtils.history.getNewQueryOptions();
+  const query = PlacesUtils.history.getNewQuery();
+  const result = PlacesUtils.history.executeQuery(query, options);
+  result.addObserver(resultObserver);
+
+  info("Check initial state");
+  Assert.equal(resultObserver.inBatchMode, false);
+  Assert.equal(resultObserver.batchingCallCount, 0);
+
+  info("Check whether batching is called when calling onBeginUpdateBatch");
+  result.onBeginUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, true);
+  Assert.equal(resultObserver.batchingCallCount, 1);
+
+  info(
+    "Check whether batching is not called when calling onBeginUpdateBatch again"
+  );
+  result.onBeginUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, true);
+  Assert.equal(resultObserver.batchingCallCount, 1);
+
+  info(
+    "Check whether batching is not called until calling onEndUpdateBatch the same number times that onBeginUpdateBatch is called"
+  );
+  result.onEndUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, true);
+  Assert.equal(resultObserver.batchingCallCount, 1);
+
+  info(
+    "Check whether batching is called when calling onEndUpdateBatch the same number times"
+  );
+  result.onEndUpdateBatch();
+  Assert.equal(resultObserver.inBatchMode, false);
+  Assert.equal(resultObserver.batchingCallCount, 2);
+
+  result.removeObserver(resultObserver);
+  resultObserver.reset();
+  await PlacesTestUtils.promiseAsyncUpdates();
+});
diff --git a/toolkit/components/places/tests/unit/test_null_interfaces.js b/toolkit/components/places/tests/unit/test_null_interfaces.js
new file mode 100644
index 0000000000..f5608285d6
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_null_interfaces.js
@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Test bug 489872 to make sure passing nulls to nsNavHistory doesn't crash.
+ */
+
+// Make an array of services to test, each specifying a class id, interface
+// and an array of function names that don't throw when passed nulls
+var testServices = [
+  [
+    "browser/nav-history-service;1",
+    ["nsINavHistoryService"],
+    [
+      "queryStringToQuery",
+      "removePagesByTimeframe",
+      "removePagesFromHost",
+      "getObservers",
+    ],
+  ],
+  [
+    "browser/nav-bookmarks-service;1",
+    ["nsINavBookmarksService"],
+    ["createFolder", "getObservers"],
+  ],
+  ["browser/favicon-service;1", ["nsIFaviconService"], []],
+  ["browser/tagging-service;1", ["nsITaggingService"], []],
+];
+info(testServices.join("\n"));
+
+function run_test() {
+  for (let [cid, ifaces, nothrow] of testServices) {
+    info(`Running test with ${cid} ${ifaces.join(", ")} ${nothrow}`);
+    let s = Cc["@mozilla.org/" + cid].getService(Ci.nsISupports);
+    for (let iface of ifaces) {
+      s.QueryInterface(Ci[iface]);
+    }
+
+    let okName = function (name) {
+      info(`Checking if function is okay to test: ${name}`);
+      let func = s[name];
+
+      let mesg = "";
+      if (typeof func != "function") {
+        mesg = "Not a function!";
+      } else if (!func.length) {
+        mesg = "No args needed!";
+      } else if (name == "QueryInterface") {
+        mesg = "Ignore QI!";
+      }
+
+      if (mesg) {
+        info(`${mesg} Skipping: ${name}`);
+        return false;
+      }
+
+      return true;
+    };
+
+    info(`Generating an array of functions to test service: ${s}`);
+    for (let n of Object.keys(s)
+      .filter(i => okName(i))
+      .sort()) {
+      info(`\nTesting ${ifaces.join(", ")} function with null args: ${n}`);
+
+      let func = s[n];
+      let num = func.length;
+      info(`Generating array of nulls for #args: ${num}`);
+      let args = Array(num).fill(null);
+
+      let tryAgain = true;
+      while (tryAgain) {
+        try {
+          info(`Calling with args: ${JSON.stringify(args)}`);
+          func.apply(s, args);
+
+          info(
+            `The function did not throw! Is it one of the nothrow? ${nothrow}`
+          );
+          Assert.notEqual(nothrow.indexOf(n), -1);
+
+          info("Must have been an expected nothrow, so no need to try again");
+          tryAgain = false;
+        } catch (ex) {
+          if (ex.result == Cr.NS_ERROR_ILLEGAL_VALUE) {
+            info(`Caught an expected exception: ${ex.name}`);
+            info("Moving on to the next test..");
+            tryAgain = false;
+          } else if (ex.result == Cr.NS_ERROR_XPC_NEED_OUT_OBJECT) {
+            let pos = Number(ex.message.match(/object arg (\d+)/)[1]);
+            info(`Function call expects an out object at ${pos}`);
+            args[pos] = {};
+          } else if (ex.result == Cr.NS_ERROR_NOT_IMPLEMENTED) {
+            info(`Method not implemented exception: ${ex.name}`);
+            info("Moving on to the next test..");
+            tryAgain = false;
+          } else {
+            throw ex;
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/toolkit/components/places/tests/unit/test_origins.js b/toolkit/components/places/tests/unit/test_origins.js
new file mode 100644
index 0000000000..4e2d0e5fa6
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_origins.js
@@ -0,0 +1,1122 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Makes sure the moz_origins table and origin frecency stats are updated
+// correctly.
+
+"use strict";
+
+// Visiting a URL with a new origin should immediately update moz_origins.
+add_task(async function visit() {
+  await checkDB([]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Repeatedly visiting a URL with an initially new origin should update
+// moz_origins (with the correct frecency).
+add_task(async function visitRepeatedly() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://example.com/" },
+    { uri: "http://example.com/" },
+  ]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Same as previous, but visits are added sequentially.
+add_task(async function visitRepeatedlySequential() {
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// After removing an origin's URLs, visiting a URL with the origin should
+// immediately update moz_origins.
+add_task(async function vistAfterDelete() {
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Visiting different URLs with the same origin should update moz_origins, and
+// moz_origins.frecency should be the sum of the URL frecencies.
+add_task(async function visitDifferentURLsSameOrigin() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/1" },
+    { uri: "http://example.com/2" },
+    { uri: "http://example.com/3" },
+  ]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/1", "http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/2");
+  await checkDB([["http://", "example.com", ["http://example.com/3"]]]);
+  await PlacesUtils.history.remove("http://example.com/3");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Same as previous, but visits are added sequentially.
+add_task(async function visitDifferentURLsSameOriginSequential() {
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/1" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/1"]]]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/2" }]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/1", "http://example.com/2"],
+    ],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/3" }]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/1", "http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/2");
+  await checkDB([["http://", "example.com", ["http://example.com/3"]]]);
+  await PlacesUtils.history.remove("http://example.com/3");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Repeatedly visiting different URLs with the same origin should update
+// moz_origins (with the correct frecencies), and moz_origins.frecency should be
+// the sum of the URL frecencies.
+add_task(async function visitDifferentURLsSameOriginRepeatedly() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/1" },
+    { uri: "http://example.com/1" },
+    { uri: "http://example.com/1" },
+    { uri: "http://example.com/2" },
+    { uri: "http://example.com/2" },
+    { uri: "http://example.com/3" },
+  ]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/1", "http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/2", "http://example.com/3"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/2");
+  await checkDB([["http://", "example.com", ["http://example.com/3"]]]);
+  await PlacesUtils.history.remove("http://example.com/3");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Visiting URLs with different origins should update moz_origins.
+add_task(async function visitDifferentOrigins() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example1.com/" },
+    { uri: "http://example2.com/" },
+    { uri: "http://example3.com/" },
+  ]);
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/"]],
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/");
+  await checkDB([["http://", "example3.com", ["http://example3.com/"]]]);
+  await PlacesUtils.history.remove("http://example3.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Same as previous, but visits are added sequentially.
+add_task(async function visitDifferentOriginsSequential() {
+  await PlacesTestUtils.addVisits([{ uri: "http://example1.com/" }]);
+  await checkDB([["http://", "example1.com", ["http://example1.com/"]]]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example2.com/" }]);
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/"]],
+    ["http://", "example2.com", ["http://example2.com/"]],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example3.com/" }]);
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/"]],
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/");
+  await checkDB([["http://", "example3.com", ["http://example3.com/"]]]);
+  await PlacesUtils.history.remove("http://example3.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Repeatedly visiting URLs with different origins should update moz_origins
+// (with the correct frecencies).
+add_task(async function visitDifferentOriginsRepeatedly() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example1.com/" },
+    { uri: "http://example1.com/" },
+    { uri: "http://example1.com/" },
+    { uri: "http://example2.com/" },
+    { uri: "http://example2.com/" },
+    { uri: "http://example3.com/" },
+  ]);
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/"]],
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/"]],
+    ["http://", "example3.com", ["http://example3.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/");
+  await checkDB([["http://", "example3.com", ["http://example3.com/"]]]);
+  await PlacesUtils.history.remove("http://example3.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Visiting URLs, some with the same and some with different origins, should
+// update moz_origins.
+add_task(async function visitDifferentOriginsDifferentURLs() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example1.com/1" },
+    { uri: "http://example1.com/2" },
+    { uri: "http://example1.com/3" },
+    { uri: "http://example2.com/1" },
+    { uri: "http://example2.com/2" },
+    { uri: "http://example3.com/1" },
+  ]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      ["http://example1.com/2", "http://example1.com/3"],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/2");
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/3"]],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/3");
+  await checkDB([
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/1");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/2"]],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/2");
+  await checkDB([["http://", "example3.com", ["http://example3.com/1"]]]);
+  await PlacesUtils.history.remove("http://example3.com/1");
+  await checkDB([]);
+});
+
+// Same as previous, but visits are added sequentially.
+add_task(async function visitDifferentOriginsDifferentURLsSequential() {
+  await PlacesTestUtils.addVisits([{ uri: "http://example1.com/1" }]);
+  await checkDB([["http://", "example1.com", ["http://example1.com/1"]]]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example1.com/2" }]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      ["http://example1.com/1", "http://example1.com/2"],
+    ],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example1.com/3" }]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example2.com/1" }]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+    ["http://", "example2.com", ["http://example2.com/1"]],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example2.com/2" }]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+  ]);
+  await PlacesTestUtils.addVisits([{ uri: "http://example3.com/1" }]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      ["http://example1.com/2", "http://example1.com/3"],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/2");
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/3"]],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/3");
+  await checkDB([
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/1");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/2"]],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/2");
+  await checkDB([["http://", "example3.com", ["http://example3.com/1"]]]);
+  await PlacesUtils.history.remove("http://example3.com/1");
+  await checkDB([]);
+});
+
+// Repeatedly visiting URLs, some with the same and some with different origins,
+// should update moz_origins (with the correct frecencies).
+add_task(async function visitDifferentOriginsDifferentURLsRepeatedly() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example1.com/1" },
+    { uri: "http://example1.com/1" },
+    { uri: "http://example1.com/1" },
+    { uri: "http://example1.com/2" },
+    { uri: "http://example1.com/2" },
+    { uri: "http://example1.com/3" },
+    { uri: "http://example2.com/1" },
+    { uri: "http://example2.com/1" },
+    { uri: "http://example2.com/1" },
+    { uri: "http://example2.com/2" },
+    { uri: "http://example2.com/2" },
+    { uri: "http://example3.com/1" },
+    { uri: "http://example3.com/1" },
+  ]);
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      [
+        "http://example1.com/1",
+        "http://example1.com/2",
+        "http://example1.com/3",
+      ],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/1");
+  await checkDB([
+    [
+      "http://",
+      "example1.com",
+      ["http://example1.com/2", "http://example1.com/3"],
+    ],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/2");
+  await checkDB([
+    ["http://", "example1.com", ["http://example1.com/3"]],
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example1.com/3");
+  await checkDB([
+    [
+      "http://",
+      "example2.com",
+      ["http://example2.com/1", "http://example2.com/2"],
+    ],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/1");
+  await checkDB([
+    ["http://", "example2.com", ["http://example2.com/2"]],
+    ["http://", "example3.com", ["http://example3.com/1"]],
+  ]);
+  await PlacesUtils.history.remove("http://example2.com/2");
+  await checkDB([["http://", "example3.com", ["http://example3.com/1"]]]);
+  await PlacesUtils.history.remove("http://example3.com/1");
+  await checkDB([]);
+});
+
+// Makes sure URIs with the same TLD but different www subdomains are recognized
+// as different origins.  Makes sure removing one doesn't remove the others.
+add_task(async function www1() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://www.example.com/" },
+    { uri: "http://www.www.example.com/" },
+  ]);
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+    ["http://", "www.www.example.com", ["http://www.www.example.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+    ["http://", "www.www.example.com", ["http://www.www.example.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://www.example.com/");
+  await checkDB([
+    ["http://", "www.www.example.com", ["http://www.www.example.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://www.www.example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Same as www1, but removes URIs in a different order.
+add_task(async function www2() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://www.example.com/" },
+    { uri: "http://www.www.example.com/" },
+  ]);
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+    ["http://", "www.www.example.com", ["http://www.www.example.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://www.www.example.com/");
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+  ]);
+  await PlacesUtils.history.remove("http://www.example.com/");
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Makes sure removing an origin without a port doesn't remove the same host
+// with a port.
+add_task(async function ports1() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://example.com:8888/" },
+  ]);
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "example.com:8888", ["http://example.com:8888/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([
+    ["http://", "example.com:8888", ["http://example.com:8888/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example.com:8888/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Makes sure removing an origin with a port doesn't remove the same host
+// without a port.
+add_task(async function ports2() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://example.com:8888/" },
+  ]);
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "example.com:8888", ["http://example.com:8888/"]],
+  ]);
+  await PlacesUtils.history.remove("http://example.com:8888/");
+  await checkDB([["http://", "example.com", ["http://example.com/"]]]);
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Makes sure multiple URIs with the same origin don't create duplicate origins.
+add_task(async function duplicates() {
+  await PlacesTestUtils.addVisits([
+    { uri: "http://example.com/" },
+    { uri: "http://www.example.com/" },
+    { uri: "http://www.www.example.com/" },
+    { uri: "https://example.com/" },
+    { uri: "ftp://example.com/" },
+    { uri: "foo://example.com/" },
+    { uri: "bar:example.com/" },
+    { uri: "http://example.com:8888/" },
+
+    { uri: "http://example.com/dupe" },
+    { uri: "http://www.example.com/dupe" },
+    { uri: "http://www.www.example.com/dupe" },
+    { uri: "https://example.com/dupe" },
+    { uri: "ftp://example.com/dupe" },
+    { uri: "foo://example.com/dupe" },
+    { uri: "bar:example.com/dupe" },
+    { uri: "http://example.com:8888/dupe" },
+  ]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/", "http://example.com/dupe"],
+    ],
+    [
+      "http://",
+      "www.example.com",
+      ["http://www.example.com/", "http://www.example.com/dupe"],
+    ],
+    [
+      "http://",
+      "www.www.example.com",
+      ["http://www.www.example.com/", "http://www.www.example.com/dupe"],
+    ],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("http://example.com/");
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/dupe"]],
+    [
+      "http://",
+      "www.example.com",
+      ["http://www.example.com/", "http://www.example.com/dupe"],
+    ],
+    [
+      "http://",
+      "www.www.example.com",
+      ["http://www.www.example.com/", "http://www.www.example.com/dupe"],
+    ],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://example.com/dupe");
+  await checkDB([
+    [
+      "http://",
+      "www.example.com",
+      ["http://www.example.com/", "http://www.example.com/dupe"],
+    ],
+    [
+      "http://",
+      "www.www.example.com",
+      ["http://www.www.example.com/", "http://www.www.example.com/dupe"],
+    ],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("http://www.example.com/");
+  await checkDB([
+    ["http://", "www.example.com", ["http://www.example.com/dupe"]],
+    [
+      "http://",
+      "www.www.example.com",
+      ["http://www.www.example.com/", "http://www.www.example.com/dupe"],
+    ],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://www.example.com/dupe");
+  await checkDB([
+    [
+      "http://",
+      "www.www.example.com",
+      ["http://www.www.example.com/", "http://www.www.example.com/dupe"],
+    ],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("http://www.www.example.com/");
+  await checkDB([
+    ["http://", "www.www.example.com", ["http://www.www.example.com/dupe"]],
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("http://www.www.example.com/dupe");
+  await checkDB([
+    [
+      "https://",
+      "example.com",
+      ["https://example.com/", "https://example.com/dupe"],
+    ],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("https://example.com/");
+  await checkDB([
+    ["https://", "example.com", ["https://example.com/dupe"]],
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("https://example.com/dupe");
+  await checkDB([
+    ["ftp://", "example.com", ["ftp://example.com/", "ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("ftp://example.com/");
+  await checkDB([
+    ["ftp://", "example.com", ["ftp://example.com/dupe"]],
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("ftp://example.com/dupe");
+  await checkDB([
+    ["foo://", "example.com", ["foo://example.com/", "foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("foo://example.com/");
+  await checkDB([
+    ["foo://", "example.com", ["foo://example.com/dupe"]],
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("foo://example.com/dupe");
+  await checkDB([
+    ["bar:", "example.com", ["bar:example.com/", "bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("bar:example.com/");
+  await checkDB([
+    ["bar:", "example.com", ["bar:example.com/dupe"]],
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+  await PlacesUtils.history.remove("bar:example.com/dupe");
+  await checkDB([
+    [
+      "http://",
+      "example.com:8888",
+      ["http://example.com:8888/", "http://example.com:8888/dupe"],
+    ],
+  ]);
+
+  await PlacesUtils.history.remove("http://example.com:8888/");
+  await checkDB([
+    ["http://", "example.com:8888", ["http://example.com:8888/dupe"]],
+  ]);
+  await PlacesUtils.history.remove("http://example.com:8888/dupe");
+  await checkDB([]);
+
+  await cleanUp();
+});
+
+// Makes sure adding and removing bookmarks creates origins.
+add_task(async function addRemoveBookmarks() {
+  let bookmarks = [];
+  let urls = ["http://example.com/", "http://www.example.com/"];
+  for (let url of urls) {
+    bookmarks.push(
+      await PlacesUtils.bookmarks.insert({
+        url,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      })
+    );
+  }
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+  ]);
+  await PlacesUtils.bookmarks.remove(bookmarks[0]);
+  await PlacesUtils.history.clear();
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([["http://", "www.example.com", ["http://www.example.com/"]]]);
+  await PlacesUtils.bookmarks.remove(bookmarks[1]);
+  await PlacesUtils.history.clear();
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([]);
+  await cleanUp();
+});
+
+// Makes sure changing bookmarks also changes the corresponding origins.
+add_task(async function changeBookmarks() {
+  let bookmarks = [];
+  let urls = ["http://example.com/", "http://www.example.com/"];
+  for (let url of urls) {
+    bookmarks.push(
+      await PlacesUtils.bookmarks.insert({
+        url,
+        parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+      })
+    );
+  }
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([
+    ["http://", "example.com", ["http://example.com/"]],
+    ["http://", "www.example.com", ["http://www.example.com/"]],
+  ]);
+  await PlacesUtils.bookmarks.update({
+    url: "http://www.example.com/",
+    guid: bookmarks[0].guid,
+  });
+  await PlacesUtils.history.clear();
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([["http://", "www.example.com", ["http://www.example.com/"]]]);
+  await cleanUp();
+});
+
+// A slightly more complex test to make sure origin frecency stats are updated
+// when visits and bookmarks are added and removed.
+add_task(async function moreOriginFrecencyStats() {
+  await checkDB([]);
+
+  // Add a URL 0 visit.
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/0" }]);
+  await checkDB([["http://", "example.com", ["http://example.com/0"]]]);
+
+  // Add a URL 1 visit.
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/1" }]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1"],
+    ],
+  ]);
+
+  // Add a URL 2 visit.
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/2" }]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1", "http://example.com/2"],
+    ],
+  ]);
+
+  // Add another URL 2 visit.
+  await PlacesTestUtils.addVisits([{ uri: "http://example.com/2" }]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1", "http://example.com/2"],
+    ],
+  ]);
+
+  // Remove URL 2's visits.
+  await PlacesUtils.history.remove(["http://example.com/2"]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1"],
+    ],
+  ]);
+
+  // Bookmark URL 1.
+  let bookmark = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "A bookmark",
+    url: NetUtil.newURI("http://example.com/1"),
+  });
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1"],
+    ],
+  ]);
+
+  // Remove URL 1's visit.
+  await PlacesUtils.history.remove(["http://example.com/1"]);
+  await checkDB([
+    [
+      "http://",
+      "example.com",
+      ["http://example.com/0", "http://example.com/1"],
+    ],
+  ]);
+
+  // Remove URL 1's bookmark.  Also need to call history.remove() again to
+  // remove the URL from moz_places.  Otherwise it sticks around and keeps
+  // contributing to the frecency stats.
+  await PlacesUtils.bookmarks.remove(bookmark);
+  await PlacesUtils.history.remove("http://example.com/1");
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  await checkDB([["http://", "example.com", ["http://example.com/0"]]]);
+
+  // Remove URL 0.
+  await PlacesUtils.history.remove(["http://example.com/0"]);
+  await checkDB([]);
+
+  await cleanUp();
+});
+
+/**
+ * Returns the expected frecency of the origin of the given URLs, i.e., the sum
+ * of their frecencies.  Each URL is expected to have the same origin.
+ *
+ * @param  urls
+ *         An array of URL strings.
+ * @return The expected origin frecency.
+ */
+async function expectedOriginFrecency(urls) {
+  let value = 0;
+  for (let url of urls) {
+    let v = Math.max(
+      await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", { url }),
+      0
+    );
+    value += v;
+  }
+  return value;
+}
+
+/**
+ * Asserts that the moz_origins table and the origin frecency stats are correct.
+ *
+ * @param expectedOrigins
+ *        An array of expected origins.  Each origin in the array is itself an
+ *        array that looks like this:
+ *          [prefix, host, [url1, url2, ..., urln]]
+ *        The element at index 2 is an array of all the URLs with the origin.
+ *        If you don't care about checking frecencies and origin frecency stats,
+ *        this element can be `undefined`.
+ */
+async function checkDB(expectedOrigins) {
+  // Frencencies for bookmarks are generated asynchronously but not within the
+  // await cycle for bookmarks.insert() etc, so wait for them to happen.
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT prefix, host, frecency
+    FROM moz_origins
+    ORDER BY id ASC
+  `);
+  let checkFrecencies =
+    !expectedOrigins.length || expectedOrigins[0][2] !== undefined;
+  let actualOrigins = rows.map(row => {
+    let o = [];
+    for (let c = 0; c < (checkFrecencies ? 3 : 2); c++) {
+      o.push(row.getResultByIndex(c));
+    }
+    return o;
+  });
+  let expected = [];
+  for (let origin of expectedOrigins) {
+    expected.push(
+      origin
+        .slice(0, 2)
+        .concat(checkFrecencies ? await expectedOriginFrecency(origin[2]) : [])
+    );
+  }
+  Assert.deepEqual(actualOrigins, expected);
+  if (checkFrecencies) {
+    await checkStats(expected.map(o => o[2]).filter(o => o > 0));
+  }
+}
+
+/**
+ * Asserts that the origin frecency stats are correct.
+ *
+ * @param expectedOriginFrecencies
+ *        An array of expected origin frecencies.
+ */
+async function checkStats(expectedOriginFrecencies) {
+  let stats = await promiseStats();
+  Assert.equal(stats.count, expectedOriginFrecencies.length);
+  Assert.equal(
+    stats.sum,
+    expectedOriginFrecencies.reduce((sum, f) => sum + f, 0)
+  );
+  Assert.equal(
+    stats.squares,
+    expectedOriginFrecencies.reduce((squares, f) => squares + f * f, 0)
+  );
+}
+
+/**
+ * Returns the origin frecency stats.
+ *
+ * @return An object: { count, sum, squares }
+ */
+async function promiseStats() {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT
+      IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_count'), 0),
+      IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_sum'), 0),
+      IFNULL((SELECT value FROM moz_meta WHERE key = 'origin_frecency_sum_of_squares'), 0)
+  `);
+  return {
+    count: rows[0].getResultByIndex(0),
+    sum: rows[0].getResultByIndex(1),
+    squares: rows[0].getResultByIndex(2),
+  };
+}
+
+async function cleanUp() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+}
diff --git a/toolkit/components/places/tests/unit/test_origins_parsing.js b/toolkit/components/places/tests/unit/test_origins_parsing.js
new file mode 100644
index 0000000000..35ba8bdd0d
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_origins_parsing.js
@@ -0,0 +1,104 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test is a companion to test_origins.js.  It adds many URLs to the
+// database and makes sure that their prefixes and hosts are correctly parsed.
+// This test can take a while to run, which is why it's split out from
+// test_origins.js.
+
+"use strict";
+
+add_task(async function parsing() {
+  let prefixes = ["http://", "https://", "ftp://", "foo://", "bar:"];
+
+  let userinfos = ["", "user:pass@", "user:pass:word@", "user:@"];
+
+  let ports = ["", ":8888"];
+
+  let paths = [
+    "",
+
+    "/",
+    "/1",
+    "/1/2",
+
+    "?",
+    "?1",
+    "#",
+    "#1",
+
+    "/?",
+    "/1?",
+    "/?1",
+    "/1?2",
+
+    "/#",
+    "/1#",
+    "/#1",
+    "/1#2",
+
+    "/?#",
+    "/1?#",
+    "/?1#",
+    "/?#1",
+    "/1?2#",
+    "/1?#2",
+    "/?1#2",
+  ];
+
+  for (let userinfo of userinfos) {
+    for (let port of ports) {
+      for (let path of paths) {
+        info(`Testing userinfo='${userinfo}' port='${port}' path='${path}'`);
+        let expectedOrigins = prefixes.map(prefix => [
+          prefix,
+          "example.com" + port,
+        ]);
+        let uris = expectedOrigins.map(
+          ([prefix, hostPort]) => prefix + userinfo + hostPort + path
+        );
+
+        await PlacesTestUtils.addVisits(uris.map(uri => ({ uri })));
+        await checkDB(expectedOrigins);
+
+        // Remove each URI, one at a time, and make sure the remaining origins
+        // in the database are correct.
+        for (let i = 0; i < uris.length; i++) {
+          await PlacesUtils.history.remove(uris[i]);
+          await checkDB(expectedOrigins.slice(i + 1, expectedOrigins.length));
+        }
+        await cleanUp();
+      }
+    }
+  }
+  await checkDB([]);
+});
+
+/**
+ * Asserts that the moz_origins table is correct.
+ *
+ * @param expectedOrigins
+ *        An array of expected origins.  Each origin in the array is itself an
+ *        array that looks like this: [prefix, host]
+ */
+async function checkDB(expectedOrigins) {
+  let db = await PlacesUtils.promiseDBConnection();
+  let rows = await db.execute(`
+    SELECT prefix, host
+    FROM moz_origins
+    ORDER BY id ASC
+  `);
+  let actualOrigins = rows.map(row => {
+    let o = [];
+    for (let c = 0; c < 2; c++) {
+      o.push(row.getResultByIndex(c));
+    }
+    return o;
+  });
+  Assert.deepEqual(actualOrigins, expectedOrigins);
+}
+
+async function cleanUp() {
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+}
diff --git a/toolkit/components/places/tests/unit/test_pageGuid_bookmarkGuid.js b/toolkit/components/places/tests/unit/test_pageGuid_bookmarkGuid.js
new file mode 100644
index 0000000000..e1fa64a88c
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_pageGuid_bookmarkGuid.js
@@ -0,0 +1,256 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const histsvc = PlacesUtils.history;
+
+add_task(async function test_addBookmarksAndCheckGuids() {
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "test folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            url: "http://test1.com/",
+            title: "1 title",
+          },
+          {
+            url: "http://test2.com/",
+            title: "2 title",
+          },
+          {
+            url: "http://test3.com/",
+            title: "3 title",
+          },
+          {
+            type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+          },
+          {
+            title: "test folder 2",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          },
+        ],
+      },
+    ],
+  });
+
+  let root = PlacesUtils.getFolderContents(bookmarks[0].guid).root;
+  Assert.equal(root.childCount, 5);
+
+  // check bookmark guids
+  let bookmarkGuidZero = root.getChild(0).bookmarkGuid;
+  Assert.equal(bookmarkGuidZero.length, 12);
+  // bookmarks have bookmark guids
+  Assert.equal(root.getChild(1).bookmarkGuid.length, 12);
+  Assert.equal(root.getChild(2).bookmarkGuid.length, 12);
+  // separator has bookmark guid
+  Assert.equal(root.getChild(3).bookmarkGuid.length, 12);
+  // folder has bookmark guid
+  Assert.equal(root.getChild(4).bookmarkGuid.length, 12);
+  // all bookmark guids are different.
+  Assert.notEqual(bookmarkGuidZero, root.getChild(1).bookmarkGuid);
+  Assert.notEqual(root.getChild(1).bookmarkGuid, root.getChild(2).bookmarkGuid);
+  Assert.notEqual(root.getChild(2).bookmarkGuid, root.getChild(3).bookmarkGuid);
+  Assert.notEqual(root.getChild(3).bookmarkGuid, root.getChild(4).bookmarkGuid);
+
+  // check page guids
+  let pageGuidZero = root.getChild(0).pageGuid;
+  Assert.equal(pageGuidZero.length, 12);
+  // bookmarks have page guids
+  Assert.equal(root.getChild(1).pageGuid.length, 12);
+  Assert.equal(root.getChild(2).pageGuid.length, 12);
+  // folder and separator don't have page guids
+  Assert.equal(root.getChild(3).pageGuid, "");
+  Assert.equal(root.getChild(4).pageGuid, "");
+
+  Assert.notEqual(pageGuidZero, root.getChild(1).pageGuid);
+  Assert.notEqual(root.getChild(1).pageGuid, root.getChild(2).pageGuid);
+
+  root.containerOpen = false;
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_updateBookmarksAndCheckGuids() {
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "test folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            url: "http://test1.com/",
+            title: "1 title",
+          },
+          {
+            title: "test folder 2",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+          },
+        ],
+      },
+    ],
+  });
+
+  let root = PlacesUtils.getFolderContents(bookmarks[0].guid).root;
+  Assert.equal(root.childCount, 2);
+
+  // ensure the bookmark and page guids remain the same after modifing other property.
+  let bookmarkGuidZero = root.getChild(0).bookmarkGuid;
+  let pageGuidZero = root.getChild(0).pageGuid;
+  await PlacesUtils.bookmarks.update({
+    guid: bookmarks[1].guid,
+    title: "1 title mod",
+  });
+  Assert.equal(root.getChild(0).title, "1 title mod");
+  Assert.equal(root.getChild(0).bookmarkGuid, bookmarkGuidZero);
+  Assert.equal(root.getChild(0).pageGuid, pageGuidZero);
+
+  let bookmarkGuidOne = root.getChild(1).bookmarkGuid;
+  let pageGuidOne = root.getChild(1).pageGuid;
+
+  await PlacesUtils.bookmarks.update({
+    guid: bookmarks[2].guid,
+    title: "test foolder 234",
+  });
+  Assert.equal(root.getChild(1).title, "test foolder 234");
+  Assert.equal(root.getChild(1).bookmarkGuid, bookmarkGuidOne);
+  Assert.equal(root.getChild(1).pageGuid, pageGuidOne);
+
+  root.containerOpen = false;
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_addVisitAndCheckGuid() {
+  // add a visit and test page guid and non-existing bookmark guids.
+  let sourceURI = uri("http://test4.com/");
+  await PlacesTestUtils.addVisits({ uri: sourceURI });
+  Assert.equal(await PlacesUtils.bookmarks.fetch({ url: sourceURI }, null));
+
+  let options = histsvc.getNewQueryOptions();
+  let query = histsvc.getNewQuery();
+  query.uri = sourceURI;
+  let root = histsvc.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+
+  do_check_valid_places_guid(root.getChild(0).pageGuid);
+  Assert.equal(root.getChild(0).bookmarkGuid, "");
+  root.containerOpen = false;
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_addItemsWithInvalidGUIDsFails() {
+  const INVALID_GUID = "XYZ";
+  try {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      guid: INVALID_GUID,
+      title: "XYZ folder",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    });
+    do_throw("Adding a folder with an invalid guid should fail");
+  } catch (ex) {}
+
+  let folder = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    title: "test folder",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+  });
+  try {
+    PlacesUtils.bookmarks.insert({
+      parentGuid: folder.guid,
+      guid: INVALID_GUID,
+      title: "title",
+      url: "http://test.tld",
+    });
+    do_throw("Adding a bookmark with an invalid guid should fail");
+  } catch (ex) {}
+
+  try {
+    PlacesUtils.bookmarks.insert({
+      parentGuid: folder.guid,
+      guid: INVALID_GUID,
+      type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+    });
+    do_throw("Adding a separator with an invalid guid should fail");
+  } catch (ex) {}
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_addItemsWithGUIDs() {
+  const FOLDER_GUID = "FOLDER--GUID";
+  const BOOKMARK_GUID = "BM------GUID";
+  const SEPARATOR_GUID = "SEP-----GUID";
+
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "test folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        guid: FOLDER_GUID,
+        children: [
+          {
+            url: "http://test1.com",
+            title: "1 title",
+            guid: BOOKMARK_GUID,
+          },
+          {
+            type: PlacesUtils.bookmarks.TYPE_SEPARATOR,
+            guid: SEPARATOR_GUID,
+          },
+        ],
+      },
+    ],
+  });
+
+  let root = PlacesUtils.getFolderContents(bookmarks[0].guid).root;
+  Assert.equal(root.childCount, 2);
+  Assert.equal(root.bookmarkGuid, FOLDER_GUID);
+  Assert.equal(root.getChild(0).bookmarkGuid, BOOKMARK_GUID);
+  Assert.equal(root.getChild(1).bookmarkGuid, SEPARATOR_GUID);
+
+  root.containerOpen = false;
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_emptyGUIDFails() {
+  try {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      guid: "",
+      title: "test folder",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    });
+    do_throw("Adding a folder with an empty guid should fail");
+  } catch (ex) {}
+});
+
+add_task(async function test_usingSameGUIDFails() {
+  const GUID = "XYZXYZXYZXYZ";
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    guid: GUID,
+    title: "test folder",
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+  });
+  try {
+    await PlacesUtils.bookmarks.insert({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      guid: GUID,
+      title: "test folder 2",
+      type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    });
+    do_throw("Using the same guid twice should fail");
+  } catch (ex) {}
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
diff --git a/toolkit/components/places/tests/unit/test_placeURIs.js b/toolkit/components/places/tests/unit/test_placeURIs.js
new file mode 100644
index 0000000000..093dc89f68
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_placeURIs.js
@@ -0,0 +1,18 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function run_test() {
+  // TODO: Improve testing coverage for QueryToQueryString and QueryStringToQuery
+
+  // Bug 376798
+  let query = PlacesUtils.history.getNewQuery();
+  let options = PlacesUtils.history.getNewQueryOptions();
+  query.setParents([PlacesUtils.bookmarks.rootGuid]);
+  Assert.equal(
+    PlacesUtils.history.queryToQueryString(query, options),
+    `place:parent=${PlacesUtils.bookmarks.rootGuid}`
+  );
+}
diff --git a/toolkit/components/places/tests/unit/test_promiseBookmarksTree.js b/toolkit/components/places/tests/unit/test_promiseBookmarksTree.js
new file mode 100644
index 0000000000..9824908cef
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_promiseBookmarksTree.js
@@ -0,0 +1,282 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+async function check_has_child(aParentGuid, aChildGuid) {
+  let parentTree = await PlacesUtils.promiseBookmarksTree(aParentGuid);
+  Assert.ok("children" in parentTree);
+  Assert.notEqual(
+    parentTree.children.find(e => e.guid == aChildGuid),
+    null
+  );
+}
+
+async function compareToNode(aItem, aNode, aIsRootItem, aExcludedGuids = []) {
+  // itemId==-1 indicates a non-bookmark node, which is unexpected.
+  Assert.notEqual(aNode.itemId, -1);
+
+  function check_unset(...aProps) {
+    for (let p of aProps) {
+      if (p in aItem) {
+        Assert.ok(false, `Unexpected property ${p} with value ${aItem[p]}`);
+      }
+    }
+  }
+
+  function compare_prop(aItemProp, aNodeProp = aItemProp, aOptional = false) {
+    if (aOptional && aNode[aNodeProp] === null) {
+      check_unset(aItemProp);
+    } else {
+      Assert.strictEqual(aItem[aItemProp], aNode[aNodeProp]);
+    }
+  }
+
+  // Bug 1013053 - bookmarkIndex is unavailable for the query's root
+  if (aNode.bookmarkIndex == -1) {
+    let bookmark = await PlacesUtils.bookmarks.fetch(aNode.bookmarkGuid);
+    Assert.strictEqual(aItem.index, bookmark.index);
+  } else {
+    compare_prop("index", "bookmarkIndex");
+  }
+
+  compare_prop("dateAdded");
+  compare_prop("lastModified");
+
+  if (aIsRootItem && aNode.bookmarkGuid != PlacesUtils.bookmarks.rootGuid) {
+    Assert.ok("parentGuid" in aItem);
+    await check_has_child(aItem.parentGuid, aItem.guid);
+  } else {
+    check_unset("parentGuid");
+  }
+
+  const BOOKMARK_ONLY_PROPS = ["uri", "iconUri", "tags", "charset", "keyword"];
+  const FOLDER_ONLY_PROPS = ["children", "root"];
+
+  let nodesCount = 1;
+
+  switch (aNode.type) {
+    case Ci.nsINavHistoryResultNode.RESULT_TYPE_FOLDER:
+      Assert.equal(aItem.type, PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER);
+      Assert.equal(aItem.typeCode, PlacesUtils.bookmarks.TYPE_FOLDER);
+      compare_prop("title", "title", true);
+      check_unset(...BOOKMARK_ONLY_PROPS);
+
+      let expectedChildrenNodes = [];
+
+      PlacesUtils.asContainer(aNode);
+      if (!aNode.containerOpen) {
+        aNode.containerOpen = true;
+      }
+
+      for (let i = 0; i < aNode.childCount; i++) {
+        let childNode = aNode.getChild(i);
+        if (
+          childNode.itemId == PlacesUtils.tagsFolderId ||
+          aExcludedGuids.includes(childNode.bookmarkGuid)
+        ) {
+          continue;
+        }
+        expectedChildrenNodes.push(childNode);
+      }
+
+      if (expectedChildrenNodes.length) {
+        Assert.ok(Array.isArray(aItem.children));
+        Assert.equal(aItem.children.length, expectedChildrenNodes.length);
+        for (let i = 0; i < aItem.children.length; i++) {
+          nodesCount += await compareToNode(
+            aItem.children[i],
+            expectedChildrenNodes[i],
+            false,
+            aExcludedGuids
+          );
+        }
+      } else {
+        check_unset("children");
+      }
+
+      let rootName = mapItemGuidToInternalRootName(aItem.guid);
+      if (rootName) {
+        Assert.equal(aItem.root, rootName);
+      } else {
+        check_unset("root");
+      }
+      break;
+    case Ci.nsINavHistoryResultNode.RESULT_TYPE_SEPARATOR:
+      Assert.equal(aItem.type, PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR);
+      Assert.equal(aItem.typeCode, PlacesUtils.bookmarks.TYPE_SEPARATOR);
+      check_unset(...BOOKMARK_ONLY_PROPS, ...FOLDER_ONLY_PROPS);
+      break;
+    default:
+      Assert.equal(aItem.type, PlacesUtils.TYPE_X_MOZ_PLACE);
+      Assert.equal(aItem.typeCode, PlacesUtils.bookmarks.TYPE_BOOKMARK);
+      compare_prop("uri");
+      // node.tags's format is "a, b" whilst promiseBoookmarksTree is "a,b"
+      if (aNode.tags === null) {
+        check_unset("tags");
+      } else {
+        Assert.equal(aItem.tags, aNode.tags.replace(/, /g, ","));
+      }
+
+      if (aNode.icon) {
+        try {
+          await compareFavicons(aNode.icon, aItem.iconUri);
+        } catch (ex) {
+          info(ex);
+          todo_check_true(false);
+        }
+      } else {
+        check_unset(aItem.iconUri);
+      }
+
+      check_unset(...FOLDER_ONLY_PROPS);
+
+      let pageInfo = await PlacesUtils.history.fetch(aNode.uri, {
+        includeAnnotations: true,
+      });
+      let expectedCharset = pageInfo.annotations.get(PlacesUtils.CHARSET_ANNO);
+      if (expectedCharset) {
+        Assert.equal(aItem.charset, expectedCharset);
+      } else {
+        check_unset("charset");
+      }
+
+      let entry = await PlacesUtils.keywords.fetch({ url: aNode.uri });
+      if (entry) {
+        Assert.equal(aItem.keyword, entry.keyword);
+      } else {
+        check_unset("keyword");
+      }
+
+      if ("title" in aItem) {
+        compare_prop("title");
+      } else {
+        Assert.equal(null, aNode.title);
+      }
+  }
+
+  if (aIsRootItem) {
+    Assert.strictEqual(aItem.itemsCount, nodesCount);
+  }
+
+  return nodesCount;
+}
+
+var itemsCount = 0;
+async function new_bookmark(aInfo) {
+  ++itemsCount;
+  if (!("url" in aInfo)) {
+    aInfo.url = uri("http://test.item." + itemsCount);
+  }
+
+  if (!("title" in aInfo)) {
+    aInfo.title = "Test Item (bookmark) " + itemsCount;
+  }
+
+  await PlacesTransactions.NewBookmark(aInfo).transact();
+}
+
+function new_folder(aInfo) {
+  if (!("title" in aInfo)) {
+    aInfo.title = "Test Item (folder) " + itemsCount;
+  }
+  return PlacesTransactions.NewFolder(aInfo).transact();
+}
+
+// Walks a result nodes tree and test promiseBookmarksTree for each node.
+// DO NOT COPY THIS LOGIC:  It is done here to accomplish a more comprehensive
+// test of the API (the entire hierarchy data is available in the very test).
+async function test_promiseBookmarksTreeForEachNode(
+  aNode,
+  aOptions,
+  aExcludedGuids
+) {
+  Assert.ok(aNode.bookmarkGuid && !!aNode.bookmarkGuid.length);
+  let item = await PlacesUtils.promiseBookmarksTree(
+    aNode.bookmarkGuid,
+    aOptions
+  );
+  await compareToNode(item, aNode, true, aExcludedGuids);
+
+  if (!PlacesUtils.nodeIsContainer(aNode)) {
+    return item;
+  }
+
+  for (let i = 0; i < aNode.childCount; i++) {
+    let child = aNode.getChild(i);
+    if (child.itemId != PlacesUtils.tagsFolderId) {
+      await test_promiseBookmarksTreeForEachNode(
+        child,
+        { includeItemIds: true },
+        aExcludedGuids
+      );
+    }
+  }
+  return item;
+}
+
+async function test_promiseBookmarksTreeAgainstResult(
+  aItemGuid = PlacesUtils.bookmarks.rootGuid,
+  aOptions = { includeItemIds: true },
+  aExcludedGuids
+) {
+  let node = PlacesUtils.getFolderContents(aItemGuid).root;
+  return test_promiseBookmarksTreeForEachNode(node, aOptions, aExcludedGuids);
+}
+
+add_task(async function () {
+  // Add some bookmarks to cover various use cases.
+  await new_bookmark({ parentGuid: PlacesUtils.bookmarks.toolbarGuid });
+  await new_folder({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    annotations: [
+      { name: "TestAnnoA", value: "TestVal" },
+      { name: "TestAnnoB", value: 0 },
+    ],
+  });
+  let sepInfo = { parentGuid: PlacesUtils.bookmarks.menuGuid };
+  await PlacesTransactions.NewSeparator(sepInfo).transact();
+  let folderGuid = await new_folder({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+  });
+  await new_bookmark({
+    title: null,
+    parentGuid: folderGuid,
+    keyword: "test_keyword",
+    tags: ["TestTagA", "TestTagB"],
+    annotations: [{ name: "TestAnnoA", value: "TestVal2" }],
+  });
+  let urlWithCharsetAndFavicon = uri("http://charset.and.favicon");
+  await new_bookmark({ parentGuid: folderGuid, url: urlWithCharsetAndFavicon });
+  await PlacesUtils.history.update({
+    url: urlWithCharsetAndFavicon,
+    annotations: new Map([[PlacesUtils.CHARSET_ANNO, "UTF-16"]]),
+  });
+  await setFaviconForPage(urlWithCharsetAndFavicon, SMALLPNG_DATA_URI);
+  // Test the default places root without specifying it.
+  await test_promiseBookmarksTreeAgainstResult();
+
+  // Do specify it
+  await test_promiseBookmarksTreeAgainstResult(PlacesUtils.bookmarks.rootGuid);
+
+  // Exclude the bookmarks menu.
+  // The calllback should be four times - once for the toolbar, once for
+  // the bookmark we inserted under, and once for the menu (and not
+  // at all for any of its descendants) and once for the unsorted bookmarks
+  // folder.  However, promiseBookmarksTree is called multiple times, so
+  // rather than counting the calls, we count the number of unique items
+  // passed in.
+  let guidsPassedToExcludeCallback = new Set();
+  let placesRootWithoutTheMenu = await test_promiseBookmarksTreeAgainstResult(
+    PlacesUtils.bookmarks.rootGuid,
+    {
+      excludeItemsCallback: aItem => {
+        guidsPassedToExcludeCallback.add(aItem.guid);
+        return aItem.root == "bookmarksMenuFolder";
+      },
+      includeItemIds: true,
+    },
+    [PlacesUtils.bookmarks.menuGuid]
+  );
+  Assert.equal(guidsPassedToExcludeCallback.size, 5);
+  Assert.equal(placesRootWithoutTheMenu.children.length, 3);
+});
diff --git a/toolkit/components/places/tests/unit/test_resolveNullBookmarkTitles.js b/toolkit/components/places/tests/unit/test_resolveNullBookmarkTitles.js
new file mode 100644
index 0000000000..d8830c5686
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_resolveNullBookmarkTitles.js
@@ -0,0 +1,39 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+add_task(async function () {
+  let uri1 = uri("http://foo.tld/");
+  let uri2 = uri("https://bar.tld/");
+
+  await PlacesTestUtils.addVisits([
+    { uri: uri1, title: "foo title" },
+    { uri: uri2, title: "bar title" },
+  ]);
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: uri1,
+    title: null,
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: uri2,
+    title: null,
+  });
+
+  PlacesUtils.tagging.tagURI(uri1, ["tag 1"]);
+  PlacesUtils.tagging.tagURI(uri2, ["tag 2"]);
+});
+
+add_task(async function testTagQuery() {
+  let options = PlacesUtils.history.getNewQueryOptions();
+  let query = PlacesUtils.history.getNewQuery();
+  query.tags = ["tag 1"];
+  let root = PlacesUtils.history.executeQuery(query, options).root;
+  root.containerOpen = true;
+  Assert.equal(root.childCount, 1);
+  Assert.equal(root.getChild(0).title, "");
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_result_sort.js b/toolkit/components/places/tests/unit/test_result_sort.js
new file mode 100644
index 0000000000..840b40c339
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_result_sort.js
@@ -0,0 +1,112 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const NHQO = Ci.nsINavHistoryQueryOptions;
+
+add_task(async function test() {
+  const uri1 = "http://foo.tld/a";
+  const uri2 = "http://foo.tld/b";
+
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "Result-sort functionality tests root",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            title: "b",
+            url: uri1,
+          },
+          {
+            title: "a",
+            url: uri2,
+          },
+          {
+            // url of the first child, title of second
+            title: "a",
+            url: uri1,
+          },
+        ],
+      },
+    ],
+  });
+
+  let guid1 = bookmarks[1].guid;
+  let guid2 = bookmarks[2].guid;
+  let guid3 = bookmarks[3].guid;
+
+  // query with natural order
+  let result = PlacesUtils.getFolderContents(bookmarks[0].guid);
+  let root = result.root;
+
+  Assert.equal(root.childCount, 3);
+
+  function checkOrder(a, b, c) {
+    Assert.equal(root.getChild(0).bookmarkGuid, a);
+    Assert.equal(root.getChild(1).bookmarkGuid, b);
+    Assert.equal(root.getChild(2).bookmarkGuid, c);
+  }
+
+  // natural order
+  info("Natural order");
+  checkOrder(guid1, guid2, guid3);
+
+  // title: guid3 should precede guid2 since we fall-back to URI-based sorting
+  info("Sort by title asc");
+  result.sortingMode = NHQO.SORT_BY_TITLE_ASCENDING;
+  checkOrder(guid3, guid2, guid1);
+
+  // In reverse
+  info("Sort by title desc");
+  result.sortingMode = NHQO.SORT_BY_TITLE_DESCENDING;
+  checkOrder(guid1, guid2, guid3);
+
+  // uri sort: guid1 should precede guid3 since we fall-back to natural order
+  info("Sort by uri asc");
+  result.sortingMode = NHQO.SORT_BY_URI_ASCENDING;
+  checkOrder(guid1, guid3, guid2);
+
+  // test live update
+  info("Change bookmark uri liveupdate");
+  await PlacesUtils.bookmarks.update({
+    guid: guid1,
+    url: uri2,
+  });
+  checkOrder(guid3, guid1, guid2);
+  await PlacesUtils.bookmarks.update({
+    guid: guid1,
+    url: uri1,
+  });
+  checkOrder(guid1, guid3, guid2);
+
+  // XXXtodo: test history sortings (visit count, visit date)
+  // XXXtodo: test different item types once folderId and bookmarkId are merged.
+
+  // XXXtodo:  test dateAdded sort
+  // XXXtodo:  test lastModified sort
+
+  // Add a visit, then check frecency ordering.
+
+  await PlacesTestUtils.addVisits({ uri: uri2, transition: TRANSITION_TYPED });
+
+  info("Sort by frecency desc");
+  result.sortingMode = NHQO.SORT_BY_FRECENCY_DESCENDING;
+  for (let i = 0; i < root.childCount; ++i) {
+    print(root.getChild(i).uri + " " + root.getChild(i).title);
+  }
+  // For guid1 and guid3, since they have same frecency and no visits, fallback
+  // to sort by the newest bookmark.
+  checkOrder(guid2, guid3, guid1);
+  info("Sort by frecency asc");
+  result.sortingMode = NHQO.SORT_BY_FRECENCY_ASCENDING;
+  for (let i = 0; i < root.childCount; ++i) {
+    print(root.getChild(i).uri + " " + root.getChild(i).title);
+  }
+  checkOrder(guid1, guid3, guid2);
+
+  root.containerOpen = false;
+});
diff --git a/toolkit/components/places/tests/unit/test_resultsAsVisit_details.js b/toolkit/components/places/tests/unit/test_resultsAsVisit_details.js
new file mode 100644
index 0000000000..2291eddeff
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_resultsAsVisit_details.js
@@ -0,0 +1,106 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+const { history } = PlacesUtils;
+
+add_task(async function test_addVisitCheckFields() {
+  let uri = NetUtil.newURI("http://test4.com/");
+  await PlacesTestUtils.addVisits([
+    { uri },
+    { uri, referrer: uri },
+    { uri, transition: history.TRANSITION_TYPED },
+  ]);
+
+  let options = history.getNewQueryOptions();
+  let query = history.getNewQuery();
+
+  query.uri = uri;
+
+  // Check RESULTS_AS_VISIT node.
+  options.resultType = options.RESULTS_AS_VISIT;
+
+  let root = history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  equal(root.childCount, 3);
+
+  let child = root.getChild(0);
+  equal(
+    child.visitType,
+    history.TRANSITION_LINK,
+    "Visit type should be TRANSITION_LINK"
+  );
+  equal(child.visitId, 1, "Visit ID should be 1");
+  equal(child.fromVisitId, -1, "Should have no referrer visit ID");
+
+  child = root.getChild(1);
+  equal(
+    child.visitType,
+    history.TRANSITION_LINK,
+    "Visit type should be TRANSITION_LINK"
+  );
+  equal(child.visitId, 2, "Visit ID should be 2");
+  equal(child.fromVisitId, 1, "First visit should be the referring visit");
+
+  child = root.getChild(2);
+  equal(
+    child.visitType,
+    history.TRANSITION_TYPED,
+    "Visit type should be TRANSITION_TYPED"
+  );
+  equal(child.visitId, 3, "Visit ID should be 3");
+  equal(child.fromVisitId, -1, "Should have no referrer visit ID");
+
+  root.containerOpen = false;
+
+  // Check RESULTS_AS_URI node.
+  options.resultType = options.RESULTS_AS_URI;
+
+  root = history.executeQuery(query, options).root;
+  root.containerOpen = true;
+
+  equal(root.childCount, 1);
+
+  child = root.getChild(0);
+  equal(child.visitType, 0, "Visit type should be 0");
+  equal(child.visitId, -1, "Visit ID should be -1");
+  equal(child.fromVisitId, -1, "Referrer visit id should be -1");
+
+  root.containerOpen = false;
+
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function test_bookmarkFields() {
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        title: "test folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            title: "test title",
+            url: "http://test4.com",
+          },
+        ],
+      },
+    ],
+  });
+
+  let root = PlacesUtils.getFolderContents(bookmarks[0].guid).root;
+  equal(root.childCount, 1);
+
+  equal(root.visitType, 0, "Visit type should be 0");
+  equal(root.visitId, -1, "Visit ID should be -1");
+  equal(root.fromVisitId, -1, "Referrer visit id should be -1");
+
+  let child = root.getChild(0);
+  equal(child.visitType, 0, "Visit type should be 0");
+  equal(child.visitId, -1, "Visit ID should be -1");
+  equal(child.fromVisitId, -1, "Referrer visit id should be -1");
+
+  root.containerOpen = false;
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
diff --git a/toolkit/components/places/tests/unit/test_sql_function_origin.js b/toolkit/components/places/tests/unit/test_sql_function_origin.js
new file mode 100644
index 0000000000..0314ff5040
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_sql_function_origin.js
@@ -0,0 +1,79 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests the origin-related SQL functions, which are:
+// * get_host_and_port
+// * get_prefix
+// * strip_prefix_and_userinfo
+
+// Tests actual URL strings.
+add_task(async function urls() {
+  let sets = [
+    ["http:"],
+    ["", "//"],
+    ["", "user@", "user:@", "user:pass@", "user:pass:word@"],
+    ["example.com"],
+    ["", ":8888"],
+    ["", "/", "/foo"],
+    ["", "?", "?bar"],
+    ["", "#", "#baz"],
+  ];
+  let db = await PlacesUtils.promiseDBConnection();
+  for (let parts of permute(sets)) {
+    let spec = parts.join("");
+    let funcs = {
+      get_prefix: parts.slice(0, 2).join(""),
+      get_host_and_port: parts.slice(3, 5).join(""),
+      strip_prefix_and_userinfo: parts.slice(3).join(""),
+    };
+    for (let [func, expectedValue] of Object.entries(funcs)) {
+      let rows = await db.execute(`
+        SELECT ${func}("${spec}");
+      `);
+      let value = rows[0].getString(0);
+      Assert.equal(value, expectedValue, `function=${func} spec="${spec}"`);
+    }
+  }
+});
+
+// Tests strings that aren't URLs.
+add_task(async function nonURLs() {
+  let db = await PlacesUtils.promiseDBConnection();
+
+  let value = (
+    await db.execute(`
+    SELECT get_prefix("hello");
+  `)
+  )[0].getString(0);
+  Assert.equal(value, "");
+
+  value = (
+    await db.execute(`
+    SELECT get_host_and_port("hello");
+  `)
+  )[0].getString(0);
+  Assert.equal(value, "hello");
+
+  value = (
+    await db.execute(`
+    SELECT strip_prefix_and_userinfo("hello");
+  `)
+  )[0].getString(0);
+  Assert.equal(value, "hello");
+});
+
+function permute(sets = []) {
+  if (!sets.length) {
+    return [[]];
+  }
+  let firstSet = sets[0];
+  let otherSets = sets.slice(1);
+  let permutedSequences = [];
+  let otherPermutedSequences = permute(otherSets);
+  for (let other of otherPermutedSequences) {
+    for (let value of firstSet) {
+      permutedSequences.push([value].concat(other));
+    }
+  }
+  return permutedSequences;
+}
diff --git a/toolkit/components/places/tests/unit/test_sql_guid_functions.js b/toolkit/components/places/tests/unit/test_sql_guid_functions.js
new file mode 100644
index 0000000000..b4fbdac612
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_sql_guid_functions.js
@@ -0,0 +1,94 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * This file tests that the guid function generates a guid of the proper length,
+ * with no invalid characters.
+ */
+
+/**
+ * Checks all our invariants about our guids for a given result.
+ *
+ * @param aGuid
+ *        The guid to check.
+ */
+function check_invariants(aGuid) {
+  info("Checking guid '" + aGuid + "'");
+
+  do_check_valid_places_guid(aGuid);
+}
+
+// Test Functions
+
+function test_guid_invariants() {
+  const kExpectedChars = 64;
+  const kAllowedChars =
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_";
+  Assert.equal(kAllowedChars.length, kExpectedChars);
+  const kGuidLength = 12;
+
+  let checkedChars = [];
+  for (let i = 0; i < kGuidLength; i++) {
+    checkedChars[i] = {};
+    for (let j = 0; j < kAllowedChars; j++) {
+      checkedChars[i][kAllowedChars[j]] = false;
+    }
+  }
+
+  // We run this until we've seen every character that we expect to see in every
+  // position.
+  let seenChars = 0;
+  let stmt = DBConn().createStatement("SELECT GENERATE_GUID()");
+  while (seenChars != kExpectedChars * kGuidLength) {
+    Assert.ok(stmt.executeStep());
+    let guid = stmt.getString(0);
+    check_invariants(guid);
+
+    for (let i = 0; i < guid.length; i++) {
+      let character = guid[i];
+      if (!checkedChars[i][character]) {
+        checkedChars[i][character] = true;
+        seenChars++;
+      }
+    }
+    stmt.reset();
+  }
+  stmt.finalize();
+
+  // One last reality check - make sure all of our characters were seen.
+  for (let i = 0; i < kGuidLength; i++) {
+    for (let j = 0; j < kAllowedChars; j++) {
+      Assert.ok(checkedChars[i][kAllowedChars[j]]);
+    }
+  }
+
+  run_next_test();
+}
+
+function test_guid_on_background() {
+  // We should not assert if we execute this asynchronously.
+  let stmt = DBConn().createAsyncStatement("SELECT GENERATE_GUID()");
+  let checked = false;
+  stmt.executeAsync({
+    handleResult(aResult) {
+      try {
+        let row = aResult.getNextRow();
+        check_invariants(row.getResultByIndex(0));
+        Assert.equal(aResult.getNextRow(), null);
+        checked = true;
+      } catch (e) {
+        do_throw(e);
+      }
+    },
+    handleCompletion(aReason) {
+      Assert.equal(aReason, Ci.mozIStorageStatementCallback.REASON_FINISHED);
+      Assert.ok(checked);
+      run_next_test();
+    },
+  });
+  stmt.finalize();
+}
+
+// Test Runner
+
+[test_guid_invariants, test_guid_on_background].forEach(fn => add_test(fn));
diff --git a/toolkit/components/places/tests/unit/test_tag_autocomplete_search.js b/toolkit/components/places/tests/unit/test_tag_autocomplete_search.js
new file mode 100644
index 0000000000..43f899c237
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_tag_autocomplete_search.js
@@ -0,0 +1,119 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function AutoCompleteInput(aSearches) {
+  this.searches = aSearches;
+}
+AutoCompleteInput.prototype = {
+  constructor: AutoCompleteInput,
+
+  searches: null,
+
+  minResultsForPopup: 0,
+  timeout: 10,
+  searchParam: "",
+  textValue: "",
+  disableAutoComplete: false,
+  completeDefaultIndex: false,
+
+  get searchCount() {
+    return this.searches.length;
+  },
+
+  getSearchAt(aIndex) {
+    return this.searches[aIndex];
+  },
+
+  onSearchBegin() {},
+  onSearchComplete() {},
+
+  popupOpen: false,
+
+  popup: {
+    setSelectedIndex(aIndex) {},
+    invalidate() {},
+
+    // nsISupports implementation
+    QueryInterface: ChromeUtils.generateQI(["nsIAutoCompletePopup"]),
+  },
+
+  // nsISupports implementation
+  QueryInterface: ChromeUtils.generateQI(["nsIAutoCompleteInput"]),
+};
+
+async function ensure_tag_results(results, searchTerm) {
+  var controller = Cc["@mozilla.org/autocomplete/controller;1"].getService(
+    Ci.nsIAutoCompleteController
+  );
+
+  // Make an AutoCompleteInput that uses our searches
+  // and confirms results on search complete
+  var input = new AutoCompleteInput(["places-tag-autocomplete"]);
+
+  controller.input = input;
+
+  return new Promise(resolve => {
+    var numSearchesStarted = 0;
+    input.onSearchBegin = function input_onSearchBegin() {
+      numSearchesStarted++;
+      Assert.equal(numSearchesStarted, 1);
+    };
+
+    input.onSearchComplete = function input_onSearchComplete() {
+      Assert.equal(numSearchesStarted, 1);
+      if (results.length) {
+        Assert.equal(
+          controller.searchStatus,
+          Ci.nsIAutoCompleteController.STATUS_COMPLETE_MATCH
+        );
+      } else {
+        Assert.equal(
+          controller.searchStatus,
+          Ci.nsIAutoCompleteController.STATUS_COMPLETE_NO_MATCH
+        );
+      }
+
+      Assert.equal(controller.matchCount, results.length);
+      for (var i = 0; i < controller.matchCount; i++) {
+        Assert.equal(controller.getValueAt(i), results[i]);
+      }
+
+      resolve();
+    };
+
+    controller.startSearch(searchTerm);
+  });
+}
+
+var uri1 = uri("http://site.tld/1");
+
+var tests = [
+  () => ensure_tag_results(["bar", "Baz", "boo"], "b"),
+  () => ensure_tag_results(["bar", "Baz"], "ba"),
+  () => ensure_tag_results(["bar", "Baz"], "Ba"),
+  () => ensure_tag_results(["bar"], "bar"),
+  () => ensure_tag_results(["Baz"], "Baz"),
+  () => ensure_tag_results([], "barb"),
+  () => ensure_tag_results([], "foo"),
+  () =>
+    ensure_tag_results(["first tag, bar", "first tag, Baz"], "first tag, ba"),
+  () =>
+    ensure_tag_results(
+      ["first tag;  bar", "first tag;  Baz"],
+      "first tag;  ba"
+    ),
+];
+
+/**
+ * Test tag autocomplete
+ */
+add_task(async function test_tag_autocomplete() {
+  PlacesUtils.tagging.tagURI(uri1, ["bar", "Baz", "boo", "*nix"]);
+
+  for (let tagTest of tests) {
+    await tagTest();
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_tagging.js b/toolkit/components/places/tests/unit/test_tagging.js
new file mode 100644
index 0000000000..f38db0d0f6
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_tagging.js
@@ -0,0 +1,188 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Notice we use createInstance because later we will have to terminate the
+// service and restart it.
+var tagssvc = Cc["@mozilla.org/browser/tagging-service;1"]
+  .createInstance()
+  .QueryInterface(Ci.nsITaggingService);
+
+function run_test() {
+  var options = PlacesUtils.history.getNewQueryOptions();
+  var query = PlacesUtils.history.getNewQuery();
+
+  query.setParents([PlacesUtils.bookmarks.tagsGuid]);
+  var result = PlacesUtils.history.executeQuery(query, options);
+  var tagRoot = result.root;
+  tagRoot.containerOpen = true;
+
+  Assert.equal(tagRoot.childCount, 0);
+
+  var uri1 = uri("http://foo.tld/");
+  var uri2 = uri("https://bar.tld/");
+
+  // this also tests that the multiple folders are not created for the same tag
+  tagssvc.tagURI(uri1, ["tag 1"]);
+  tagssvc.tagURI(uri2, ["tag 1"]);
+  Assert.equal(tagRoot.childCount, 1);
+
+  var tag1node = tagRoot
+    .getChild(0)
+    .QueryInterface(Ci.nsINavHistoryContainerResultNode);
+  var tag1itemId = tag1node.itemId;
+
+  Assert.equal(tag1node.title, "tag 1");
+  tag1node.containerOpen = true;
+  Assert.equal(tag1node.childCount, 2);
+
+  // Tagging the same url twice (or even thrice!) with the same tag should be a
+  // no-op
+  tagssvc.tagURI(uri1, ["tag 1"]);
+  Assert.equal(tag1node.childCount, 2);
+  tagssvc.tagURI(uri1, [tag1itemId]);
+  Assert.equal(tag1node.childCount, 2);
+  Assert.equal(tagRoot.childCount, 1);
+
+  // also tests bug 407575
+  tagssvc.tagURI(uri1, [tag1itemId, "tag 1", "tag 2", "Tag 1", "Tag 2"]);
+  Assert.equal(tagRoot.childCount, 2);
+  Assert.equal(tag1node.childCount, 2);
+
+  // test getTagsForURI
+  var uri1tags = tagssvc.getTagsForURI(uri1);
+  Assert.equal(uri1tags.length, 2);
+  Assert.equal(uri1tags[0], "Tag 1");
+  Assert.equal(uri1tags[1], "Tag 2");
+  var uri2tags = tagssvc.getTagsForURI(uri2);
+  Assert.equal(uri2tags.length, 1);
+  Assert.equal(uri2tags[0], "Tag 1");
+
+  // test untagging
+  tagssvc.untagURI(uri1, ["tag 1"]);
+  Assert.equal(tag1node.childCount, 1);
+
+  let wait = PlacesTestUtils.waitForNotification("bookmark-removed", events =>
+    events.some(event => event.id == tagRoot.itemId)
+  );
+  // removing the last uri from a tag should remove the tag-container
+  tagssvc.untagURI(uri2, ["tag 1"]);
+  wait.then(() => {
+    Assert.equal(tagRoot.childCount, 1);
+  });
+
+  // cleanup
+  tag1node.containerOpen = false;
+
+  // get array of tag folder ids => title
+  // for testing tagging with mixed folder ids and tags
+  var child = tagRoot.getChild(0);
+  var tagId = child.itemId;
+  var tagTitle = child.title;
+
+  // test mixed id/name tagging
+  // as well as non-id numeric tags
+  var uri3 = uri("http://testuri/3");
+  tagssvc.tagURI(uri3, [tagId, "tag 3", "456"]);
+  var tags = tagssvc.getTagsForURI(uri3);
+  Assert.ok(tags.includes(tagTitle));
+  Assert.ok(tags.includes("tag 3"));
+  Assert.ok(tags.includes("456"));
+
+  // test mixed id/name tagging
+  tagssvc.untagURI(uri3, [tagId, "tag 3", "456"]);
+  tags = tagssvc.getTagsForURI(uri3);
+  Assert.equal(tags.length, 0);
+
+  // Terminate tagging service, fire up a new instance and check that existing
+  // tags are there.  This will ensure that any internal caching system is
+  // correctly filled at startup and we are not losing previously existing tags.
+  var uri4 = uri("http://testuri/4");
+  tagssvc.tagURI(uri4, [tagId, "tag 3", "456"]);
+  tagssvc = null;
+  tagssvc = Cc["@mozilla.org/browser/tagging-service;1"].getService(
+    Ci.nsITaggingService
+  );
+  var uri4Tags = tagssvc.getTagsForURI(uri4);
+  Assert.equal(uri4Tags.length, 3);
+  Assert.ok(uri4Tags.includes(tagTitle));
+  Assert.ok(uri4Tags.includes("tag 3"));
+  Assert.ok(uri4Tags.includes("456"));
+
+  // Test sparse arrays.
+  let curChildCount = tagRoot.childCount;
+
+  try {
+    tagssvc.tagURI(uri1, [undefined, "tagSparse"]);
+    Assert.equal(tagRoot.childCount, curChildCount + 1);
+  } catch (ex) {
+    do_throw("Passing a sparse array should not throw");
+  }
+  try {
+    wait = PlacesTestUtils.waitForNotification("bookmark-removed", events =>
+      events.some(event => event.id == tagRoot.itemId)
+    );
+    tagssvc.untagURI(uri1, [undefined, "tagSparse"]);
+    wait.then(() => {
+      Assert.equal(tagRoot.childCount, curChildCount);
+    });
+  } catch (ex) {
+    do_throw("Passing a sparse array should not throw");
+  }
+
+  // Test that the API throws for bad arguments.
+  try {
+    tagssvc.tagURI(uri1, ["", "test"]);
+    do_throw("Passing a bad tags array should throw");
+  } catch (ex) {
+    Assert.equal(ex.name, "NS_ERROR_ILLEGAL_VALUE");
+  }
+  try {
+    tagssvc.untagURI(uri1, ["", "test"]);
+    do_throw("Passing a bad tags array should throw");
+  } catch (ex) {
+    Assert.equal(ex.name, "NS_ERROR_ILLEGAL_VALUE");
+  }
+  try {
+    tagssvc.tagURI(uri1, [0, "test"]);
+    do_throw("Passing a bad tags array should throw");
+  } catch (ex) {
+    Assert.equal(ex.name, "NS_ERROR_ILLEGAL_VALUE");
+  }
+  try {
+    tagssvc.tagURI(uri1, [0, "test"]);
+    do_throw("Passing a bad tags array should throw");
+  } catch (ex) {
+    Assert.equal(ex.name, "NS_ERROR_ILLEGAL_VALUE");
+  }
+
+  // Tag name length should be limited to PlacesUtils.bookmarks.MAX_TAG_LENGTH (bug407821)
+  try {
+    // generate a long tag name. i.e. looooo...oong_tag
+    var n = PlacesUtils.bookmarks.MAX_TAG_LENGTH;
+    var someOos = new Array(n).join("o");
+    var longTagName = "l" + someOos + "ng_tag";
+
+    tagssvc.tagURI(uri1, ["short_tag", longTagName]);
+    do_throw("Passing a bad tags array should throw");
+  } catch (ex) {
+    Assert.equal(ex.name, "NS_ERROR_ILLEGAL_VALUE");
+  }
+
+  // cleanup
+  tagRoot.containerOpen = false;
+
+  // Tagging service should trim tags (Bug967196)
+  let exampleURI = uri("http://www.example.com/");
+  PlacesUtils.tagging.tagURI(exampleURI, [" test "]);
+
+  let exampleTags = PlacesUtils.tagging.getTagsForURI(exampleURI);
+  Assert.equal(exampleTags.length, 1);
+  Assert.equal(exampleTags[0], "test");
+
+  PlacesUtils.tagging.untagURI(exampleURI, ["test"]);
+  exampleTags = PlacesUtils.tagging.getTagsForURI(exampleURI);
+  Assert.equal(exampleTags.length, 0);
+}
diff --git a/toolkit/components/places/tests/unit/test_telemetry.js b/toolkit/components/places/tests/unit/test_telemetry.js
new file mode 100644
index 0000000000..b48bb76a7f
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_telemetry.js
@@ -0,0 +1,151 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Tests common Places telemetry probes by faking the telemetry service.
+
+// Enable the collection (during test) for all products so even products
+// that don't collect the data will be able to run the test without failure.
+Services.prefs.setBoolPref(
+  "toolkit.telemetry.testing.overrideProductsCheck",
+  true
+);
+
+const { PlacesDBUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/PlacesDBUtils.sys.mjs"
+);
+
+const histograms = {
+  PLACES_PAGES_COUNT: val => Assert.equal(val, 1),
+  PLACES_BOOKMARKS_COUNT: val => Assert.equal(val, 1),
+  PLACES_TAGS_COUNT: val => Assert.equal(val, 1),
+  PLACES_KEYWORDS_COUNT: val => Assert.equal(val, 1),
+  PLACES_SORTED_BOOKMARKS_PERC: val => Assert.equal(val, 100),
+  PLACES_TAGGED_BOOKMARKS_PERC: val => Assert.equal(val, 100),
+  PLACES_DATABASE_FILESIZE_MB: val => Assert.ok(val > 0),
+  PLACES_DATABASE_FAVICONS_FILESIZE_MB: val => Assert.ok(val > 0),
+  PLACES_EXPIRATION_STEPS_TO_CLEAN2: val => Assert.ok(val > 1),
+  PLACES_IDLE_MAINTENANCE_TIME_MS: val => Assert.ok(val > 0),
+  PLACES_ANNOS_PAGES_COUNT: val => Assert.equal(val, 1),
+  PLACES_MAINTENANCE_DAYSFROMLAST: val => Assert.ok(val >= 0),
+};
+
+const scalars = {
+  pages_need_frecency_recalculation: 1, // 1 bookmark is added causing recalc.
+};
+
+/**
+ * Forces an expiration run.
+ *
+ * @param [optional] aLimit
+ *        Limit for the expiration.  Pass -1 for unlimited.
+ *        Any other non-positive value will just expire orphans.
+ *
+ * @return {Promise}
+ * @resolves When expiration finishes.
+ * @rejects Never.
+ */
+function promiseForceExpirationStep(aLimit) {
+  let promise = promiseTopicObserved(PlacesUtils.TOPIC_EXPIRATION_FINISHED);
+  let expire = Cc["@mozilla.org/places/expiration;1"].getService(
+    Ci.nsIObserver
+  );
+  expire.observe(null, "places-debug-start-expiration", aLimit);
+  return promise;
+}
+
+/**
+ * Returns a PRTime in the past usable to add expirable visits.
+ *
+ * param [optional] daysAgo
+ *       Expiration ignores any visit added in the last 7 days, so by default
+ *       this will be set to 7.
+ * @note to be safe against DST issues we go back one day more.
+ */
+function getExpirablePRTime(daysAgo = 7) {
+  let dateObj = new Date();
+  // Normalize to midnight
+  dateObj.setHours(0);
+  dateObj.setMinutes(0);
+  dateObj.setSeconds(0);
+  dateObj.setMilliseconds(0);
+  dateObj = new Date(dateObj.getTime() - (daysAgo + 1) * 86400000);
+  return dateObj.getTime() * 1000;
+}
+
+add_task(async function test_execute() {
+  // Put some trash in the database.
+  let uri = Services.io.newURI("http://moz.org/");
+
+  PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.unfiledGuid,
+    children: [
+      {
+        title: "moz test",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            title: "moz test",
+            url: uri,
+          },
+        ],
+      },
+    ],
+  });
+
+  PlacesUtils.tagging.tagURI(uri, ["tag"]);
+  await PlacesUtils.keywords.insert({ url: uri.spec, keyword: "keyword" });
+
+  // Set a large annotation.
+  let content = "";
+  while (content.length < 1024) {
+    content += "0";
+  }
+
+  await PlacesUtils.history.update({
+    url: uri,
+    annotations: new Map([["test-anno", content]]),
+  });
+
+  await PlacesDBUtils.telemetry();
+
+  await PlacesTestUtils.promiseAsyncUpdates();
+
+  // Test expiration probes.
+  let timeInMicroseconds = getExpirablePRTime(8);
+
+  function newTimeInMicroseconds() {
+    timeInMicroseconds = timeInMicroseconds + 1000;
+    return timeInMicroseconds;
+  }
+
+  for (let i = 0; i < 3; i++) {
+    await PlacesTestUtils.addVisits({
+      uri: NetUtil.newURI("http://" + i + ".moz.org/"),
+      visitDate: newTimeInMicroseconds(),
+    });
+  }
+  Services.prefs.setIntPref("places.history.expiration.max_pages", 0);
+  await promiseForceExpirationStep(2);
+  await promiseForceExpirationStep(2);
+
+  // Test idle probes.
+  await PlacesDBUtils.maintenanceOnIdle();
+
+  for (let histogramId in histograms) {
+    info("checking histogram " + histogramId);
+    let validate = histograms[histogramId];
+    let snapshot = Services.telemetry.getHistogramById(histogramId).snapshot();
+    validate(snapshot.sum);
+    Assert.ok(Object.values(snapshot.values).reduce((a, b) => a + b, 0) > 0);
+  }
+  for (let scalarName in scalars) {
+    let scalar = "places." + scalarName;
+    info("checking scalar " + scalar);
+    TelemetryTestUtils.assertScalar(
+      TelemetryTestUtils.getProcessScalars("parent"),
+      scalar,
+      scalars[scalarName],
+      "Verify scalar value matches"
+    );
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js b/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js
new file mode 100644
index 0000000000..fa03c0e06b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_update_frecency_after_delete.js
@@ -0,0 +1,211 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Bug 455315
+ * https://bugzilla.mozilla.org/show_bug.cgi?id=412132
+ *
+ * Ensures that the frecency of a bookmark's URI is what it should be after the
+ * bookmark is deleted.
+ */
+
+add_task(async function removed_bookmark() {
+  info(
+    "After removing bookmark, frecency of bookmark's URI should be " +
+      "zero if URI is unvisited and no longer bookmarked."
+  );
+  const TEST_URI = Services.io.newURI("http://example.com/1");
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URI,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Bookmarked => frecency of URI should be != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.remove(bm);
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Unvisited URI no longer bookmarked => frecency should = 0");
+  Assert.equal(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function removed_but_visited_bookmark() {
+  info(
+    "After removing bookmark, frecency of bookmark's URI should " +
+      "not be zero if URI is visited."
+  );
+  const TEST_URI = Services.io.newURI("http://example.com/1");
+  let bm = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URI,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Bookmarked => frecency of URI should be != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesTestUtils.addVisits(TEST_URI);
+  await PlacesUtils.bookmarks.remove(bm);
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("*Visited* URI no longer bookmarked => frecency should != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function remove_bookmark_still_bookmarked() {
+  info(
+    "After removing bookmark, frecency of bookmark's URI should " +
+      "not be zero if URI is still bookmarked."
+  );
+  const TEST_URI = Services.io.newURI("http://example.com/1");
+  let bm1 = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark 1 title",
+    url: TEST_URI,
+  });
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark 2 title",
+    url: TEST_URI,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Bookmarked => frecency of URI should be != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.remove(bm1);
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("URI still bookmarked => frecency should != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function cleared_parent_of_visited_bookmark() {
+  info(
+    "After removing all children from bookmark's parent, frecency " +
+      "of bookmark's URI should not be zero if URI is visited."
+  );
+  const TEST_URI = Services.io.newURI("http://example.com/1");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    title: "bookmark title",
+    url: TEST_URI,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Bookmarked => frecency of URI should be != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesTestUtils.addVisits(TEST_URI);
+  await PlacesUtils.bookmarks.eraseEverything();
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("*Visited* URI no longer bookmarked => frecency should != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
+
+add_task(async function cleared_parent_of_bookmark_still_bookmarked() {
+  info(
+    "After removing all children from bookmark's parent, frecency " +
+      "of bookmark's URI should not be zero if URI is still " +
+      "bookmarked."
+  );
+  const TEST_URI = Services.io.newURI("http://example.com/1");
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "bookmark 1 title",
+    url: TEST_URI,
+  });
+
+  let folder = await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.unfiledGuid,
+    type: PlacesUtils.bookmarks.TYPE_FOLDER,
+    title: "bookmark 2 folder",
+  });
+  await PlacesUtils.bookmarks.insert({
+    title: "bookmark 2 title",
+    parentGuid: folder.guid,
+    url: TEST_URI,
+  });
+
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  info("Bookmarked => frecency of URI should be != 0");
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.remove(folder);
+  await PlacesFrecencyRecalculator.recalculateAnyOutdatedFrecencies();
+  // URI still bookmarked => frecency should != 0.
+  Assert.notEqual(
+    await PlacesTestUtils.getDatabaseValue("moz_places", "frecency", {
+      url: TEST_URI,
+    }),
+    0
+  );
+
+  await PlacesUtils.bookmarks.eraseEverything();
+  await PlacesUtils.history.clear();
+});
diff --git a/toolkit/components/places/tests/unit/test_utils_backups_create.js b/toolkit/components/places/tests/unit/test_utils_backups_create.js
new file mode 100644
index 0000000000..a0bc506695
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_utils_backups_create.js
@@ -0,0 +1,152 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Check for correct functionality of bookmarks backups
+ */
+
+const NUMBER_OF_BACKUPS = 10;
+
+async function createBackups(nBackups, dateObj, bookmarksBackupDir) {
+  // Generate random dates.
+  let dates = [];
+  while (dates.length < nBackups) {
+    // Use last year to ensure today's backup is the newest.
+    let randomDate = new Date(
+      dateObj.getFullYear() - 1,
+      Math.floor(12 * Math.random()),
+      Math.floor(28 * Math.random())
+    );
+    if (!dates.includes(randomDate.getTime())) {
+      dates.push(randomDate.getTime());
+    }
+  }
+  // Sort dates from oldest to newest.
+  dates.sort();
+
+  // Fake backups are created backwards to ensure we won't consider file
+  // creation time.
+  // Create fake backups for the newest dates.
+  for (let i = dates.length - 1; i >= 0; i--) {
+    let backupFilename = PlacesBackups.getFilenameForDate(new Date(dates[i]));
+    let backupFile = bookmarksBackupDir.clone();
+    backupFile.append(backupFilename);
+    backupFile.create(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0666", 8));
+    info("Creating fake backup " + backupFile.leafName);
+    if (!backupFile.exists()) {
+      do_throw("Unable to create fake backup " + backupFile.leafName);
+    }
+  }
+
+  return dates;
+}
+
+async function checkBackups(dates, bookmarksBackupDir) {
+  // Check backups.  We have 11 dates but we the max number is 10 so the
+  // oldest backup should have been removed.
+  for (let i = 0; i < dates.length; i++) {
+    let backupFilename;
+    let shouldExist;
+    let backupFile;
+    if (i > 0) {
+      let files = bookmarksBackupDir.directoryEntries;
+      while (files.hasMoreElements()) {
+        let entry = files.nextFile;
+        if (PlacesBackups.filenamesRegex.test(entry.leafName)) {
+          backupFilename = entry.leafName;
+          backupFile = entry;
+          break;
+        }
+      }
+      shouldExist = true;
+    } else {
+      backupFilename = PlacesBackups.getFilenameForDate(new Date(dates[i]));
+      backupFile = bookmarksBackupDir.clone();
+      backupFile.append(backupFilename);
+      shouldExist = false;
+    }
+    if (backupFile.exists() != shouldExist) {
+      do_throw(
+        "Backup should " +
+          (shouldExist ? "" : "not") +
+          " exist: " +
+          backupFilename
+      );
+    }
+  }
+}
+
+async function cleanupFiles(bookmarksBackupDir) {
+  // Cleanup backups folder.
+  // XXX: Can't use bookmarksBackupDir.remove(true) because file lock happens
+  // on WIN XP.
+  let files = bookmarksBackupDir.directoryEntries;
+  while (files.hasMoreElements()) {
+    let entry = files.nextFile;
+    entry.remove(false);
+  }
+  // Clear cache to match the manual removing of files
+  delete PlacesBackups._backupFiles;
+  Assert.ok(!bookmarksBackupDir.directoryEntries.hasMoreElements());
+}
+
+add_task(async function test_create_backups() {
+  let backupFolderPath = await PlacesBackups.getBackupFolder();
+  let bookmarksBackupDir = new FileUtils.File(backupFolderPath);
+
+  let dateObj = new Date();
+  let dates = await createBackups(
+    NUMBER_OF_BACKUPS,
+    dateObj,
+    bookmarksBackupDir
+  );
+  // Add today's backup.
+  await PlacesBackups.create(NUMBER_OF_BACKUPS);
+  dates.push(dateObj.getTime());
+  await checkBackups(dates, bookmarksBackupDir);
+  await cleanupFiles(bookmarksBackupDir);
+});
+
+add_task(async function test_saveBookmarks_with_no_backups() {
+  let backupFolderPath = await PlacesBackups.getBackupFolder();
+  let bookmarksBackupDir = new FileUtils.File(backupFolderPath);
+
+  Services.prefs.setIntPref("browser.bookmarks.max_backups", 0);
+
+  let filePath = PathUtils.join(do_get_tempdir().path, "backup.json");
+  await PlacesBackups.saveBookmarksToJSONFile(filePath);
+  let files = bookmarksBackupDir.directoryEntries;
+  Assert.ok(!files.hasMoreElements(), "Should have no backup files.");
+  await IOUtils.remove(filePath);
+  // We don't need to call cleanupFiles as we are not creating any
+  // backups but need to reset the cache.
+  delete PlacesBackups._backupFiles;
+});
+
+add_task(async function test_saveBookmarks_with_backups() {
+  let backupFolderPath = await PlacesBackups.getBackupFolder();
+  let bookmarksBackupDir = new FileUtils.File(backupFolderPath);
+
+  Services.prefs.setIntPref("browser.bookmarks.max_backups", NUMBER_OF_BACKUPS);
+
+  let filePath = PathUtils.join(do_get_tempdir().path, "backup.json");
+  let dateObj = new Date();
+  let dates = await createBackups(
+    NUMBER_OF_BACKUPS,
+    dateObj,
+    bookmarksBackupDir
+  );
+
+  await PlacesBackups.saveBookmarksToJSONFile(filePath);
+
+  let backupPath = await PlacesBackups.getMostRecentBackup();
+  Assert.ok(await IOUtils.read(backupPath, { decompress: true }));
+
+  dates.push(dateObj.getTime());
+  await checkBackups(dates, bookmarksBackupDir);
+  await IOUtils.remove(filePath);
+  await cleanupFiles(bookmarksBackupDir);
+});
diff --git a/toolkit/components/places/tests/unit/test_utils_backups_hasRecent.js b/toolkit/components/places/tests/unit/test_utils_backups_hasRecent.js
new file mode 100644
index 0000000000..77d356b032
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_utils_backups_hasRecent.js
@@ -0,0 +1,43 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Check for correct functionality of bookmarks backups
+ */
+
+/**
+ * Creates a fake empty dated backup.
+ * @param {Date} date the Date to use for the backup file name
+ * @param {string} backupsFolderPath the path to the backups folder
+ * @returns path of the created backup file
+ */
+async function createFakeBackup(date, backupsFolderPath) {
+  let backupFilePath = PathUtils.join(
+    backupsFolderPath,
+    PlacesBackups.getFilenameForDate(date)
+  );
+  await IOUtils.write(backupFilePath, new Uint8Array());
+  return backupFilePath;
+}
+
+add_task(async function test_hasRecentBackup() {
+  let backupFolderPath = await PlacesBackups.getBackupFolder();
+  Assert.ok(!(await PlacesBackups.hasRecentBackup()), "Check no recent backup");
+
+  await createFakeBackup(new Date(Date.now() - 4 * 86400), backupFolderPath);
+  Assert.ok(!(await PlacesBackups.hasRecentBackup()), "Check no recent backup");
+  PlacesBackups.invalidateCache();
+  await createFakeBackup(new Date(Date.now() - 2 * 86400), backupFolderPath);
+  Assert.ok(await PlacesBackups.hasRecentBackup(), "Check has recent backup");
+  PlacesBackups.invalidateCache();
+
+  try {
+    await IOUtils.remove(backupFolderPath, { recursive: true });
+  } catch (ex) {
+    // On Windows the files may be locked.
+    info("Unable to cleanup the backups test folder");
+  }
+});
diff --git a/toolkit/components/places/tests/unit/test_utils_getURLsForContainerNode.js b/toolkit/components/places/tests/unit/test_utils_getURLsForContainerNode.js
new file mode 100644
index 0000000000..2d092ef591
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_utils_getURLsForContainerNode.js
@@ -0,0 +1,266 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Check for correct functionality of PlacesUtils.getURLsForContainerNode and
+ * PlacesUtils.hasChildURIs (those helpers share almost all of their code)
+ */
+
+var PU = PlacesUtils;
+var hs = PU.history;
+
+add_task(async function test_getURLsForContainerNode_folder() {
+  info("*** TEST: folder");
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        // This is the folder we will check for children.
+        title: "folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        // Create a folder and a query node inside it, these should not be considered
+        // uri nodes.
+        children: [
+          {
+            title: "inside folder",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            children: [
+              {
+                url: "place:sort=1",
+                title: "inside query",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+
+  var query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var options = hs.getNewQueryOptions();
+
+  info("Check folder without uri nodes");
+  check_uri_nodes(query, options, 0);
+
+  info("Check folder with uri nodes");
+  // Add an uri node, this should be considered.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: bookmarks[0].guid,
+    url: "http://www.mozilla.org/",
+    title: "bookmark",
+  });
+  check_uri_nodes(query, options, 1);
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_getURLsForContainerNode_folder_excludeItems() {
+  info("*** TEST: folder in an excludeItems root");
+  let bookmarks = await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.toolbarGuid,
+    children: [
+      {
+        // This is the folder we will check for children.
+        title: "folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        // Create a folder and a query node inside it, these should not be considered
+        // uri nodes.
+        children: [
+          {
+            title: "inside folder",
+            type: PlacesUtils.bookmarks.TYPE_FOLDER,
+            children: [
+              {
+                url: "place:sort=1",
+                title: "inside query",
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  });
+
+  var query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var options = hs.getNewQueryOptions();
+  options.excludeItems = true;
+
+  info("Check folder without uri nodes");
+  check_uri_nodes(query, options, 0);
+
+  info("Check folder with uri nodes");
+  // Add an uri node, this should be considered.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: bookmarks[0].guid,
+    url: "http://www.mozilla.org/",
+    title: "bookmark",
+  });
+  check_uri_nodes(query, options, 1);
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_getURLsForContainerNode_query() {
+  info("*** TEST: query");
+  // This is the query we will check for children.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "inside query",
+    url: `place:parent=${PlacesUtils.bookmarks.menuGuid}&sort=1`,
+  });
+
+  // Create a folder and a query node inside it, these should not be considered
+  // uri nodes.
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "inside folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            url: "place:sort=1",
+            title: "inside query",
+          },
+        ],
+      },
+    ],
+  });
+
+  var query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var options = hs.getNewQueryOptions();
+
+  info("Check query without uri nodes");
+  check_uri_nodes(query, options, 0);
+
+  info("Check query with uri nodes");
+  // Add an uri node, this should be considered.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://www.mozilla.org/",
+    title: "bookmark",
+  });
+  check_uri_nodes(query, options, 1);
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_getURLsForContainerNode_query_excludeItems() {
+  info("*** TEST: excludeItems Query");
+  // This is the query we will check for children.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "inside query",
+    url: `place:parent=${PlacesUtils.bookmarks.menuGuid}&sort=1`,
+  });
+
+  // Create a folder and a query node inside it, these should not be considered
+  // uri nodes.
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "inside folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            url: "place:sort=1",
+            title: "inside query",
+          },
+        ],
+      },
+    ],
+  });
+
+  var query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var options = hs.getNewQueryOptions();
+  options.excludeItems = true;
+
+  info("Check folder without uri nodes");
+  check_uri_nodes(query, options, 0);
+
+  info("Check folder with uri nodes");
+  // Add an uri node, this should be considered.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://www.mozilla.org/",
+    title: "bookmark",
+  });
+  check_uri_nodes(query, options, 1);
+
+  await PlacesUtils.bookmarks.eraseEverything();
+});
+
+add_task(async function test_getURLsForContainerNode_query_excludeQueries() {
+  info("*** TEST: !expandQueries Query");
+  // This is the query we will check for children.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+    title: "inside query",
+    url: `place:parent=${PlacesUtils.bookmarks.menuGuid}&sort=1`,
+  });
+
+  // Create a folder and a query node inside it, these should not be considered
+  // uri nodes.
+  await PlacesUtils.bookmarks.insertTree({
+    guid: PlacesUtils.bookmarks.menuGuid,
+    children: [
+      {
+        title: "inside folder",
+        type: PlacesUtils.bookmarks.TYPE_FOLDER,
+        children: [
+          {
+            url: "place:sort=1",
+            title: "inside query",
+          },
+        ],
+      },
+    ],
+  });
+
+  var query = hs.getNewQuery();
+  query.setParents([PlacesUtils.bookmarks.toolbarGuid]);
+  var options = hs.getNewQueryOptions();
+  options.expandQueries = false;
+
+  info("Check folder without uri nodes");
+  check_uri_nodes(query, options, 0);
+
+  info("Check folder with uri nodes");
+  // Add an uri node, this should be considered.
+  await PlacesUtils.bookmarks.insert({
+    parentGuid: PlacesUtils.bookmarks.menuGuid,
+    url: "http://www.mozilla.org/",
+    title: "bookmark",
+  });
+  check_uri_nodes(query, options, 1);
+});
+
+/**
+ * Executes a query and checks number of uri nodes in the first container in
+ * query's results.  To correctly test a container ensure that the query will
+ * return only your container in the first level.
+ *
+ * @param  aQuery
+ *         nsINavHistoryQuery object defining the query
+ * @param  aOptions
+ *         nsINavHistoryQueryOptions object defining the query's options
+ * @param  aExpectedURINodes
+ *         number of expected uri nodes
+ */
+function check_uri_nodes(aQuery, aOptions, aExpectedURINodes) {
+  var result = hs.executeQuery(aQuery, aOptions);
+  var root = result.root;
+  root.containerOpen = true;
+  var node = root.getChild(0);
+  Assert.equal(PU.hasChildURIs(node), aExpectedURINodes > 0);
+  Assert.equal(PU.getURLsForContainerNode(node).length, aExpectedURINodes);
+  root.containerOpen = false;
+}
diff --git a/toolkit/components/places/tests/unit/test_utils_timeConversion.js b/toolkit/components/places/tests/unit/test_utils_timeConversion.js
new file mode 100644
index 0000000000..c71effe709
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_utils_timeConversion.js
@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Check time conversion utils.
+ */
+
+add_task(async function toDate() {
+  Assert.throws(() => PlacesUtils.toDate(), /Invalid value/, "Should throw");
+  Assert.throws(() => PlacesUtils.toDate(NaN), /Invalid value/, "Should throw");
+  Assert.throws(
+    () => PlacesUtils.toDate(null),
+    /Invalid value/,
+    "Should throw"
+  );
+  Assert.throws(() => PlacesUtils.toDate("1"), /Invalid value/, "Should throw");
+
+  const now = Date.now();
+  const usecs = now * 1000;
+  Assert.deepEqual(PlacesUtils.toDate(usecs), new Date(now));
+});
+
+add_task(async function toPRTime() {
+  Assert.throws(() => PlacesUtils.toPRTime(), /TypeError/, "Should throw");
+  Assert.throws(() => PlacesUtils.toPRTime(null), /TypeError/, "Should throw");
+  Assert.throws(
+    () => PlacesUtils.toPRTime({}),
+    /Invalid value/,
+    "Should throw"
+  );
+  Assert.throws(
+    () => PlacesUtils.toPRTime(NaN),
+    /Invalid value/,
+    "Should throw"
+  );
+  Assert.throws(
+    () => PlacesUtils.toPRTime(new Date(NaN)),
+    /Invalid value/,
+    "Should throw"
+  );
+  Assert.throws(
+    () => PlacesUtils.toPRTime(new URL("https://test.moz")),
+    /Invalid value/,
+    "Should throw"
+  );
+
+  const now = Date.now();
+  const usecs = now * 1000;
+  Assert.strictEqual(PlacesUtils.toPRTime(now), usecs);
+  Assert.strictEqual(PlacesUtils.toPRTime(new Date(now)), usecs);
+});
diff --git a/toolkit/components/places/tests/unit/test_visitsInDB.js b/toolkit/components/places/tests/unit/test_visitsInDB.js
new file mode 100644
index 0000000000..cbd0a04c4b
--- /dev/null
+++ b/toolkit/components/places/tests/unit/test_visitsInDB.js
@@ -0,0 +1,12 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+
+add_task(async function test_execute() {
+  const TEST_URI = uri("http://mozilla.com");
+
+  Assert.equal(0, await PlacesTestUtils.visitsInDB(TEST_URI));
+  await PlacesTestUtils.addVisits({ uri: TEST_URI });
+  Assert.equal(1, await PlacesTestUtils.visitsInDB(TEST_URI));
+  await PlacesTestUtils.addVisits({ uri: TEST_URI });
+  Assert.equal(2, await PlacesTestUtils.visitsInDB(TEST_URI));
+});
diff --git a/toolkit/components/places/tests/unit/xpcshell.ini b/toolkit/components/places/tests/unit/xpcshell.ini
new file mode 100644
index 0000000000..99f8d04754
--- /dev/null
+++ b/toolkit/components/places/tests/unit/xpcshell.ini
@@ -0,0 +1,120 @@
+[DEFAULT]
+head = head_bookmarks.js
+firefox-appdir = browser
+prefs = places.loglevel="All"
+support-files =
+  bookmarks.corrupt.html
+  bookmarks.json
+  bookmarks_corrupt.json
+  bookmarks.preplaces.html
+  bookmarks_html_localized.html
+  bookmarks_html_singleframe.html
+  bookmarks_iconuri.json
+  mobile_bookmarks_folder_import.json
+  mobile_bookmarks_folder_merge.json
+  mobile_bookmarks_multiple_folders.json
+  mobile_bookmarks_root_import.json
+  mobile_bookmarks_root_merge.json
+  places.sparse.sqlite
+
+[test_331487.js]
+[test_384370.js]
+[test_385397.js]
+[test_399266.js]
+skip-if = os == "linux" # Bug 821781
+[test_402799.js]
+[test_412132.js]
+[test_415460.js]
+[test_415757.js]
+[test_419792_node_tags_property.js]
+[test_425563.js]
+[test_429505_remove_shortcuts.js]
+[test_433317_query_title_update.js]
+[test_433525_hasChildren_crash.js]
+[test_454977.js]
+[test_463863.js]
+[test_485442_crash_bug_nsNavHistoryQuery_GetUri.js]
+[test_486978_sort_by_date_queries.js]
+[test_536081.js]
+[test_1085291.js]
+[test_1105208.js]
+[test_1105866.js]
+[test_1606731.js]
+[test_PlacesQuery_history.js]
+[test_asyncExecuteLegacyQueries.js]
+[test_async_transactions.js]
+[test_autocomplete_match_fallbackTitle.js]
+[test_bookmark-tags-changed_frequency.js]
+[test_bookmarks_json.js]
+[test_bookmarks_json_corrupt.js]
+[test_bookmarks_html.js]
+[test_bookmarks_html_corrupt.js]
+[test_bookmarks_html_escape_entities.js]
+[test_bookmarks_html_import_tags.js]
+[test_bookmarks_html_localized.js]
+[test_bookmarks_html_singleframe.js]
+[test_bookmarks_restore_notification.js]
+[test_broken_folderShortcut_result.js]
+[test_browserhistory.js]
+[test_childlessTags.js]
+[test_frecency_decay.js]
+[test_frecency_origins_alternative.js]
+[test_frecency_origins_recalc.js]
+[test_frecency_pages_alternative.js]
+prefs = places.frecency.pages.alternative.featureGate=true
+[test_frecency_pages_recalc_alt.js]
+prefs = places.frecency.pages.alternative.featureGate=true
+[test_frecency_recalc_triggers.js]
+[test_frecency_recalculator.js]
+[test_frecency_unvisited_bookmark.js]
+[test_frecency_zero_updated.js]
+[test_getChildIndex.js]
+[test_hash.js]
+[test_history.js]
+[test_history_clear.js]
+[test_history_notifications.js]
+[test_history_observer.js]
+[test_history_sidebar.js]
+[test_import_mobile_bookmarks.js]
+[test_isPageInDB.js]
+[test_isURIVisited.js]
+[test_isvisited.js]
+[test_keywords.js]
+[test_lastModified.js]
+[test_markpageas.js]
+[test_metadata.js]
+[test_missing_builtin_folders.js]
+support-files = missingBuiltIn.sqlite
+[test_missing_root_folder.js]
+support-files = noRoot.sqlite
+[test_multi_observation.js]
+[test_multi_word_tags.js]
+[test_nested_notifications.js]
+[test_nsINavHistoryViewer.js]
+[test_null_interfaces.js]
+[test_origins.js]
+[test_origins_parsing.js]
+[test_pageGuid_bookmarkGuid.js]
+[test_frecency_observers.js]
+[test_PlacesDBUtils_removeOldCorruptDBs.js]
+[test_placeURIs.js]
+[test_PlacesUtils_invalidateCachedGuidFor.js]
+[test_PlacesUtils_invalidateCachedGuids.js]
+[test_PlacesUtils_isRootItem.js]
+[test_PlacesUtils_unwrapNodes_place.js]
+[test_promiseBookmarksTree.js]
+[test_resolveNullBookmarkTitles.js]
+[test_result_sort.js]
+[test_resultsAsVisit_details.js]
+[test_sql_function_origin.js]
+[test_sql_guid_functions.js]
+[test_tag_autocomplete_search.js]
+[test_tagging.js]
+[test_telemetry.js]
+[test_update_frecency_after_delete.js]
+[test_utils_backups_create.js]
+[test_utils_backups_hasRecent.js]
+[test_utils_getURLsForContainerNode.js]
+[test_utils_timeConversion.js]
+[test_visitsInDB.js]
+[test_get_query_param_sql_function.js]
-- 
cgit v1.2.3