summaryrefslogtreecommitdiffstats
path: root/toolkit/components/places/BookmarkJSONUtils.sys.mjs
diff options
context:
space:
mode:
Diffstat (limited to 'toolkit/components/places/BookmarkJSONUtils.sys.mjs')
-rw-r--r--toolkit/components/places/BookmarkJSONUtils.sys.mjs581
1 files changed, 581 insertions, 0 deletions
diff --git a/toolkit/components/places/BookmarkJSONUtils.sys.mjs b/toolkit/components/places/BookmarkJSONUtils.sys.mjs
new file mode 100644
index 0000000000..29967b5395
--- /dev/null
+++ b/toolkit/components/places/BookmarkJSONUtils.sys.mjs
@@ -0,0 +1,581 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { PlacesUtils } from "resource://gre/modules/PlacesUtils.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs",
+});
+
+// This is used to translate old folder pseudonyms in queries with their newer
+// guids.
+const OLD_BOOKMARK_QUERY_TRANSLATIONS = {
+ PLACES_ROOT: PlacesUtils.bookmarks.rootGuid,
+ BOOKMARKS_MENU: PlacesUtils.bookmarks.menuGuid,
+ TAGS: PlacesUtils.bookmarks.tagsGuid,
+ UNFILED_BOOKMARKS: PlacesUtils.bookmarks.unfiledGuid,
+ TOOLBAR: PlacesUtils.bookmarks.toolbarGuid,
+ MOBILE_BOOKMARKS: PlacesUtils.bookmarks.mobileGuid,
+};
+
+/**
+ * Generates an hash for the given string.
+ *
+ * @note The generated hash is returned in base64 form. Mind the fact base64
+ * is case-sensitive if you are going to reuse this code.
+ */
+function generateHash(aString) {
+ let cryptoHash = Cc["@mozilla.org/security/hash;1"].createInstance(
+ Ci.nsICryptoHash
+ );
+ cryptoHash.init(Ci.nsICryptoHash.MD5);
+ let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(
+ Ci.nsIStringInputStream
+ );
+ stringStream.setUTF8Data(aString);
+ cryptoHash.updateFromStream(stringStream, -1);
+ // base64 allows the '/' char, but we can't use it for filenames.
+ return cryptoHash.finish(true).replace(/\//g, "-");
+}
+
+export var BookmarkJSONUtils = Object.freeze({
+ /**
+ * Import bookmarks from a url.
+ *
+ * @param {string} aSpec
+ * url of the bookmark data.
+ * @param {boolean} [options.replace]
+ * Whether we should erase existing bookmarks before importing.
+ * @param {PlacesUtils.bookmarks.SOURCES} [options.source]
+ * The bookmark change source, used to determine the sync status for
+ * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or
+ * `IMPORT` otherwise.
+ *
+ * @returns {Promise<number>} The number of imported bookmarks, not including
+ * folders and separators.
+ * @resolves When the new bookmarks have been created.
+ * @rejects JavaScript exception.
+ */
+ async importFromURL(
+ aSpec,
+ {
+ replace: aReplace = false,
+ source: aSource = aReplace
+ ? PlacesUtils.bookmarks.SOURCES.RESTORE
+ : PlacesUtils.bookmarks.SOURCES.IMPORT,
+ } = {}
+ ) {
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace);
+ let bookmarkCount = 0;
+ try {
+ let importer = new BookmarkImporter(aReplace, aSource);
+ bookmarkCount = await importer.importFromURL(aSpec);
+
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace);
+ } catch (ex) {
+ console.error(`Failed to restore bookmarks from ${aSpec}:`, ex);
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace);
+ throw ex;
+ }
+ return bookmarkCount;
+ },
+
+ /**
+ * Restores bookmarks and tags from a JSON file.
+ *
+ * @param aFilePath
+ * OS.File path string of bookmarks in JSON or JSONlz4 format to be restored.
+ * @param [options.replace]
+ * Whether we should erase existing bookmarks before importing.
+ * @param [options.source]
+ * The bookmark change source, used to determine the sync status for
+ * imported bookmarks. Defaults to `RESTORE` if `replace = true`, or
+ * `IMPORT` otherwise.
+ *
+ * @returns {Promise<number>} The number of imported bookmarks, not including
+ * folders and separators.
+ * @resolves When the new bookmarks have been created.
+ * @rejects JavaScript exception.
+ */
+ async importFromFile(
+ aFilePath,
+ {
+ replace: aReplace = false,
+ source: aSource = aReplace
+ ? PlacesUtils.bookmarks.SOURCES.RESTORE
+ : PlacesUtils.bookmarks.SOURCES.IMPORT,
+ } = {}
+ ) {
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN, aReplace);
+ let bookmarkCount = 0;
+ try {
+ if (!(await IOUtils.exists(aFilePath))) {
+ throw new Error("Cannot restore from nonexisting json file");
+ }
+
+ let importer = new BookmarkImporter(aReplace, aSource);
+ if (aFilePath.endsWith("jsonlz4")) {
+ bookmarkCount = await importer.importFromCompressedFile(aFilePath);
+ } else {
+ bookmarkCount = await importer.importFromURL(
+ PathUtils.toFileURI(aFilePath)
+ );
+ }
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS, aReplace);
+ } catch (ex) {
+ console.error(`Failed to restore bookmarks from ${aFilePath}:`, ex);
+ notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED, aReplace);
+ throw ex;
+ }
+ return bookmarkCount;
+ },
+
+ /**
+ * Serializes bookmarks using JSON, and writes to the supplied file path.
+ *
+ * @param {path} aFilePath
+ * Path string for the bookmarks file to be created.
+ * @param {object} [aOptions]
+ * @param {string} [failIfHashIs]
+ * If the generated file would have the same hash defined here, will reject
+ * with ex.becauseSameHash
+ * @param {boolean} [compress]
+ * If true, writes file using lz4 compression
+ * @return {Promise}
+ * @resolves once the file has been created, to an object with the
+ * following properties:
+ * - count: number of exported bookmarks
+ * - hash: file hash for contents comparison
+ * @rejects JavaScript exception.
+ */
+ async exportToFile(aFilePath, aOptions = {}) {
+ let [bookmarks, count] = await lazy.PlacesBackups.getBookmarksTree();
+ let startTime = Date.now();
+ let jsonString = JSON.stringify(bookmarks);
+ // Report the time taken to convert the tree to JSON.
+ try {
+ Services.telemetry
+ .getHistogramById("PLACES_BACKUPS_TOJSON_MS")
+ .add(Date.now() - startTime);
+ } catch (ex) {
+ console.error("Unable to report telemetry.");
+ }
+
+ let hash = generateHash(jsonString);
+
+ if (hash === aOptions.failIfHashIs) {
+ let e = new Error("Hash conflict");
+ e.becauseSameHash = true;
+ throw e;
+ }
+
+ // Do not write to the tmp folder, otherwise if it has a different
+ // filesystem writeAtomic will fail. Eventual dangling .tmp files should
+ // be cleaned up by the caller.
+ await IOUtils.writeUTF8(aFilePath, jsonString, {
+ compress: aOptions.compress,
+ tmpPath: PathUtils.join(aFilePath + ".tmp"),
+ });
+ return { count, hash };
+ },
+});
+
+function BookmarkImporter(aReplace, aSource) {
+ this._replace = aReplace;
+ this._source = aSource;
+}
+BookmarkImporter.prototype = {
+ /**
+ * Import bookmarks from a url.
+ *
+ * @param {string} aSpec
+ * url of the bookmark data.
+ *
+ * @returns {Promise<number>} The number of imported bookmarks, not including
+ * folders and separators.
+ * @resolves When the new bookmarks have been created.
+ * @rejects JavaScript exception.
+ */
+ async importFromURL(spec) {
+ if (!spec.startsWith("chrome://") && !spec.startsWith("file://")) {
+ throw new Error(
+ "importFromURL can only be used with chrome:// and file:// URLs"
+ );
+ }
+ let nodes = await (await fetch(spec)).json();
+
+ if (!nodes.children || !nodes.children.length) {
+ return 0;
+ }
+
+ return this.import(nodes);
+ },
+
+ /**
+ * Import bookmarks from a compressed file.
+ *
+ * @param aFilePath
+ * OS.File path string of the bookmark data.
+ *
+ * @returns {Promise<number>} The number of imported bookmarks, not including
+ * folders and separators.
+ * @resolves When the new bookmarks have been created.
+ * @rejects JavaScript exception.
+ */
+ importFromCompressedFile: async function BI_importFromCompressedFile(
+ aFilePath
+ ) {
+ // We read as UTF8 rather than JSON, as PlacesUtils.unwrapNodes expects
+ // a JSON string.
+ let result = await IOUtils.readUTF8(aFilePath, { decompress: true });
+ return this.importFromJSON(result);
+ },
+
+ /**
+ * Import bookmarks from a JSON string.
+ *
+ * @param {String} aString JSON string of serialized bookmark data.
+ * @returns {Promise<number>} The number of imported bookmarks, not including
+ * folders and separators.
+ * @resolves When the new bookmarks have been created.
+ * @rejects JavaScript exception.
+ */
+ async importFromJSON(aString) {
+ let nodes = PlacesUtils.unwrapNodes(
+ aString,
+ PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER
+ );
+
+ if (!nodes.length || !nodes[0].children || !nodes[0].children.length) {
+ return 0;
+ }
+
+ return this.import(nodes[0]);
+ },
+
+ async import(rootNode) {
+ // Change to rootNode.children as we don't import the root, and also filter
+ // out any obsolete "tagsFolder" sections.
+ let nodes = rootNode.children.filter(node => node.root !== "tagsFolder");
+
+ // If we're replacing, then erase existing bookmarks first.
+ if (this._replace) {
+ await PlacesUtils.bookmarks.eraseEverything({ source: this._source });
+ }
+
+ let folderIdToGuidMap = {};
+
+ // Now do some cleanup on the imported nodes so that the various guids
+ // match what we need for insertTree, and we also have mappings of folders
+ // so we can repair any searches after inserting the bookmarks (see bug 824502).
+ for (let node of nodes) {
+ if (!node.children || !node.children.length) {
+ continue;
+ } // Nothing to restore for this root
+
+ // Ensure we set the source correctly.
+ node.source = this._source;
+
+ // Translate the node for insertTree.
+ let folders = translateTreeTypes(node);
+
+ folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders);
+ }
+
+ let bookmarkCount = 0;
+ // Now we can add the actual nodes to the database.
+ for (let node of nodes) {
+ // Drop any nodes without children, we can't insert them.
+ if (!node.children || !node.children.length) {
+ continue;
+ }
+
+ // Drop any roots whose guid we don't recognise - we don't support anything
+ // apart from the built-in roots.
+ if (!PlacesUtils.bookmarks.userContentRoots.includes(node.guid)) {
+ continue;
+ }
+
+ fixupSearchQueries(node, folderIdToGuidMap);
+
+ let bookmarks = await PlacesUtils.bookmarks.insertTree(node, {
+ fixupOrSkipInvalidEntries: true,
+ });
+ // We want to count only bookmarks, not folders or separators
+ bookmarkCount += bookmarks.filter(
+ bookmark => bookmark.type == PlacesUtils.bookmarks.TYPE_BOOKMARK
+ ).length;
+ // Now add any favicons.
+ try {
+ insertFaviconsForTree(node);
+ } catch (ex) {
+ console.error("Failed to insert favicons:", ex);
+ }
+ }
+ return bookmarkCount;
+ },
+};
+
+function notifyObservers(topic, replace) {
+ Services.obs.notifyObservers(null, topic, replace ? "json" : "json-append");
+}
+
+/**
+ * Iterates through a node, fixing up any place: URL queries that are found. This
+ * replaces any old (pre Firefox 62) queries that contain "folder=<id>" parts with
+ * "parent=<guid>".
+ *
+ * @param {Object} aNode The node to search.
+ * @param {Array} aFolderIdMap An array mapping of old folder IDs to new folder GUIDs.
+ */
+function fixupSearchQueries(aNode, aFolderIdMap) {
+ if (aNode.url && aNode.url.startsWith("place:")) {
+ aNode.url = fixupQuery(aNode.url, aFolderIdMap);
+ }
+ if (aNode.children) {
+ for (let child of aNode.children) {
+ fixupSearchQueries(child, aFolderIdMap);
+ }
+ }
+}
+
+/**
+ * Replaces imported folder ids with their local counterparts in a place: URI.
+ *
+ * @param {String} aQueryURL
+ * A place: URI with folder ids.
+ * @param {Object} aFolderIdMap
+ * An array mapping of old folder IDs to new folder GUIDs.
+ * @return {String} the fixed up URI if all matched. If some matched, it returns
+ * the URI with only the matching folders included. If none matched
+ * it returns the input URI unchanged.
+ */
+function fixupQuery(aQueryURL, aFolderIdMap) {
+ let invalid = false;
+ let convert = function (str, existingFolderId) {
+ let guid;
+ if (
+ Object.keys(OLD_BOOKMARK_QUERY_TRANSLATIONS).includes(existingFolderId)
+ ) {
+ guid = OLD_BOOKMARK_QUERY_TRANSLATIONS[existingFolderId];
+ } else {
+ guid = aFolderIdMap[existingFolderId];
+ if (!guid) {
+ invalid = true;
+ return `invalidOldParentId=${existingFolderId}`;
+ }
+ }
+ return `parent=${guid}`;
+ };
+
+ let url = aQueryURL.replace(/folder=([A-Za-z0-9_]+)/g, convert);
+ if (invalid) {
+ // One or more of the folders don't exist, cause an empty query so that
+ // we don't try to display the whole database.
+ url += "&excludeItems=1";
+ }
+ return url;
+}
+
+/**
+ * A mapping of root folder names to Guids. To help fixupRootFolderGuid.
+ */
+const rootToFolderGuidMap = {
+ placesRoot: PlacesUtils.bookmarks.rootGuid,
+ bookmarksMenuFolder: PlacesUtils.bookmarks.menuGuid,
+ unfiledBookmarksFolder: PlacesUtils.bookmarks.unfiledGuid,
+ toolbarFolder: PlacesUtils.bookmarks.toolbarGuid,
+ mobileFolder: PlacesUtils.bookmarks.mobileGuid,
+};
+
+/**
+ * Updates a bookmark node from the json version to the places GUID. This
+ * will only change GUIDs for the built-in folders. Other folders will remain
+ * unchanged.
+ *
+ * @param {Object} A bookmark node that is updated with the new GUID if necessary.
+ */
+function fixupRootFolderGuid(node) {
+ if (!node.guid && node.root && node.root in rootToFolderGuidMap) {
+ node.guid = rootToFolderGuidMap[node.root];
+ }
+}
+
+/**
+ * Translates the JSON types for a node and its children into Places compatible
+ * types. Also handles updating of other parameters e.g. dateAdded and lastModified.
+ *
+ * @param {Object} node A node to be updated. If it contains children, they will
+ * be updated as well.
+ * @return {Array} An array containing two items:
+ * - {Object} A map of current folder ids to GUIDS
+ * - {Array} An array of GUIDs for nodes that contain query URIs
+ */
+function translateTreeTypes(node) {
+ let folderIdToGuidMap = {};
+
+ // Do the uri fixup first, so we can be consistent in this function.
+ if (node.uri) {
+ node.url = node.uri;
+ delete node.uri;
+ }
+
+ switch (node.type) {
+ case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
+ node.type = PlacesUtils.bookmarks.TYPE_FOLDER;
+
+ // Older type mobile folders have a random guid with an annotation. We need
+ // to make sure those go into the proper mobile folder.
+ let isMobileFolder =
+ node.annos &&
+ node.annos.some(anno => anno.name == PlacesUtils.MOBILE_ROOT_ANNO);
+ if (isMobileFolder) {
+ node.guid = PlacesUtils.bookmarks.mobileGuid;
+ } else {
+ // In case the Guid is broken, we need to fix it up.
+ fixupRootFolderGuid(node);
+ }
+
+ // Record the current id and the guid so that we can update any search
+ // queries later.
+ folderIdToGuidMap[node.id] = node.guid;
+ break;
+ case PlacesUtils.TYPE_X_MOZ_PLACE:
+ node.type = PlacesUtils.bookmarks.TYPE_BOOKMARK;
+ break;
+ case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
+ node.type = PlacesUtils.bookmarks.TYPE_SEPARATOR;
+ if ("title" in node) {
+ delete node.title;
+ }
+ break;
+ default:
+ // No need to throw/reject here, insertTree will remove this node automatically.
+ console.error("Unexpected bookmark type", node.type);
+ break;
+ }
+
+ if (node.dateAdded) {
+ node.dateAdded = PlacesUtils.toDate(node.dateAdded);
+ }
+
+ if (node.lastModified) {
+ let lastModified = PlacesUtils.toDate(node.lastModified);
+ // Ensure we get a last modified date that's later or equal to the dateAdded
+ // so that we don't upset the Bookmarks API.
+ if (lastModified >= node.dateAdded) {
+ node.lastModified = lastModified;
+ } else {
+ delete node.lastModified;
+ }
+ }
+
+ if (node.tags) {
+ // Separate any tags into an array, and ignore any that are too long.
+ node.tags = node.tags
+ .split(",")
+ .filter(
+ aTag =>
+ !!aTag.length && aTag.length <= PlacesUtils.bookmarks.MAX_TAG_LENGTH
+ );
+
+ // If we end up with none, then delete the property completely.
+ if (!node.tags.length) {
+ delete node.tags;
+ }
+ }
+
+ // Sometimes postData can be null, so delete it to make the validators happy.
+ if (node.postData == null) {
+ delete node.postData;
+ }
+
+ // Now handle any children.
+ if (!node.children) {
+ return folderIdToGuidMap;
+ }
+
+ // First sort the children by index.
+ node.children = node.children.sort((a, b) => {
+ return a.index - b.index;
+ });
+
+ // Now do any adjustments required for the children.
+ for (let child of node.children) {
+ let folders = translateTreeTypes(child);
+ folderIdToGuidMap = Object.assign(folderIdToGuidMap, folders);
+ }
+
+ return folderIdToGuidMap;
+}
+
+/**
+ * Handles inserting favicons into the database for a bookmark node.
+ * It is assumed the node has already been inserted into the bookmarks
+ * database.
+ *
+ * @param {Object} node The bookmark node for icons to be inserted.
+ */
+function insertFaviconForNode(node) {
+ if (node.icon) {
+ try {
+ // Create a fake faviconURI to use (FIXME: bug 523932)
+ let faviconURI = Services.io.newURI("fake-favicon-uri:" + node.url);
+ PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+ faviconURI,
+ node.icon,
+ 0,
+ Services.scriptSecurityManager.getSystemPrincipal()
+ );
+ PlacesUtils.favicons.setAndFetchFaviconForPage(
+ Services.io.newURI(node.url),
+ faviconURI,
+ false,
+ PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+ null,
+ Services.scriptSecurityManager.getSystemPrincipal()
+ );
+ } catch (ex) {
+ console.error("Failed to import favicon data:", ex);
+ }
+ }
+
+ if (!node.iconUri) {
+ return;
+ }
+
+ try {
+ PlacesUtils.favicons.setAndFetchFaviconForPage(
+ Services.io.newURI(node.url),
+ Services.io.newURI(node.iconUri),
+ false,
+ PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+ null,
+ Services.scriptSecurityManager.getSystemPrincipal()
+ );
+ } catch (ex) {
+ console.error("Failed to import favicon URI:" + ex);
+ }
+}
+
+/**
+ * Handles inserting favicons into the database for a bookmark tree - a node
+ * and its children.
+ *
+ * It is assumed the nodes have already been inserted into the bookmarks
+ * database.
+ *
+ * @param {Object} nodeTree The bookmark node tree for icons to be inserted.
+ */
+function insertFaviconsForTree(nodeTree) {
+ insertFaviconForNode(nodeTree);
+
+ if (nodeTree.children) {
+ for (let child of nodeTree.children) {
+ insertFaviconsForTree(child);
+ }
+ }
+}