diff options
Diffstat (limited to 'toolkit/modules/NewTabUtils.sys.mjs')
| -rw-r--r-- | toolkit/modules/NewTabUtils.sys.mjs | 2363 |
1 files changed, 2363 insertions, 0 deletions
diff --git a/toolkit/modules/NewTabUtils.sys.mjs b/toolkit/modules/NewTabUtils.sys.mjs new file mode 100644 index 0000000000..55d7d75ee5 --- /dev/null +++ b/toolkit/modules/NewTabUtils.sys.mjs @@ -0,0 +1,2363 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +// Android tests don't import these properly, so guard against that +let shortURL = {}; +let searchShortcuts = {}; +let didSuccessfulImport = false; +try { + shortURL = ChromeUtils.import("resource://activity-stream/lib/ShortURL.jsm"); + searchShortcuts = ChromeUtils.importESModule( + "resource://activity-stream/lib/SearchShortcuts.sys.mjs" + ); + didSuccessfulImport = true; +} catch (e) { + // The test failed to import these files +} + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BinarySearch: "resource://gre/modules/BinarySearch.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Pocket: "chrome://pocket/content/Pocket.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); + +let BrowserWindowTracker; +try { + BrowserWindowTracker = ChromeUtils.import( + "resource:///modules/BrowserWindowTracker.jsm" + ).BrowserWindowTracker; +} catch (e) { + // BrowserWindowTracker is used to determine devicePixelRatio in + // _addFavicons. We fallback to the value 2 if we can't find a window, + // so it's safe to do nothing with this here. +} + +XPCOMUtils.defineLazyGetter(lazy, "gCryptoHash", function () { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); + +// Boolean preferences that control newtab content +const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled"; + +// The maximum number of results PlacesProvider retrieves from history. +const HISTORY_RESULTS_LIMIT = 100; + +// The maximum number of links Links.getLinks will return. +const LINKS_GET_LINKS_LIMIT = 100; + +// The gather telemetry topic. +const TOPIC_GATHER_TELEMETRY = "gather-telemetry"; + +// Some default frecency threshold for Activity Stream requests +const ACTIVITY_STREAM_DEFAULT_FRECENCY = 150; + +// Some default query limit for Activity Stream requests +const ACTIVITY_STREAM_DEFAULT_LIMIT = 12; + +// Some default seconds ago for Activity Stream recent requests +const ACTIVITY_STREAM_DEFAULT_RECENT = 5 * 24 * 60 * 60; + +// The fallback value for the width of smallFavicon in pixels. +// This value will be multiplied by the current window's devicePixelRatio. +// If devicePixelRatio cannot be found, it will be multiplied by 2. +const DEFAULT_SMALL_FAVICON_WIDTH = 16; + +const POCKET_UPDATE_TIME = 24 * 60 * 60 * 1000; // 1 day +const POCKET_INACTIVE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week +const PREF_POCKET_LATEST_SINCE = "extensions.pocket.settings.latestSince"; + +/** + * Calculate the MD5 hash for a string. + * @param aValue + * The string to convert. + * @return The base64 representation of the MD5 hash. + */ +function toHash(aValue) { + let value = new TextEncoder().encode(aValue); + lazy.gCryptoHash.init(lazy.gCryptoHash.MD5); + lazy.gCryptoHash.update(value, value.length); + return lazy.gCryptoHash.finish(true); +} + +/** + * Singleton that provides storage functionality. + */ +XPCOMUtils.defineLazyGetter(lazy, "Storage", function () { + return new LinksStorage(); +}); + +function LinksStorage() { + // Handle migration of data across versions. + try { + if (this._storedVersion < this._version) { + // This is either an upgrade, or version information is missing. + if (this._storedVersion < 1) { + // Version 1 moved data from DOM Storage to prefs. Since migrating from + // version 0 is no more supported, we just reportError a dataloss later. + throw new Error("Unsupported newTab storage version"); + } + // Add further migration steps here. + } else { + // This is a downgrade. Since we cannot predict future, upgrades should + // be backwards compatible. We will set the version to the old value + // regardless, so, on next upgrade, the migration steps will run again. + // For this reason, they should also be able to run multiple times, even + // on top of an already up-to-date storage. + } + } catch (ex) { + // Something went wrong in the update process, we can't recover from here, + // so just clear the storage and start from scratch (dataloss!). + console.error( + "Unable to migrate the newTab storage to the current version. " + + "Restarting from scratch.\n", + ex + ); + this.clear(); + } + + // Set the version to the current one. + this._storedVersion = this._version; +} + +LinksStorage.prototype = { + get _version() { + return 1; + }, + + get _prefs() { + return Object.freeze({ + pinnedLinks: "browser.newtabpage.pinned", + blockedLinks: "browser.newtabpage.blocked", + }); + }, + + get _storedVersion() { + if (this.__storedVersion === undefined) { + // When the pref is not set, the storage version is unknown, so either: + // - it's a new profile + // - it's a profile where versioning information got lost + // In this case we still run through all of the valid migrations, + // starting from 1, as if it was a downgrade. As previously stated the + // migrations should already support running on an updated store. + this.__storedVersion = Services.prefs.getIntPref( + "browser.newtabpage.storageVersion", + 1 + ); + } + return this.__storedVersion; + }, + set _storedVersion(aValue) { + Services.prefs.setIntPref("browser.newtabpage.storageVersion", aValue); + this.__storedVersion = aValue; + }, + + /** + * Gets the value for a given key from the storage. + * @param aKey The storage key (a string). + * @param aDefault A default value if the key doesn't exist. + * @return The value for the given key. + */ + get: function Storage_get(aKey, aDefault) { + let value; + try { + let prefValue = Services.prefs.getStringPref(this._prefs[aKey]); + value = JSON.parse(prefValue); + } catch (e) {} + return value || aDefault; + }, + + /** + * Sets the storage value for a given key. + * @param aKey The storage key (a string). + * @param aValue The value to set. + */ + set: function Storage_set(aKey, aValue) { + // Page titles may contain unicode, thus use complex values. + Services.prefs.setStringPref(this._prefs[aKey], JSON.stringify(aValue)); + }, + + /** + * Removes the storage value for a given key. + * @param aKey The storage key (a string). + */ + remove: function Storage_remove(aKey) { + Services.prefs.clearUserPref(this._prefs[aKey]); + }, + + /** + * Clears the storage and removes all values. + */ + clear: function Storage_clear() { + for (let key in this._prefs) { + this.remove(key); + } + }, +}; + +/** + * Singleton that serves as a registry for all open 'New Tab Page's. + */ +var AllPages = { + /** + * The array containing all active pages. + */ + _pages: [], + + /** + * Cached value that tells whether the New Tab Page feature is enabled. + */ + _enabled: null, + + /** + * Adds a page to the internal list of pages. + * @param aPage The page to register. + */ + register: function AllPages_register(aPage) { + this._pages.push(aPage); + this._addObserver(); + }, + + /** + * Removes a page from the internal list of pages. + * @param aPage The page to unregister. + */ + unregister: function AllPages_unregister(aPage) { + let index = this._pages.indexOf(aPage); + if (index > -1) { + this._pages.splice(index, 1); + } + }, + + /** + * Returns whether the 'New Tab Page' is enabled. + */ + get enabled() { + if (this._enabled === null) { + this._enabled = Services.prefs.getBoolPref(PREF_NEWTAB_ENABLED); + } + + return this._enabled; + }, + + /** + * Enables or disables the 'New Tab Page' feature. + */ + set enabled(aEnabled) { + if (this.enabled != aEnabled) { + Services.prefs.setBoolPref(PREF_NEWTAB_ENABLED, !!aEnabled); + } + }, + + /** + * Returns the number of registered New Tab Pages (i.e. the number of open + * about:newtab instances). + */ + get length() { + return this._pages.length; + }, + + /** + * Updates all currently active pages but the given one. + * @param aExceptPage The page to exclude from updating. + * @param aReason The reason for updating all pages. + */ + update(aExceptPage, aReason = "") { + for (let page of this._pages.slice()) { + if (aExceptPage != page) { + page.update(aReason); + } + } + }, + + /** + * Implements the nsIObserver interface to get notified when the preference + * value changes or when a new copy of a page thumbnail is available. + */ + observe: function AllPages_observe(aSubject, aTopic, aData) { + if (aTopic == "nsPref:changed") { + // Clear the cached value. + switch (aData) { + case PREF_NEWTAB_ENABLED: + this._enabled = null; + break; + } + } + // and all notifications get forwarded to each page. + this._pages.forEach(function (aPage) { + aPage.observe(aSubject, aTopic, aData); + }, this); + }, + + /** + * Adds a preference and new thumbnail observer and turns itself into a + * no-op after the first invokation. + */ + _addObserver: function AllPages_addObserver() { + Services.prefs.addObserver(PREF_NEWTAB_ENABLED, this, true); + Services.obs.addObserver(this, "page-thumbnail:create", true); + this._addObserver = function () {}; + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), +}; + +/** + * Singleton that keeps track of all pinned links and their positions in the + * grid. + */ +var PinnedLinks = { + /** + * The cached list of pinned links. + */ + _links: null, + + /** + * The array of pinned links. + */ + get links() { + if (!this._links) { + this._links = lazy.Storage.get("pinnedLinks", []); + } + + return this._links; + }, + + /** + * Pins a link at the given position. + * @param aLink The link to pin. + * @param aIndex The grid index to pin the cell at. + * @return true if link changes, false otherwise + */ + pin: function PinnedLinks_pin(aLink, aIndex) { + // Clear the link's old position, if any. + this.unpin(aLink); + + // change pinned link into a history link + let changed = this._makeHistoryLink(aLink); + this.links[aIndex] = aLink; + this.save(); + return changed; + }, + + /** + * Unpins a given link. + * @param aLink The link to unpin. + */ + unpin: function PinnedLinks_unpin(aLink) { + let index = this._indexOfLink(aLink); + if (index == -1) { + return; + } + let links = this.links; + links[index] = null; + // trim trailing nulls + let i = links.length - 1; + while (i >= 0 && links[i] == null) { + i--; + } + links.splice(i + 1); + this.save(); + }, + + /** + * Saves the current list of pinned links. + */ + save: function PinnedLinks_save() { + lazy.Storage.set("pinnedLinks", this.links); + }, + + /** + * Checks whether a given link is pinned. + * @params aLink The link to check. + * @return whether The link is pinned. + */ + isPinned: function PinnedLinks_isPinned(aLink) { + return this._indexOfLink(aLink) != -1; + }, + + /** + * Resets the links cache. + */ + resetCache: function PinnedLinks_resetCache() { + this._links = null; + }, + + /** + * Finds the index of a given link in the list of pinned links. + * @param aLink The link to find an index for. + * @return The link's index. + */ + _indexOfLink: function PinnedLinks_indexOfLink(aLink) { + for (let i = 0; i < this.links.length; i++) { + let link = this.links[i]; + if (link && link.url == aLink.url) { + return i; + } + } + + // The given link is unpinned. + return -1; + }, + + /** + * Transforms link into a "history" link + * @param aLink The link to change + * @return true if link changes, false otherwise + */ + _makeHistoryLink: function PinnedLinks_makeHistoryLink(aLink) { + if (!aLink.type || aLink.type == "history") { + return false; + } + aLink.type = "history"; + return true; + }, + + /** + * Replaces existing link with another link. + * @param aUrl The url of existing link + * @param aLink The replacement link + */ + replace: function PinnedLinks_replace(aUrl, aLink) { + let index = this._indexOfLink({ url: aUrl }); + if (index == -1) { + return; + } + this.links[index] = aLink; + this.save(); + }, +}; + +/** + * Singleton that keeps track of all blocked links in the grid. + */ +var BlockedLinks = { + /** + * A list of objects that are observing blocked link changes. + */ + _observers: [], + + /** + * The cached list of blocked links. + */ + _links: null, + + /** + * Registers an object that will be notified when the blocked links change. + */ + addObserver(aObserver) { + this._observers.push(aObserver); + }, + + /** + * Remove the observers. + */ + removeObservers() { + this._observers = []; + }, + + /** + * The list of blocked links. + */ + get links() { + if (!this._links) { + this._links = lazy.Storage.get("blockedLinks", {}); + } + + return this._links; + }, + + /** + * Blocks a given link. Adjusts siteMap accordingly, and notifies listeners. + * @param aLink The link to block. + */ + block: function BlockedLinks_block(aLink) { + this._callObservers("onLinkBlocked", aLink); + this.links[toHash(aLink.url)] = 1; + this.save(); + + // Make sure we unpin blocked links. + PinnedLinks.unpin(aLink); + }, + + /** + * Unblocks a given link. Adjusts siteMap accordingly, and notifies listeners. + * @param aLink The link to unblock. + */ + unblock: function BlockedLinks_unblock(aLink) { + if (this.isBlocked(aLink)) { + delete this.links[toHash(aLink.url)]; + this.save(); + this._callObservers("onLinkUnblocked", aLink); + } + }, + + /** + * Saves the current list of blocked links. + */ + save: function BlockedLinks_save() { + lazy.Storage.set("blockedLinks", this.links); + }, + + /** + * Returns whether a given link is blocked. + * @param aLink The link to check. + */ + isBlocked: function BlockedLinks_isBlocked(aLink) { + return toHash(aLink.url) in this.links; + }, + + /** + * Checks whether the list of blocked links is empty. + * @return Whether the list is empty. + */ + isEmpty: function BlockedLinks_isEmpty() { + return !Object.keys(this.links).length; + }, + + /** + * Resets the links cache. + */ + resetCache: function BlockedLinks_resetCache() { + this._links = null; + }, + + _callObservers(methodName, ...args) { + for (let obs of this._observers) { + if (typeof obs[methodName] == "function") { + try { + obs[methodName](...args); + } catch (err) { + console.error(err); + } + } + } + }, +}; + +/** + * Singleton that serves as the default link provider for the grid. It queries + * the history to retrieve the most frequently visited sites. + */ +var PlacesProvider = { + /** + * Set this to change the maximum number of links the provider will provide. + */ + maxNumLinks: HISTORY_RESULTS_LIMIT, + + /** + * Must be called before the provider is used. + */ + init: function PlacesProvider_init() { + this._placesObserver = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + PlacesObservers.addListener( + ["page-visited", "page-title-changed", "pages-rank-changed"], + this._placesObserver + ); + }, + + /** + * Gets the current set of links delivered by this provider. + * @param aCallback The function that the array of links is passed to. + */ + getLinks: function PlacesProvider_getLinks(aCallback) { + let options = lazy.PlacesUtils.history.getNewQueryOptions(); + options.maxResults = this.maxNumLinks; + + // Sort by frecency, descending. + options.sortingMode = + Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_DESCENDING; + + let links = []; + + let callback = { + handleResult(aResultSet) { + let row; + + while ((row = aResultSet.getNextRow())) { + let url = row.getResultByIndex(1); + if (LinkChecker.checkLoadURI(url)) { + let title = row.getResultByIndex(2); + let frecency = row.getResultByIndex(12); + let lastVisitDate = row.getResultByIndex(5); + links.push({ + url, + title, + frecency, + lastVisitDate, + type: "history", + }); + } + } + }, + + handleError(aError) { + // Should we somehow handle this error? + aCallback([]); + }, + + handleCompletion(aReason) { + // The Places query breaks ties in frecency by place ID descending, but + // that's different from how Links.compareLinks breaks ties, because + // compareLinks doesn't have access to place IDs. It's very important + // that the initial list of links is sorted in the same order imposed by + // compareLinks, because Links uses compareLinks to perform binary + // searches on the list. So, ensure the list is so ordered. + let i = 1; + let outOfOrder = []; + while (i < links.length) { + if (Links.compareLinks(links[i - 1], links[i]) > 0) { + outOfOrder.push(links.splice(i, 1)[0]); + } else { + i++; + } + } + for (let link of outOfOrder) { + i = lazy.BinarySearch.insertionIndexOf( + Links.compareLinks, + links, + link + ); + links.splice(i, 0, link); + } + + aCallback(links); + }, + }; + + // Execute the query. + let query = lazy.PlacesUtils.history.getNewQuery(); + lazy.PlacesUtils.history.asyncExecuteLegacyQuery(query, options, callback); + }, + + /** + * Registers an object that will be notified when the provider's links change. + * @param aObserver An object with the following optional properties: + * * onLinkChanged: A function that's called when a single link + * changes. It's passed the provider and the link object. Only the + * link's `url` property is guaranteed to be present. If its `title` + * property is present, then its title has changed, and the + * property's value is the new title. If any sort properties are + * present, then its position within the provider's list of links may + * have changed, and the properties' values are the new sort-related + * values. Note that this link may not necessarily have been present + * in the lists returned from any previous calls to getLinks. + * * onManyLinksChanged: A function that's called when many links + * change at once. It's passed the provider. You should call + * getLinks to get the provider's new list of links. + */ + addObserver: function PlacesProvider_addObserver(aObserver) { + this._observers.push(aObserver); + }, + + _observers: [], + + handlePlacesEvents(aEvents) { + for (let event of aEvents) { + switch (event.type) { + case "page-visited": { + if (event.visitCount == 1 && event.lastKnownTitle) { + this._callObservers("onLinkChanged", { + url: event.url, + title: event.lastKnownTitle, + }); + } + break; + } + case "page-title-changed": { + this._callObservers("onLinkChanged", { + url: event.url, + title: event.title, + }); + break; + } + case "pages-rank-changed": { + this._callObservers("onManyLinksChanged"); + break; + } + } + } + }, + + _callObservers: function PlacesProvider__callObservers(aMethodName, aArg) { + for (let obs of this._observers) { + if (obs[aMethodName]) { + try { + obs[aMethodName](this, aArg); + } catch (err) { + console.error(err); + } + } + } + }, +}; + +/** + * Queries history to retrieve the most frecent sites. Emits events when the + * history changes. + */ +var ActivityStreamProvider = { + THUMB_FAVICON_SIZE: 96, + + /** + * Shared adjustment for selecting potentially blocked links. + */ + _adjustLimitForBlocked({ ignoreBlocked, numItems }) { + // Just use the usual number if blocked links won't be filtered out + if (ignoreBlocked) { + return numItems; + } + // Additionally select the number of blocked links in case they're removed + return Object.keys(BlockedLinks.links).length + numItems; + }, + + /** + * Shared sub-SELECT to get the guid of a bookmark of the current url while + * avoiding LEFT JOINs on moz_bookmarks. This avoids gettings tags. The guid + * could be one of multiple possible guids. Assumes `moz_places h` is in FROM. + */ + _commonBookmarkGuidSelect: `( + SELECT guid + FROM moz_bookmarks b + WHERE fk = h.id + AND type = :bookmarkType + AND ( + SELECT id + FROM moz_bookmarks p + WHERE p.id = b.parent + AND p.parent <> :tagsFolderId + ) NOTNULL + ) AS bookmarkGuid`, + + /** + * Shared WHERE expression filtering out undesired pages, e.g., hidden, + * unvisited, and non-http/s urls. Assumes moz_places is in FROM / JOIN. + * + * NB: SUBSTR(url) is used even without an index instead of url_hash because + * most desired pages will match http/s, so it will only run on the ~10s of + * rows matched. If url_hash were to be used, it should probably *not* be used + * by the query optimizer as we primarily want it optimized for the other + * conditions, e.g., most frecent first. + */ + _commonPlacesWhere: ` + AND hidden = 0 + AND last_visit_date > 0 + AND (SUBSTR(url, 1, 6) == "https:" + OR SUBSTR(url, 1, 5) == "http:") + `, + + /** + * Shared parameters for getting correct bookmarks and LIMITed queries. + */ + _getCommonParams(aOptions, aParams = {}) { + return Object.assign( + { + bookmarkType: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + limit: this._adjustLimitForBlocked(aOptions), + tagsFolderId: lazy.PlacesUtils.tagsFolderId, + }, + aParams + ); + }, + + /** + * Shared columns for Highlights related queries. + */ + _highlightsColumns: [ + "bookmarkGuid", + "description", + "guid", + "preview_image_url", + "title", + "url", + ], + + /** + * Shared post-processing of Highlights links. + */ + _processHighlights(aLinks, aOptions, aType) { + // Filter out blocked if necessary + if (!aOptions.ignoreBlocked) { + aLinks = aLinks.filter( + link => + !BlockedLinks.isBlocked( + link.pocket_id ? { url: link.open_url } : link + ) + ); + } + + // Limit the results to the requested number and set a type corresponding to + // which query selected it + return aLinks.slice(0, aOptions.numItems).map(item => + Object.assign(item, { + type: aType, + }) + ); + }, + + /** + * From an Array of links, if favicons are present, convert to data URIs + * + * @param {Array} aLinks + * an array containing objects with favicon data and mimeTypes + * + * @returns {Array} an array of links with favicons as data uri + */ + _faviconBytesToDataURI(aLinks) { + return aLinks.map(link => { + if (link.favicon) { + let encodedData = btoa(String.fromCharCode.apply(null, link.favicon)); + link.favicon = `data:${link.mimeType};base64,${encodedData}`; + delete link.mimeType; + } + + if (link.smallFavicon) { + let encodedData = btoa( + String.fromCharCode.apply(null, link.smallFavicon) + ); + link.smallFavicon = `data:${link.smallFaviconMimeType};base64,${encodedData}`; + delete link.smallFaviconMimeType; + } + + return link; + }); + }, + + /** + * Get favicon data (and metadata) for a uri. Fetches both the largest favicon + * available, for Activity Stream; and a normal-sized favicon, for the Urlbar. + * + * @param {nsIURI} aUri Page to check for favicon data + * @param {number} preferredFaviconWidth + * The preferred width of the of the normal-sized favicon in pixels. + * @returns A promise of an object (possibly empty) containing the data. + */ + async _loadIcons(aUri, preferredFaviconWidth) { + let iconData = {}; + // Fetch the largest icon available. + let faviconData; + try { + faviconData = await lazy.PlacesUtils.promiseFaviconData( + aUri, + this.THUMB_FAVICON_SIZE + ); + Object.assign(iconData, { + favicon: faviconData.data, + faviconLength: faviconData.dataLen, + faviconRef: faviconData.uri.ref, + faviconSize: faviconData.size, + mimeType: faviconData.mimeType, + }); + } catch (e) { + // Return early because fetching the largest favicon is the primary + // purpose of NewTabUtils. + return null; + } + + // Also fetch a smaller icon. + try { + faviconData = await lazy.PlacesUtils.promiseFaviconData( + aUri, + preferredFaviconWidth + ); + Object.assign(iconData, { + smallFavicon: faviconData.data, + smallFaviconLength: faviconData.dataLen, + smallFaviconRef: faviconData.uri.ref, + smallFaviconSize: faviconData.size, + smallFaviconMimeType: faviconData.mimeType, + }); + } catch (e) { + // Do nothing with the error since we still have the large favicon fields. + } + + return iconData; + }, + + /** + * Computes favicon data for each url in a set of links + * + * @param {Array} links + * an array containing objects without favicon data or mimeTypes yet + * + * @returns {Promise} Returns a promise with the array of links with the largest + * favicon available (as a byte array), mimeType, byte array + * length, and favicon size (width) + */ + _addFavicons(aLinks) { + let win; + if (BrowserWindowTracker) { + win = BrowserWindowTracker.getTopWindow(); + } + // We fetch two copies of a page's favicon: the largest available, for + // Activity Stream; and a smaller size appropriate for the Urlbar. + const preferredFaviconWidth = + DEFAULT_SMALL_FAVICON_WIDTH * (win ? win.devicePixelRatio : 2); + // Each link in the array needs a favicon for it's page - so we fire off a + // promise for each link to compute the favicon data and attach it back to + // the original link object. We must wait until all favicons for the array + // of links are computed before returning + return Promise.all( + aLinks.map( + link => + // eslint-disable-next-line no-async-promise-executor + new Promise(async resolve => { + // Never add favicon data for pocket items + if (link.type === "pocket") { + resolve(link); + return; + } + let iconData; + try { + let linkUri = Services.io.newURI(link.url); + iconData = await this._loadIcons(linkUri, preferredFaviconWidth); + + // Switch the scheme to try again with the other + if (!iconData) { + linkUri = linkUri + .mutate() + .setScheme(linkUri.scheme === "https" ? "http" : "https") + .finalize(); + iconData = await this._loadIcons( + linkUri, + preferredFaviconWidth + ); + } + } catch (e) { + // We just won't put icon data on the link + } + + // Add the icon data to the link if we have any + resolve(Object.assign(link, iconData)); + }) + ) + ); + }, + + /** + * Helper function which makes the call to the Pocket API to fetch the user's + * saved Pocket items. + */ + fetchSavedPocketItems(requestData) { + const latestSince = + Services.prefs.getStringPref(PREF_POCKET_LATEST_SINCE, 0) * 1000; + + // Do not fetch Pocket items for users that have been inactive for too long, or are not logged in + if ( + !lazy.pktApi.isUserLoggedIn() || + Date.now() - latestSince > POCKET_INACTIVE_TIME + ) { + return Promise.resolve(null); + } + + return new Promise((resolve, reject) => { + lazy.pktApi.retrieve(requestData, { + success(data) { + resolve(data); + }, + error(error) { + reject(error); + }, + }); + }); + }, + + /** + * Get the most recently Pocket-ed items from a user's Pocket list. See: + * https://getpocket.com/developer/docs/v3/retrieve for details + * + * @param {Object} aOptions + * {int} numItems: The max number of pocket items to fetch + */ + async getRecentlyPocketed(aOptions) { + const pocketSecondsAgo = + Math.floor(Date.now() / 1000) - ACTIVITY_STREAM_DEFAULT_RECENT; + const requestData = { + detailType: "complete", + count: aOptions.numItems, + since: pocketSecondsAgo, + }; + let data; + try { + data = await this.fetchSavedPocketItems(requestData); + if (!data) { + return []; + } + } catch (e) { + console.error(e); + return []; + } + /* Extract relevant parts needed to show this card as a highlight: + * url, preview image, title, description, and the unique item_id + * necessary for Pocket to identify the item + */ + let items = Object.values(data.list) + // status "0" means not archived or deleted + .filter(item => item.status === "0") + .map(item => ({ + date_added: item.time_added * 1000, + description: item.excerpt, + preview_image_url: item.image && item.image.src, + title: item.resolved_title, + url: item.resolved_url, + pocket_id: item.item_id, + open_url: item.open_url, + })); + + // Append the query param to let Pocket know this item came from highlights + for (let item of items) { + let url = new URL(item.open_url); + url.searchParams.append("src", "fx_new_tab"); + item.open_url = url.href; + } + + return this._processHighlights(items, aOptions, "pocket"); + }, + + /** + * Get most-recently-created visited bookmarks for Activity Stream. + * + * @param {Object} aOptions + * {num} bookmarkSecondsAgo: Maximum age of added bookmark. + * {bool} ignoreBlocked: Do not filter out blocked links. + * {int} numItems: Maximum number of items to return. + */ + async getRecentBookmarks(aOptions) { + const options = Object.assign( + { + bookmarkSecondsAgo: ACTIVITY_STREAM_DEFAULT_RECENT, + ignoreBlocked: false, + numItems: ACTIVITY_STREAM_DEFAULT_LIMIT, + }, + aOptions || {} + ); + + const sqlQuery = ` + SELECT + b.guid AS bookmarkGuid, + description, + h.guid, + preview_image_url, + b.title, + b.dateAdded / 1000 AS date_added, + url + FROM moz_bookmarks b + JOIN moz_bookmarks p + ON p.id = b.parent + JOIN moz_places h + ON h.id = b.fk + WHERE b.dateAdded >= :dateAddedThreshold + AND b.title NOTNULL + AND b.type = :bookmarkType + AND p.parent <> :tagsFolderId + ${this._commonPlacesWhere} + ORDER BY b.dateAdded DESC + LIMIT :limit + `; + + return this._processHighlights( + await this.executePlacesQuery(sqlQuery, { + columns: [...this._highlightsColumns, "date_added"], + params: this._getCommonParams(options, { + dateAddedThreshold: + (Date.now() - options.bookmarkSecondsAgo * 1000) * 1000, + }), + }), + options, + "bookmark" + ); + }, + + /** + * Get total count of all bookmarks. + * Note: this includes default bookmarks + * + * @return {int} The number bookmarks in the places DB. + */ + async getTotalBookmarksCount() { + let sqlQuery = ` + SELECT count(*) FROM moz_bookmarks b + JOIN moz_bookmarks t ON t.id = b.parent + AND t.parent <> :tags_folder + WHERE b.type = :type_bookmark + `; + + const result = await this.executePlacesQuery(sqlQuery, { + params: { + tags_folder: lazy.PlacesUtils.tagsFolderId, + type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK, + }, + }); + + return result[0][0]; + }, + + /** + * Get most-recently-visited history with metadata for Activity Stream. + * + * @param {Object} aOptions + * {bool} ignoreBlocked: Do not filter out blocked links. + * {int} numItems: Maximum number of items to return. + */ + async getRecentHistory(aOptions) { + const options = Object.assign( + { + ignoreBlocked: false, + numItems: ACTIVITY_STREAM_DEFAULT_LIMIT, + }, + aOptions || {} + ); + + const sqlQuery = ` + SELECT + ${this._commonBookmarkGuidSelect}, + description, + guid, + preview_image_url, + title, + url + FROM moz_places h + WHERE description NOTNULL + AND preview_image_url NOTNULL + ${this._commonPlacesWhere} + ORDER BY last_visit_date DESC + LIMIT :limit + `; + + return this._processHighlights( + await this.executePlacesQuery(sqlQuery, { + columns: this._highlightsColumns, + params: this._getCommonParams(options), + }), + options, + "history" + ); + }, + + /* + * Gets the top frecent sites for Activity Stream. + * + * @param {Object} aOptions + * {bool} ignoreBlocked: Do not filter out blocked links. + * {int} numItems: Maximum number of items to return. + * {int} topsiteFrecency: Minimum amount of frecency for a site. + * {bool} onePerDomain: Dedupe the resulting list. + * {bool} includeFavicon: Include favicons if available. + * {string} hideWithSearchParam: URLs that contain this search param will be + * excluded from the returned links. This value should be either undefined + * or a string with one of the following forms: + * - undefined: Fall back to the value of pref + * `browser.newtabpage.activity-stream.hideTopSitesWithSearchParam` + * - "" (empty) - Disable this feature + * - "key" - Search param named "key" with any or no value + * - "key=" - Search param named "key" with no value + * - "key=value" - Search param named "key" with value "value" + * + * @returns {Promise} Returns a promise with the array of links as payload. + */ + async getTopFrecentSites(aOptions) { + const options = Object.assign( + { + ignoreBlocked: false, + numItems: ACTIVITY_STREAM_DEFAULT_LIMIT, + topsiteFrecency: ACTIVITY_STREAM_DEFAULT_FRECENCY, + onePerDomain: true, + includeFavicon: true, + hideWithSearchParam: Services.prefs.getCharPref( + "browser.newtabpage.activity-stream.hideTopSitesWithSearchParam", + "" + ), + }, + aOptions || {} + ); + + // Double the item count in case the host is deduped between with www or + // not-www (i.e., 2 hosts) and an extra buffer for multiple pages per host. + const origNumItems = options.numItems; + if (options.onePerDomain) { + options.numItems *= 2 * 10; + } + + // Keep this query fast with frecency-indexed lookups (even with excess + // rows) and shift the more complex logic to post-processing afterwards + const sqlQuery = ` + SELECT + ${this._commonBookmarkGuidSelect}, + frecency, + guid, + last_visit_date / 1000 AS lastVisitDate, + rev_host, + title, + url, + "history" as type + FROM moz_places h + WHERE frecency >= :frecencyThreshold + ${this._commonPlacesWhere} + ORDER BY frecency DESC + LIMIT :limit + `; + + let links = await this.executePlacesQuery(sqlQuery, { + columns: [ + "bookmarkGuid", + "frecency", + "guid", + "lastVisitDate", + "title", + "url", + "type", + ], + params: this._getCommonParams(options, { + frecencyThreshold: options.topsiteFrecency, + }), + }); + + // Determine if the other link is "better" (larger frecency, more recent, + // lexicographically earlier url) + function isOtherBetter(link, other) { + if (other.frecency === link.frecency) { + if (other.lastVisitDate === link.lastVisitDate) { + return other.url < link.url; + } + return other.lastVisitDate > link.lastVisitDate; + } + return other.frecency > link.frecency; + } + + // Update a host Map with the better link + function setBetterLink(map, link, hostMatcher, combiner = () => {}) { + const host = hostMatcher(link.url)[1]; + if (map.has(host)) { + const other = map.get(host); + if (isOtherBetter(link, other)) { + link = other; + } + combiner(link, other); + } + map.set(host, link); + } + + // Convert all links that are supposed to be a seach shortcut to its canonical URL + if ( + didSuccessfulImport && + Services.prefs.getBoolPref( + `browser.newtabpage.activity-stream.${searchShortcuts.SEARCH_SHORTCUTS_EXPERIMENT}` + ) + ) { + links.forEach(link => { + let searchProvider = searchShortcuts.getSearchProvider( + shortURL.shortURL(link) + ); + if (searchProvider) { + link.url = searchProvider.url; + } + }); + } + + // Remove links that contain the hide-with search param. + if (options.hideWithSearchParam) { + let [key, value] = options.hideWithSearchParam.split("="); + links = links.filter(link => { + try { + let { searchParams } = new URL(link.url); + return value === undefined + ? !searchParams.has(key) + : !searchParams.getAll(key).includes(value); + } catch (error) {} + return true; + }); + } + + // Remove any blocked links. + if (!options.ignoreBlocked) { + links = links.filter(link => !BlockedLinks.isBlocked(link)); + } + + if (options.onePerDomain) { + // De-dup the links. + const exactHosts = new Map(); + for (const link of links) { + // First we want to find the best link for an exact host + setBetterLink(exactHosts, link, url => url.match(/:\/\/([^\/]+)/)); + } + + // Clean up exact hosts to dedupe as non-www hosts + const hosts = new Map(); + for (const link of exactHosts.values()) { + setBetterLink( + hosts, + link, + url => url.match(/:\/\/(?:www\.)?([^\/]+)/), + // Combine frecencies when deduping these links + (targetLink, otherLink) => { + targetLink.frecency = link.frecency + otherLink.frecency; + } + ); + } + + links = [...hosts.values()]; + } + // Pick out the top links using the same comparer as before + links = links.sort(isOtherBetter).slice(0, origNumItems); + + if (!options.includeFavicon) { + return links; + } + // Get the favicons as data URI for now (until we use the favicon protocol) + return this._faviconBytesToDataURI(await this._addFavicons(links)); + }, + + /** + * Gets a specific bookmark given some info about it + * + * @param {Obj} aInfo + * An object with one and only one of the following properties: + * - url + * - guid + * - parentGuid and index + */ + async getBookmark(aInfo) { + let bookmark = await lazy.PlacesUtils.bookmarks.fetch(aInfo); + if (!bookmark) { + return null; + } + let result = {}; + result.bookmarkGuid = bookmark.guid; + result.bookmarkTitle = bookmark.title; + result.lastModified = bookmark.lastModified.getTime(); + result.url = bookmark.url.href; + return result; + }, + + /** + * Count the number of visited urls grouped by day + */ + getUserMonthlyActivity() { + let sqlQuery = ` + SELECT count(*), + strftime('%Y-%m-%d', visit_date/1000000.0, 'unixepoch', 'localtime') as date_format + FROM moz_historyvisits + WHERE visit_date > 0 + AND visit_date > strftime('%s','now','localtime','start of day','-27 days','utc') * 1000000 + GROUP BY date_format + ORDER BY date_format ASC + `; + + return this.executePlacesQuery(sqlQuery); + }, + + /** + * Executes arbitrary query against places database + * + * @param {String} aQuery + * SQL query to execute + * @param {Object} [optional] aOptions + * aOptions.columns - an array of column names. if supplied the return + * items will consists of objects keyed on column names. Otherwise + * array of raw values is returned in the select order + * aOptions.param - an object of SQL binding parameters + * + * @returns {Promise} Returns a promise with the array of retrieved items + */ + async executePlacesQuery(aQuery, aOptions = {}) { + let { columns, params } = aOptions; + let items = []; + let queryError = null; + let conn = await lazy.PlacesUtils.promiseDBConnection(); + await conn.executeCached(aQuery, params, (aRow, aCancel) => { + try { + let item = null; + // if columns array is given construct an object + if (columns && Array.isArray(columns)) { + item = {}; + columns.forEach(column => { + item[column] = aRow.getResultByName(column); + }); + } else { + // if no columns - make an array of raw values + item = []; + for (let i = 0; i < aRow.numEntries; i++) { + item.push(aRow.getResultByIndex(i)); + } + } + items.push(item); + } catch (e) { + queryError = e; + aCancel(); + } + }); + if (queryError) { + throw new Error(queryError); + } + return items; + }, +}; + +/** + * A set of actions which influence what sites shown on the Activity Stream page + */ +var ActivityStreamLinks = { + _savedPocketStories: null, + _pocketLastUpdated: 0, + _pocketLastLatest: 0, + + /** + * Block a url + * + * @param {Object} aLink + * The link which contains a URL to add to the block list + */ + blockURL(aLink) { + BlockedLinks.block(aLink); + // If we're blocking a pocket item, invalidate the cache too + if (aLink.pocket_id) { + this._savedPocketStories = null; + } + }, + + onLinkBlocked(aLink) { + Services.obs.notifyObservers(null, "newtab-linkBlocked", aLink.url); + }, + + /** + * Adds a bookmark and opens up the Bookmark Dialog to show feedback that + * the bookmarking action has been successful + * + * @param {Object} aData + * aData.url The url to bookmark + * aData.title The title of the page to bookmark + * @param {Window} aBrowserWindow + * The current browser chrome window + * + * @returns {Promise} Returns a promise set to an object representing the bookmark + */ + addBookmark(aData, aBrowserWindow) { + const { url, title } = aData; + return aBrowserWindow.PlacesCommandHook.bookmarkLink(url, title); + }, + + /** + * Removes a bookmark + * + * @param {String} aBookmarkGuid + * The bookmark guid associated with the bookmark to remove + * + * @returns {Promise} Returns a promise at completion. + */ + deleteBookmark(aBookmarkGuid) { + return lazy.PlacesUtils.bookmarks.remove(aBookmarkGuid); + }, + + /** + * Removes a history link and unpins the URL if previously pinned + * + * @param {String} aUrl + * The url to be removed from history + * + * @returns {Promise} Returns a promise set to true if link was removed + */ + deleteHistoryEntry(aUrl) { + const url = aUrl; + PinnedLinks.unpin({ url }); + return lazy.PlacesUtils.history.remove(url); + }, + + /** + * Helper function which makes the call to the Pocket API to delete an item from + * a user's saved to Pocket feed. Also, invalidate the Pocket stories cache + * + * @param {Integer} aItemID + * The unique pocket ID used to find the item to be deleted + * + *@returns {Promise} Returns a promise at completion + */ + deletePocketEntry(aItemID) { + this._savedPocketStories = null; + return new Promise((success, error) => + lazy.pktApi.deleteItem(aItemID, { success, error }) + ); + }, + + /** + * Helper function which makes the call to the Pocket API to archive an item from + * a user's saved to Pocket feed. Also, invalidate the Pocket stories cache + * + * @param {Integer} aItemID + * The unique pocket ID used to find the item to be archived + * + *@returns {Promise} Returns a promise at completion + */ + archivePocketEntry(aItemID) { + this._savedPocketStories = null; + return new Promise((success, error) => + lazy.pktApi.archiveItem(aItemID, { success, error }) + ); + }, + + /** + * Helper function which makes the call to the Pocket API to save an item to + * a user's saved to Pocket feed if they are logged in. Also, invalidate the + * Pocket stories cache + * + * @param {String} aUrl + * The URL belonging to the story being saved + * @param {String} aTitle + * The title belonging to the story being saved + * @param {Browser} aBrowser + * The target browser to show the doorhanger in + * + *@returns {Promise} Returns a promise at completion + */ + addPocketEntry(aUrl, aTitle, aBrowser) { + // If the user is not logged in, show the panel to prompt them to log in + if (!lazy.pktApi.isUserLoggedIn()) { + lazy.Pocket.savePage(aBrowser, aUrl, aTitle); + return Promise.resolve(null); + } + + // If the user is logged in, just save the link to Pocket and Activity Stream + // will update the page + this._savedPocketStories = null; + return new Promise((success, error) => { + lazy.pktApi.addLink(aUrl, { + title: aTitle, + success, + error, + }); + }); + }, + + /** + * Get the Highlights links to show on Activity Stream + * + * @param {Object} aOptions + * {bool} excludeBookmarks: Don't add bookmark items. + * {bool} excludeHistory: Don't add history items. + * {bool} excludePocket: Don't add Pocket items. + * {bool} withFavicons: Add favicon data: URIs, when possible. + * {int} numItems: Maximum number of (bookmark or history) items to return. + * + * @return {Promise} Returns a promise with the array of links as the payload + */ + async getHighlights(aOptions = {}) { + aOptions.numItems = aOptions.numItems || ACTIVITY_STREAM_DEFAULT_LIMIT; + const results = []; + + // First get bookmarks if we want them + if (!aOptions.excludeBookmarks) { + results.push( + ...(await ActivityStreamProvider.getRecentBookmarks(aOptions)) + ); + } + + // Add the Pocket items if we need more and want them + if (aOptions.numItems - results.length > 0 && !aOptions.excludePocket) { + const latestSince = ~~Services.prefs.getStringPref( + PREF_POCKET_LATEST_SINCE, + 0 + ); + // Invalidate the cache, get new stories, and update timestamps if: + // 1. we do not have saved to Pocket stories already cached OR + // 2. it has been too long since we last got Pocket stories OR + // 3. there has been a paged saved to pocket since we last got new stories + if ( + !this._savedPocketStories || + Date.now() - this._pocketLastUpdated > POCKET_UPDATE_TIME || + this._pocketLastLatest < latestSince + ) { + this._savedPocketStories = + await ActivityStreamProvider.getRecentlyPocketed(aOptions); + this._pocketLastUpdated = Date.now(); + this._pocketLastLatest = latestSince; + } + results.push(...this._savedPocketStories); + } + + // Add in history if we need more and want them + if (aOptions.numItems - results.length > 0 && !aOptions.excludeHistory) { + // Use the same numItems as bookmarks above in case we remove duplicates + const history = await ActivityStreamProvider.getRecentHistory(aOptions); + + // Only include a url once in the result preferring the bookmark + const bookmarkUrls = new Set(results.map(({ url }) => url)); + for (const page of history) { + if (!bookmarkUrls.has(page.url)) { + results.push(page); + + // Stop adding pages once we reach the desired maximum + if (results.length === aOptions.numItems) { + break; + } + } + } + } + + if (aOptions.withFavicons) { + return ActivityStreamProvider._faviconBytesToDataURI( + await ActivityStreamProvider._addFavicons(results) + ); + } + + return results; + }, + + /** + * Get the top sites to show on Activity Stream + * + * @return {Promise} Returns a promise with the array of links as the payload + */ + async getTopSites(aOptions = {}) { + return ActivityStreamProvider.getTopFrecentSites(aOptions); + }, +}; + +/** + * Singleton that provides access to all links contained in the grid (including + * the ones that don't fit on the grid). A link is a plain object that looks + * like this: + * + * { + * url: "http://www.mozilla.org/", + * title: "Mozilla", + * frecency: 1337, + * lastVisitDate: 1394678824766431, + * } + */ +var Links = { + /** + * The maximum number of links returned by getLinks. + */ + maxNumLinks: LINKS_GET_LINKS_LIMIT, + + /** + * A mapping from each provider to an object { sortedLinks, siteMap, linkMap }. + * sortedLinks is the cached, sorted array of links for the provider. + * siteMap is a mapping from base domains to URL count associated with the domain. + * The count does not include blocked URLs. siteMap is used to look up a + * user's top sites that can be targeted with a suggested tile. + * linkMap is a Map from link URLs to link objects. + */ + _providers: new Map(), + + /** + * The properties of link objects used to sort them. + */ + _sortProperties: ["frecency", "lastVisitDate", "url"], + + /** + * List of callbacks waiting for the cache to be populated. + */ + _populateCallbacks: [], + + /** + * A list of objects that are observing links updates. + */ + _observers: [], + + /** + * Registers an object that will be notified when links updates. + */ + addObserver(aObserver) { + this._observers.push(aObserver); + }, + + /** + * Adds a link provider. + * @param aProvider The link provider. + */ + addProvider: function Links_addProvider(aProvider) { + this._providers.set(aProvider, null); + aProvider.addObserver(this); + }, + + /** + * Removes a link provider. + * @param aProvider The link provider. + */ + removeProvider: function Links_removeProvider(aProvider) { + if (!this._providers.delete(aProvider)) { + throw new Error("Unknown provider"); + } + }, + + /** + * Populates the cache with fresh links from the providers. + * @param aCallback The callback to call when finished (optional). + * @param aForce When true, populates the cache even when it's already filled. + */ + populateCache: function Links_populateCache(aCallback, aForce) { + let callbacks = this._populateCallbacks; + + // Enqueue the current callback. + callbacks.push(aCallback); + + // There was a callback waiting already, thus the cache has not yet been + // populated. + if (callbacks.length > 1) { + return; + } + + function executeCallbacks() { + while (callbacks.length) { + let callback = callbacks.shift(); + if (callback) { + try { + callback(); + } catch (e) { + // We want to proceed even if a callback fails. + } + } + } + } + + let numProvidersRemaining = this._providers.size; + for (let [provider /* , links */] of this._providers) { + this._populateProviderCache( + provider, + () => { + if (--numProvidersRemaining == 0) { + executeCallbacks(); + } + }, + aForce + ); + } + + this._addObserver(); + }, + + /** + * Gets the current set of links contained in the grid. + * @return The links in the grid. + */ + getLinks: function Links_getLinks() { + let pinnedLinks = Array.from(PinnedLinks.links); + let links = this._getMergedProviderLinks(); + + let sites = new Set(); + for (let link of pinnedLinks) { + if (link) { + sites.add(NewTabUtils.extractSite(link.url)); + } + } + + // Filter blocked and pinned links and duplicate base domains. + links = links.filter(function (link) { + let site = NewTabUtils.extractSite(link.url); + if (site == null || sites.has(site)) { + return false; + } + sites.add(site); + + return !BlockedLinks.isBlocked(link) && !PinnedLinks.isPinned(link); + }); + + // Try to fill the gaps between pinned links. + for (let i = 0; i < pinnedLinks.length && links.length; i++) { + if (!pinnedLinks[i]) { + pinnedLinks[i] = links.shift(); + } + } + + // Append the remaining links if any. + if (links.length) { + pinnedLinks = pinnedLinks.concat(links); + } + + for (let link of pinnedLinks) { + if (link) { + link.baseDomain = NewTabUtils.extractSite(link.url); + } + } + return pinnedLinks; + }, + + /** + * Resets the links cache. + */ + resetCache: function Links_resetCache() { + for (let provider of this._providers.keys()) { + this._providers.set(provider, null); + } + }, + + /** + * Compares two links. + * @param aLink1 The first link. + * @param aLink2 The second link. + * @return A negative number if aLink1 is ordered before aLink2, zero if + * aLink1 and aLink2 have the same ordering, or a positive number if + * aLink1 is ordered after aLink2. + * + * @note compareLinks's this object is bound to Links below. + */ + compareLinks: function Links_compareLinks(aLink1, aLink2) { + for (let prop of this._sortProperties) { + if (!(prop in aLink1) || !(prop in aLink2)) { + throw new Error("Comparable link missing required property: " + prop); + } + } + return ( + aLink2.frecency - aLink1.frecency || + aLink2.lastVisitDate - aLink1.lastVisitDate || + aLink1.url.localeCompare(aLink2.url) + ); + }, + + _incrementSiteMap(map, link) { + if (NewTabUtils.blockedLinks.isBlocked(link)) { + // Don't count blocked URLs. + return; + } + let site = NewTabUtils.extractSite(link.url); + map.set(site, (map.get(site) || 0) + 1); + }, + + _decrementSiteMap(map, link) { + if (NewTabUtils.blockedLinks.isBlocked(link)) { + // Blocked URLs are not included in map. + return; + } + let site = NewTabUtils.extractSite(link.url); + let previousURLCount = map.get(site); + if (previousURLCount === 1) { + map.delete(site); + } else { + map.set(site, previousURLCount - 1); + } + }, + + /** + * Update the siteMap cache based on the link given and whether we need + * to increment or decrement it. We do this by iterating over all stored providers + * to find which provider this link already exists in. For providers that + * have this link, we will adjust siteMap for them accordingly. + * + * @param aLink The link that will affect siteMap + * @param increment A boolean for whether to increment or decrement siteMap + */ + _adjustSiteMapAndNotify(aLink, increment = true) { + for (let [, /* provider */ cache] of this._providers) { + // We only update siteMap if aLink is already stored in linkMap. + if (cache.linkMap.get(aLink.url)) { + if (increment) { + this._incrementSiteMap(cache.siteMap, aLink); + continue; + } + this._decrementSiteMap(cache.siteMap, aLink); + } + } + this._callObservers("onLinkChanged", aLink); + }, + + onLinkBlocked(aLink) { + this._adjustSiteMapAndNotify(aLink, false); + }, + + onLinkUnblocked(aLink) { + this._adjustSiteMapAndNotify(aLink); + }, + + populateProviderCache(provider, callback) { + if (!this._providers.has(provider)) { + throw new Error( + "Can only populate provider cache for existing provider." + ); + } + + return this._populateProviderCache(provider, callback, false); + }, + + /** + * Calls getLinks on the given provider and populates our cache for it. + * @param aProvider The provider whose cache will be populated. + * @param aCallback The callback to call when finished. + * @param aForce When true, populates the provider's cache even when it's + * already filled. + */ + _populateProviderCache(aProvider, aCallback, aForce) { + let cache = this._providers.get(aProvider); + let createCache = !cache; + if (createCache) { + cache = { + // Start with a resolved promise. + populatePromise: new Promise(resolve => resolve()), + }; + this._providers.set(aProvider, cache); + } + // Chain the populatePromise so that calls are effectively queued. + cache.populatePromise = cache.populatePromise.then(() => { + return new Promise(resolve => { + if (!createCache && !aForce) { + aCallback(); + resolve(); + return; + } + aProvider.getLinks(links => { + // Filter out null and undefined links so we don't have to deal with + // them in getLinks when merging links from providers. + links = links.filter(link => !!link); + cache.sortedLinks = links; + cache.siteMap = links.reduce((map, link) => { + this._incrementSiteMap(map, link); + return map; + }, new Map()); + cache.linkMap = links.reduce((map, link) => { + map.set(link.url, link); + return map; + }, new Map()); + aCallback(); + resolve(); + }); + }); + }); + }, + + /** + * Merges the cached lists of links from all providers whose lists are cached. + * @return The merged list. + */ + _getMergedProviderLinks: function Links__getMergedProviderLinks() { + // Build a list containing a copy of each provider's sortedLinks list. + let linkLists = []; + for (let provider of this._providers.keys()) { + let links = this._providers.get(provider); + if (links && links.sortedLinks) { + linkLists.push(links.sortedLinks.slice()); + } + } + + return this.mergeLinkLists(linkLists); + }, + + mergeLinkLists: function Links_mergeLinkLists(linkLists) { + if (linkLists.length == 1) { + return linkLists[0]; + } + + function getNextLink() { + let minLinks = null; + for (let links of linkLists) { + if ( + links.length && + (!minLinks || Links.compareLinks(links[0], minLinks[0]) < 0) + ) { + minLinks = links; + } + } + return minLinks ? minLinks.shift() : null; + } + + let finalLinks = []; + for ( + let nextLink = getNextLink(); + nextLink && finalLinks.length < this.maxNumLinks; + nextLink = getNextLink() + ) { + finalLinks.push(nextLink); + } + + return finalLinks; + }, + + /** + * Called by a provider to notify us when a single link changes. + * @param aProvider The provider whose link changed. + * @param aLink The link that changed. If the link is new, it must have all + * of the _sortProperties. Otherwise, it may have as few or as + * many as is convenient. + * @param aIndex The current index of the changed link in the sortedLinks + cache in _providers. Defaults to -1 if the provider doesn't know the index + * @param aDeleted Boolean indicating if the provider has deleted the link. + */ + onLinkChanged: function Links_onLinkChanged( + aProvider, + aLink, + aIndex = -1, + aDeleted = false + ) { + if (!("url" in aLink)) { + throw new Error("Changed links must have a url property"); + } + + let links = this._providers.get(aProvider); + if (!links) { + // This is not an error, it just means that between the time the provider + // was added and the future time we call getLinks on it, it notified us of + // a change. + return; + } + + let { sortedLinks, siteMap, linkMap } = links; + let existingLink = linkMap.get(aLink.url); + let insertionLink = null; + let updatePages = false; + + if (existingLink) { + // Update our copy's position in O(lg n) by first removing it from its + // list. It's important to do this before modifying its properties. + if (this._sortProperties.some(prop => prop in aLink)) { + let idx = aIndex; + if (idx < 0) { + idx = this._indexOf(sortedLinks, existingLink); + } else if (this.compareLinks(aLink, sortedLinks[idx]) != 0) { + throw new Error("aLink should be the same as sortedLinks[idx]"); + } + + if (idx < 0) { + throw new Error("Link should be in _sortedLinks if in _linkMap"); + } + sortedLinks.splice(idx, 1); + + if (aDeleted) { + updatePages = true; + linkMap.delete(existingLink.url); + this._decrementSiteMap(siteMap, existingLink); + } else { + // Update our copy's properties. + Object.assign(existingLink, aLink); + + // Finally, reinsert our copy below. + insertionLink = existingLink; + } + } + // Update our copy's title in O(1). + if ("title" in aLink && aLink.title != existingLink.title) { + existingLink.title = aLink.title; + updatePages = true; + } + } else if (this._sortProperties.every(prop => prop in aLink)) { + // Before doing the O(lg n) insertion below, do an O(1) check for the + // common case where the new link is too low-ranked to be in the list. + if (sortedLinks.length && sortedLinks.length == aProvider.maxNumLinks) { + let lastLink = sortedLinks[sortedLinks.length - 1]; + if (this.compareLinks(lastLink, aLink) < 0) { + return; + } + } + // Copy the link object so that changes later made to it by the caller + // don't affect our copy. + insertionLink = {}; + for (let prop in aLink) { + insertionLink[prop] = aLink[prop]; + } + linkMap.set(aLink.url, insertionLink); + this._incrementSiteMap(siteMap, aLink); + } + + if (insertionLink) { + let idx = this._insertionIndexOf(sortedLinks, insertionLink); + sortedLinks.splice(idx, 0, insertionLink); + if (sortedLinks.length > aProvider.maxNumLinks) { + let lastLink = sortedLinks.pop(); + linkMap.delete(lastLink.url); + this._decrementSiteMap(siteMap, lastLink); + } + updatePages = true; + } + + if (updatePages) { + AllPages.update(null, "links-changed"); + } + }, + + /** + * Called by a provider to notify us when many links change. + */ + onManyLinksChanged: function Links_onManyLinksChanged(aProvider) { + this._populateProviderCache( + aProvider, + () => { + AllPages.update(null, "links-changed"); + }, + true + ); + }, + + _indexOf: function Links__indexOf(aArray, aLink) { + return this._binsearch(aArray, aLink, "indexOf"); + }, + + _insertionIndexOf: function Links__insertionIndexOf(aArray, aLink) { + return this._binsearch(aArray, aLink, "insertionIndexOf"); + }, + + _binsearch: function Links__binsearch(aArray, aLink, aMethod) { + return lazy.BinarySearch[aMethod](this.compareLinks, aArray, aLink); + }, + + /** + * Implements the nsIObserver interface to get notified about browser history + * sanitization. + */ + observe: function Links_observe(aSubject, aTopic, aData) { + // Make sure to update open about:newtab instances. If there are no opened + // pages we can just wait for the next new tab to populate the cache again. + if (AllPages.length && AllPages.enabled) { + this.populateCache(function () { + AllPages.update(); + }, true); + } else { + this.resetCache(); + } + }, + + _callObservers(methodName, ...args) { + for (let obs of this._observers) { + if (typeof obs[methodName] == "function") { + try { + obs[methodName](this, ...args); + } catch (err) { + console.error(err); + } + } + } + }, + + /** + * Adds a sanitization observer and turns itself into a no-op after the first + * invokation. + */ + _addObserver: function Links_addObserver() { + Services.obs.addObserver(this, "browser:purge-session-history", true); + this._addObserver = function () {}; + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), +}; + +Links.compareLinks = Links.compareLinks.bind(Links); + +/** + * Singleton used to collect telemetry data. + * + */ +var Telemetry = { + /** + * Initializes object. + */ + init: function Telemetry_init() { + Services.obs.addObserver(this, TOPIC_GATHER_TELEMETRY); + }, + + uninit: function Telemetry_uninit() { + Services.obs.removeObserver(this, TOPIC_GATHER_TELEMETRY); + }, + + /** + * Collects data. + */ + _collect: function Telemetry_collect() { + let probes = [ + { histogram: "NEWTAB_PAGE_ENABLED", value: AllPages.enabled }, + { + histogram: "NEWTAB_PAGE_PINNED_SITES_COUNT", + value: PinnedLinks.links.length, + }, + { + histogram: "NEWTAB_PAGE_BLOCKED_SITES_COUNT", + value: Object.keys(BlockedLinks.links).length, + }, + ]; + + probes.forEach(function Telemetry_collect_forEach(aProbe) { + Services.telemetry.getHistogramById(aProbe.histogram).add(aProbe.value); + }); + }, + + /** + * Listens for gather telemetry topic. + */ + observe: function Telemetry_observe(aSubject, aTopic, aData) { + this._collect(); + }, +}; + +/** + * Singleton that checks if a given link should be displayed on about:newtab + * or if we should rather not do it for security reasons. URIs that inherit + * their caller's principal will be filtered. + */ +var LinkChecker = { + _cache: {}, + + get flags() { + return ( + Ci.nsIScriptSecurityManager.DISALLOW_INHERIT_PRINCIPAL | + Ci.nsIScriptSecurityManager.DONT_REPORT_ERRORS + ); + }, + + checkLoadURI: function LinkChecker_checkLoadURI(aURI) { + if (!(aURI in this._cache)) { + this._cache[aURI] = this._doCheckLoadURI(aURI); + } + + return this._cache[aURI]; + }, + + _doCheckLoadURI: function Links_doCheckLoadURI(aURI) { + try { + // about:newtab is currently privileged. In any case, it should be + // possible for tiles to point to pretty much everything - but not + // to stuff that inherits the system principal, so we check: + let systemPrincipal = Services.scriptSecurityManager.getSystemPrincipal(); + Services.scriptSecurityManager.checkLoadURIStrWithPrincipal( + systemPrincipal, + aURI, + this.flags + ); + return true; + } catch (e) { + // We got a weird URI or one that would inherit the caller's principal. + return false; + } + }, +}; + +var ExpirationFilter = { + init: function ExpirationFilter_init() { + lazy.PageThumbs.addExpirationFilter(this); + }, + + filterForThumbnailExpiration: + function ExpirationFilter_filterForThumbnailExpiration(aCallback) { + if (!AllPages.enabled) { + aCallback([]); + return; + } + + Links.populateCache(function () { + let urls = []; + + // Add all URLs to the list that we want to keep thumbnails for. + for (let link of Links.getLinks().slice(0, 25)) { + if (link && link.url) { + urls.push(link.url); + } + } + + aCallback(urls); + }); + }, +}; + +/** + * Singleton that provides the public API of this JSM. + */ +export var NewTabUtils = { + _initialized: false, + + /** + * Extract a "site" from a url in a way that multiple urls of a "site" returns + * the same "site." + * @param aUrl Url spec string + * @return The "site" string or null + */ + extractSite: function Links_extractSite(url) { + let host; + try { + // Note that nsIURI.asciiHost throws NS_ERROR_FAILURE for some types of + // URIs, including jar and moz-icon URIs. + host = Services.io.newURI(url).asciiHost; + } catch (ex) { + return null; + } + + // Strip off common subdomains of the same site (e.g., www, load balancer) + return host.replace(/^(m|mobile|www\d*)\./, ""); + }, + + init: function NewTabUtils_init() { + if (this.initWithoutProviders()) { + PlacesProvider.init(); + Links.addProvider(PlacesProvider); + BlockedLinks.addObserver(Links); + BlockedLinks.addObserver(ActivityStreamLinks); + } + }, + + initWithoutProviders: function NewTabUtils_initWithoutProviders() { + if (!this._initialized) { + this._initialized = true; + ExpirationFilter.init(); + Telemetry.init(); + return true; + } + return false; + }, + + uninit: function NewTabUtils_uninit() { + if (this.initialized) { + Telemetry.uninit(); + BlockedLinks.removeObservers(); + } + }, + + getProviderLinks(aProvider) { + let cache = Links._providers.get(aProvider); + if (cache && cache.sortedLinks) { + return cache.sortedLinks; + } + return []; + }, + + isTopSiteGivenProvider(aSite, aProvider) { + let cache = Links._providers.get(aProvider); + if (cache && cache.siteMap) { + return cache.siteMap.has(aSite); + } + return false; + }, + + isTopPlacesSite(aSite) { + return this.isTopSiteGivenProvider(aSite, PlacesProvider); + }, + + /** + * Restores all sites that have been removed from the grid. + */ + restore: function NewTabUtils_restore() { + lazy.Storage.clear(); + Links.resetCache(); + PinnedLinks.resetCache(); + BlockedLinks.resetCache(); + + Links.populateCache(function () { + AllPages.update(); + }, true); + }, + + /** + * Undoes all sites that have been removed from the grid and keep the pinned + * tabs. + * @param aCallback the callback method. + */ + undoAll: function NewTabUtils_undoAll(aCallback) { + lazy.Storage.remove("blockedLinks"); + Links.resetCache(); + BlockedLinks.resetCache(); + Links.populateCache(aCallback, true); + }, + + links: Links, + allPages: AllPages, + pinnedLinks: PinnedLinks, + blockedLinks: BlockedLinks, + activityStreamLinks: ActivityStreamLinks, + activityStreamProvider: ActivityStreamProvider, +}; |