Adding upstream version 115.8.0esr.upstream/115.8.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 2363 insertions, 0 deletions

diff --git a/toolkit/modules/NewTabUtils.sys.mjs b/toolkit/modules/NewTabUtils.sys.mjs
new file mode 100644
index 0000000000..55d7d75ee5
--- /dev/null
+++ b/toolkit/modules/NewTabUtils.sys.mjs
@@ -0,0 +1,2363 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+// Android tests don't import these properly, so guard against that
+let shortURL = {};
+let searchShortcuts = {};
+let didSuccessfulImport = false;
+try {
+  shortURL = ChromeUtils.import("resource://activity-stream/lib/ShortURL.jsm");
+  searchShortcuts = ChromeUtils.importESModule(
+    "resource://activity-stream/lib/SearchShortcuts.sys.mjs"
+  );
+  didSuccessfulImport = true;
+} catch (e) {
+  // The test failed to import these files
+}
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  BinarySearch: "resource://gre/modules/BinarySearch.sys.mjs",
+  PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs",
+  PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+  Pocket: "chrome://pocket/content/Pocket.sys.mjs",
+  pktApi: "chrome://pocket/content/pktApi.sys.mjs",
+});
+
+let BrowserWindowTracker;
+try {
+  BrowserWindowTracker = ChromeUtils.import(
+    "resource:///modules/BrowserWindowTracker.jsm"
+  ).BrowserWindowTracker;
+} catch (e) {
+  // BrowserWindowTracker is used to determine devicePixelRatio in
+  // _addFavicons. We fallback to the value 2 if we can't find a window,
+  // so it's safe to do nothing with this here.
+}
+
+XPCOMUtils.defineLazyGetter(lazy, "gCryptoHash", function () {
+  return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
+});
+
+// Boolean preferences that control newtab content
+const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled";
+
+// The maximum number of results PlacesProvider retrieves from history.
+const HISTORY_RESULTS_LIMIT = 100;
+
+// The maximum number of links Links.getLinks will return.
+const LINKS_GET_LINKS_LIMIT = 100;
+
+// The gather telemetry topic.
+const TOPIC_GATHER_TELEMETRY = "gather-telemetry";
+
+// Some default frecency threshold for Activity Stream requests
+const ACTIVITY_STREAM_DEFAULT_FRECENCY = 150;
+
+// Some default query limit for Activity Stream requests
+const ACTIVITY_STREAM_DEFAULT_LIMIT = 12;
+
+// Some default seconds ago for Activity Stream recent requests
+const ACTIVITY_STREAM_DEFAULT_RECENT = 5 * 24 * 60 * 60;
+
+// The fallback value for the width of smallFavicon in pixels.
+// This value will be multiplied by the current window's devicePixelRatio.
+// If devicePixelRatio cannot be found, it will be multiplied by 2.
+const DEFAULT_SMALL_FAVICON_WIDTH = 16;
+
+const POCKET_UPDATE_TIME = 24 * 60 * 60 * 1000; // 1 day
+const POCKET_INACTIVE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week
+const PREF_POCKET_LATEST_SINCE = "extensions.pocket.settings.latestSince";
+
+/**
+ * Calculate the MD5 hash for a string.
+ * @param aValue
+ *        The string to convert.
+ * @return The base64 representation of the MD5 hash.
+ */
+function toHash(aValue) {
+  let value = new TextEncoder().encode(aValue);
+  lazy.gCryptoHash.init(lazy.gCryptoHash.MD5);
+  lazy.gCryptoHash.update(value, value.length);
+  return lazy.gCryptoHash.finish(true);
+}
+
+/**
+ * Singleton that provides storage functionality.
+ */
+XPCOMUtils.defineLazyGetter(lazy, "Storage", function () {
+  return new LinksStorage();
+});
+
+function LinksStorage() {
+  // Handle migration of data across versions.
+  try {
+    if (this._storedVersion < this._version) {
+      // This is either an upgrade, or version information is missing.
+      if (this._storedVersion < 1) {
+        // Version 1 moved data from DOM Storage to prefs.  Since migrating from
+        // version 0 is no more supported, we just reportError a dataloss later.
+        throw new Error("Unsupported newTab storage version");
+      }
+      // Add further migration steps here.
+    } else {
+      // This is a downgrade.  Since we cannot predict future, upgrades should
+      // be backwards compatible.  We will set the version to the old value
+      // regardless, so, on next upgrade, the migration steps will run again.
+      // For this reason, they should also be able to run multiple times, even
+      // on top of an already up-to-date storage.
+    }
+  } catch (ex) {
+    // Something went wrong in the update process, we can't recover from here,
+    // so just clear the storage and start from scratch (dataloss!).
+    console.error(
+      "Unable to migrate the newTab storage to the current version. " +
+        "Restarting from scratch.\n",
+      ex
+    );
+    this.clear();
+  }
+
+  // Set the version to the current one.
+  this._storedVersion = this._version;
+}
+
+LinksStorage.prototype = {
+  get _version() {
+    return 1;
+  },
+
+  get _prefs() {
+    return Object.freeze({
+      pinnedLinks: "browser.newtabpage.pinned",
+      blockedLinks: "browser.newtabpage.blocked",
+    });
+  },
+
+  get _storedVersion() {
+    if (this.__storedVersion === undefined) {
+      // When the pref is not set, the storage version is unknown, so either:
+      // - it's a new profile
+      // - it's a profile where versioning information got lost
+      // In this case we still run through all of the valid migrations,
+      // starting from 1, as if it was a downgrade.  As previously stated the
+      // migrations should already support running on an updated store.
+      this.__storedVersion = Services.prefs.getIntPref(
+        "browser.newtabpage.storageVersion",
+        1
+      );
+    }
+    return this.__storedVersion;
+  },
+  set _storedVersion(aValue) {
+    Services.prefs.setIntPref("browser.newtabpage.storageVersion", aValue);
+    this.__storedVersion = aValue;
+  },
+
+  /**
+   * Gets the value for a given key from the storage.
+   * @param aKey The storage key (a string).
+   * @param aDefault A default value if the key doesn't exist.
+   * @return The value for the given key.
+   */
+  get: function Storage_get(aKey, aDefault) {
+    let value;
+    try {
+      let prefValue = Services.prefs.getStringPref(this._prefs[aKey]);
+      value = JSON.parse(prefValue);
+    } catch (e) {}
+    return value || aDefault;
+  },
+
+  /**
+   * Sets the storage value for a given key.
+   * @param aKey The storage key (a string).
+   * @param aValue The value to set.
+   */
+  set: function Storage_set(aKey, aValue) {
+    // Page titles may contain unicode, thus use complex values.
+    Services.prefs.setStringPref(this._prefs[aKey], JSON.stringify(aValue));
+  },
+
+  /**
+   * Removes the storage value for a given key.
+   * @param aKey The storage key (a string).
+   */
+  remove: function Storage_remove(aKey) {
+    Services.prefs.clearUserPref(this._prefs[aKey]);
+  },
+
+  /**
+   * Clears the storage and removes all values.
+   */
+  clear: function Storage_clear() {
+    for (let key in this._prefs) {
+      this.remove(key);
+    }
+  },
+};
+
+/**
+ * Singleton that serves as a registry for all open 'New Tab Page's.
+ */
+var AllPages = {
+  /**
+   * The array containing all active pages.
+   */
+  _pages: [],
+
+  /**
+   * Cached value that tells whether the New Tab Page feature is enabled.
+   */
+  _enabled: null,
+
+  /**
+   * Adds a page to the internal list of pages.
+   * @param aPage The page to register.
+   */
+  register: function AllPages_register(aPage) {
+    this._pages.push(aPage);
+    this._addObserver();
+  },
+
+  /**
+   * Removes a page from the internal list of pages.
+   * @param aPage The page to unregister.
+   */
+  unregister: function AllPages_unregister(aPage) {
+    let index = this._pages.indexOf(aPage);
+    if (index > -1) {
+      this._pages.splice(index, 1);
+    }
+  },
+
+  /**
+   * Returns whether the 'New Tab Page' is enabled.
+   */
+  get enabled() {
+    if (this._enabled === null) {
+      this._enabled = Services.prefs.getBoolPref(PREF_NEWTAB_ENABLED);
+    }
+
+    return this._enabled;
+  },
+
+  /**
+   * Enables or disables the 'New Tab Page' feature.
+   */
+  set enabled(aEnabled) {
+    if (this.enabled != aEnabled) {
+      Services.prefs.setBoolPref(PREF_NEWTAB_ENABLED, !!aEnabled);
+    }
+  },
+
+  /**
+   * Returns the number of registered New Tab Pages (i.e. the number of open
+   * about:newtab instances).
+   */
+  get length() {
+    return this._pages.length;
+  },
+
+  /**
+   * Updates all currently active pages but the given one.
+   * @param aExceptPage The page to exclude from updating.
+   * @param aReason The reason for updating all pages.
+   */
+  update(aExceptPage, aReason = "") {
+    for (let page of this._pages.slice()) {
+      if (aExceptPage != page) {
+        page.update(aReason);
+      }
+    }
+  },
+
+  /**
+   * Implements the nsIObserver interface to get notified when the preference
+   * value changes or when a new copy of a page thumbnail is available.
+   */
+  observe: function AllPages_observe(aSubject, aTopic, aData) {
+    if (aTopic == "nsPref:changed") {
+      // Clear the cached value.
+      switch (aData) {
+        case PREF_NEWTAB_ENABLED:
+          this._enabled = null;
+          break;
+      }
+    }
+    // and all notifications get forwarded to each page.
+    this._pages.forEach(function (aPage) {
+      aPage.observe(aSubject, aTopic, aData);
+    }, this);
+  },
+
+  /**
+   * Adds a preference and new thumbnail observer and turns itself into a
+   * no-op after the first invokation.
+   */
+  _addObserver: function AllPages_addObserver() {
+    Services.prefs.addObserver(PREF_NEWTAB_ENABLED, this, true);
+    Services.obs.addObserver(this, "page-thumbnail:create", true);
+    this._addObserver = function () {};
+  },
+
+  QueryInterface: ChromeUtils.generateQI([
+    "nsIObserver",
+    "nsISupportsWeakReference",
+  ]),
+};
+
+/**
+ * Singleton that keeps track of all pinned links and their positions in the
+ * grid.
+ */
+var PinnedLinks = {
+  /**
+   * The cached list of pinned links.
+   */
+  _links: null,
+
+  /**
+   * The array of pinned links.
+   */
+  get links() {
+    if (!this._links) {
+      this._links = lazy.Storage.get("pinnedLinks", []);
+    }
+
+    return this._links;
+  },
+
+  /**
+   * Pins a link at the given position.
+   * @param aLink The link to pin.
+   * @param aIndex The grid index to pin the cell at.
+   * @return true if link changes, false otherwise
+   */
+  pin: function PinnedLinks_pin(aLink, aIndex) {
+    // Clear the link's old position, if any.
+    this.unpin(aLink);
+
+    // change pinned link into a history link
+    let changed = this._makeHistoryLink(aLink);
+    this.links[aIndex] = aLink;
+    this.save();
+    return changed;
+  },
+
+  /**
+   * Unpins a given link.
+   * @param aLink The link to unpin.
+   */
+  unpin: function PinnedLinks_unpin(aLink) {
+    let index = this._indexOfLink(aLink);
+    if (index == -1) {
+      return;
+    }
+    let links = this.links;
+    links[index] = null;
+    // trim trailing nulls
+    let i = links.length - 1;
+    while (i >= 0 && links[i] == null) {
+      i--;
+    }
+    links.splice(i + 1);
+    this.save();
+  },
+
+  /**
+   * Saves the current list of pinned links.
+   */
+  save: function PinnedLinks_save() {
+    lazy.Storage.set("pinnedLinks", this.links);
+  },
+
+  /**
+   * Checks whether a given link is pinned.
+   * @params aLink The link to check.
+   * @return whether The link is pinned.
+   */
+  isPinned: function PinnedLinks_isPinned(aLink) {
+    return this._indexOfLink(aLink) != -1;
+  },
+
+  /**
+   * Resets the links cache.
+   */
+  resetCache: function PinnedLinks_resetCache() {
+    this._links = null;
+  },
+
+  /**
+   * Finds the index of a given link in the list of pinned links.
+   * @param aLink The link to find an index for.
+   * @return The link's index.
+   */
+  _indexOfLink: function PinnedLinks_indexOfLink(aLink) {
+    for (let i = 0; i < this.links.length; i++) {
+      let link = this.links[i];
+      if (link && link.url == aLink.url) {
+        return i;
+      }
+    }
+
+    // The given link is unpinned.
+    return -1;
+  },
+
+  /**
+   * Transforms link into a "history" link
+   * @param aLink The link to change
+   * @return true if link changes, false otherwise
+   */
+  _makeHistoryLink: function PinnedLinks_makeHistoryLink(aLink) {
+    if (!aLink.type || aLink.type == "history") {
+      return false;
+    }
+    aLink.type = "history";
+    return true;
+  },
+
+  /**
+   * Replaces existing link with another link.
+   * @param aUrl The url of existing link
+   * @param aLink The replacement link
+   */
+  replace: function PinnedLinks_replace(aUrl, aLink) {
+    let index = this._indexOfLink({ url: aUrl });
+    if (index == -1) {
+      return;
+    }
+    this.links[index] = aLink;
+    this.save();
+  },
+};
+
+/**
+ * Singleton that keeps track of all blocked links in the grid.
+ */
+var BlockedLinks = {
+  /**
+   * A list of objects that are observing blocked link changes.
+   */
+  _observers: [],
+
+  /**
+   * The cached list of blocked links.
+   */
+  _links: null,
+
+  /**
+   * Registers an object that will be notified when the blocked links change.
+   */
+  addObserver(aObserver) {
+    this._observers.push(aObserver);
+  },
+
+  /**
+   * Remove the observers.
+   */
+  removeObservers() {
+    this._observers = [];
+  },
+
+  /**
+   * The list of blocked links.
+   */
+  get links() {
+    if (!this._links) {
+      this._links = lazy.Storage.get("blockedLinks", {});
+    }
+
+    return this._links;
+  },
+
+  /**
+   * Blocks a given link. Adjusts siteMap accordingly, and notifies listeners.
+   * @param aLink The link to block.
+   */
+  block: function BlockedLinks_block(aLink) {
+    this._callObservers("onLinkBlocked", aLink);
+    this.links[toHash(aLink.url)] = 1;
+    this.save();
+
+    // Make sure we unpin blocked links.
+    PinnedLinks.unpin(aLink);
+  },
+
+  /**
+   * Unblocks a given link. Adjusts siteMap accordingly, and notifies listeners.
+   * @param aLink The link to unblock.
+   */
+  unblock: function BlockedLinks_unblock(aLink) {
+    if (this.isBlocked(aLink)) {
+      delete this.links[toHash(aLink.url)];
+      this.save();
+      this._callObservers("onLinkUnblocked", aLink);
+    }
+  },
+
+  /**
+   * Saves the current list of blocked links.
+   */
+  save: function BlockedLinks_save() {
+    lazy.Storage.set("blockedLinks", this.links);
+  },
+
+  /**
+   * Returns whether a given link is blocked.
+   * @param aLink The link to check.
+   */
+  isBlocked: function BlockedLinks_isBlocked(aLink) {
+    return toHash(aLink.url) in this.links;
+  },
+
+  /**
+   * Checks whether the list of blocked links is empty.
+   * @return Whether the list is empty.
+   */
+  isEmpty: function BlockedLinks_isEmpty() {
+    return !Object.keys(this.links).length;
+  },
+
+  /**
+   * Resets the links cache.
+   */
+  resetCache: function BlockedLinks_resetCache() {
+    this._links = null;
+  },
+
+  _callObservers(methodName, ...args) {
+    for (let obs of this._observers) {
+      if (typeof obs[methodName] == "function") {
+        try {
+          obs[methodName](...args);
+        } catch (err) {
+          console.error(err);
+        }
+      }
+    }
+  },
+};
+
+/**
+ * Singleton that serves as the default link provider for the grid. It queries
+ * the history to retrieve the most frequently visited sites.
+ */
+var PlacesProvider = {
+  /**
+   * Set this to change the maximum number of links the provider will provide.
+   */
+  maxNumLinks: HISTORY_RESULTS_LIMIT,
+
+  /**
+   * Must be called before the provider is used.
+   */
+  init: function PlacesProvider_init() {
+    this._placesObserver = new PlacesWeakCallbackWrapper(
+      this.handlePlacesEvents.bind(this)
+    );
+    PlacesObservers.addListener(
+      ["page-visited", "page-title-changed", "pages-rank-changed"],
+      this._placesObserver
+    );
+  },
+
+  /**
+   * Gets the current set of links delivered by this provider.
+   * @param aCallback The function that the array of links is passed to.
+   */
+  getLinks: function PlacesProvider_getLinks(aCallback) {
+    let options = lazy.PlacesUtils.history.getNewQueryOptions();
+    options.maxResults = this.maxNumLinks;
+
+    // Sort by frecency, descending.
+    options.sortingMode =
+      Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_DESCENDING;
+
+    let links = [];
+
+    let callback = {
+      handleResult(aResultSet) {
+        let row;
+
+        while ((row = aResultSet.getNextRow())) {
+          let url = row.getResultByIndex(1);
+          if (LinkChecker.checkLoadURI(url)) {
+            let title = row.getResultByIndex(2);
+            let frecency = row.getResultByIndex(12);
+            let lastVisitDate = row.getResultByIndex(5);
+            links.push({
+              url,
+              title,
+              frecency,
+              lastVisitDate,
+              type: "history",
+            });
+          }
+        }
+      },
+
+      handleError(aError) {
+        // Should we somehow handle this error?
+        aCallback([]);
+      },
+
+      handleCompletion(aReason) {
+        // The Places query breaks ties in frecency by place ID descending, but
+        // that's different from how Links.compareLinks breaks ties, because
+        // compareLinks doesn't have access to place IDs.  It's very important
+        // that the initial list of links is sorted in the same order imposed by
+        // compareLinks, because Links uses compareLinks to perform binary
+        // searches on the list.  So, ensure the list is so ordered.
+        let i = 1;
+        let outOfOrder = [];
+        while (i < links.length) {
+          if (Links.compareLinks(links[i - 1], links[i]) > 0) {
+            outOfOrder.push(links.splice(i, 1)[0]);
+          } else {
+            i++;
+          }
+        }
+        for (let link of outOfOrder) {
+          i = lazy.BinarySearch.insertionIndexOf(
+            Links.compareLinks,
+            links,
+            link
+          );
+          links.splice(i, 0, link);
+        }
+
+        aCallback(links);
+      },
+    };
+
+    // Execute the query.
+    let query = lazy.PlacesUtils.history.getNewQuery();
+    lazy.PlacesUtils.history.asyncExecuteLegacyQuery(query, options, callback);
+  },
+
+  /**
+   * Registers an object that will be notified when the provider's links change.
+   * @param aObserver An object with the following optional properties:
+   *        * onLinkChanged: A function that's called when a single link
+   *          changes.  It's passed the provider and the link object.  Only the
+   *          link's `url` property is guaranteed to be present.  If its `title`
+   *          property is present, then its title has changed, and the
+   *          property's value is the new title.  If any sort properties are
+   *          present, then its position within the provider's list of links may
+   *          have changed, and the properties' values are the new sort-related
+   *          values.  Note that this link may not necessarily have been present
+   *          in the lists returned from any previous calls to getLinks.
+   *        * onManyLinksChanged: A function that's called when many links
+   *          change at once.  It's passed the provider.  You should call
+   *          getLinks to get the provider's new list of links.
+   */
+  addObserver: function PlacesProvider_addObserver(aObserver) {
+    this._observers.push(aObserver);
+  },
+
+  _observers: [],
+
+  handlePlacesEvents(aEvents) {
+    for (let event of aEvents) {
+      switch (event.type) {
+        case "page-visited": {
+          if (event.visitCount == 1 && event.lastKnownTitle) {
+            this._callObservers("onLinkChanged", {
+              url: event.url,
+              title: event.lastKnownTitle,
+            });
+          }
+          break;
+        }
+        case "page-title-changed": {
+          this._callObservers("onLinkChanged", {
+            url: event.url,
+            title: event.title,
+          });
+          break;
+        }
+        case "pages-rank-changed": {
+          this._callObservers("onManyLinksChanged");
+          break;
+        }
+      }
+    }
+  },
+
+  _callObservers: function PlacesProvider__callObservers(aMethodName, aArg) {
+    for (let obs of this._observers) {
+      if (obs[aMethodName]) {
+        try {
+          obs[aMethodName](this, aArg);
+        } catch (err) {
+          console.error(err);
+        }
+      }
+    }
+  },
+};
+
+/**
+ * Queries history to retrieve the most frecent sites. Emits events when the
+ * history changes.
+ */
+var ActivityStreamProvider = {
+  THUMB_FAVICON_SIZE: 96,
+
+  /**
+   * Shared adjustment for selecting potentially blocked links.
+   */
+  _adjustLimitForBlocked({ ignoreBlocked, numItems }) {
+    // Just use the usual number if blocked links won't be filtered out
+    if (ignoreBlocked) {
+      return numItems;
+    }
+    // Additionally select the number of blocked links in case they're removed
+    return Object.keys(BlockedLinks.links).length + numItems;
+  },
+
+  /**
+   * Shared sub-SELECT to get the guid of a bookmark of the current url while
+   * avoiding LEFT JOINs on moz_bookmarks. This avoids gettings tags. The guid
+   * could be one of multiple possible guids. Assumes `moz_places h` is in FROM.
+   */
+  _commonBookmarkGuidSelect: `(
+    SELECT guid
+    FROM moz_bookmarks b
+    WHERE fk = h.id
+      AND type = :bookmarkType
+      AND (
+        SELECT id
+        FROM moz_bookmarks p
+        WHERE p.id = b.parent
+          AND p.parent <> :tagsFolderId
+      ) NOTNULL
+    ) AS bookmarkGuid`,
+
+  /**
+   * Shared WHERE expression filtering out undesired pages, e.g., hidden,
+   * unvisited, and non-http/s urls. Assumes moz_places is in FROM / JOIN.
+   *
+   * NB: SUBSTR(url) is used even without an index instead of url_hash because
+   * most desired pages will match http/s, so it will only run on the ~10s of
+   * rows matched. If url_hash were to be used, it should probably *not* be used
+   * by the query optimizer as we primarily want it optimized for the other
+   * conditions, e.g., most frecent first.
+   */
+  _commonPlacesWhere: `
+    AND hidden = 0
+    AND last_visit_date > 0
+    AND (SUBSTR(url, 1, 6) == "https:"
+      OR SUBSTR(url, 1, 5) == "http:")
+  `,
+
+  /**
+   * Shared parameters for getting correct bookmarks and LIMITed queries.
+   */
+  _getCommonParams(aOptions, aParams = {}) {
+    return Object.assign(
+      {
+        bookmarkType: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        limit: this._adjustLimitForBlocked(aOptions),
+        tagsFolderId: lazy.PlacesUtils.tagsFolderId,
+      },
+      aParams
+    );
+  },
+
+  /**
+   * Shared columns for Highlights related queries.
+   */
+  _highlightsColumns: [
+    "bookmarkGuid",
+    "description",
+    "guid",
+    "preview_image_url",
+    "title",
+    "url",
+  ],
+
+  /**
+   * Shared post-processing of Highlights links.
+   */
+  _processHighlights(aLinks, aOptions, aType) {
+    // Filter out blocked if necessary
+    if (!aOptions.ignoreBlocked) {
+      aLinks = aLinks.filter(
+        link =>
+          !BlockedLinks.isBlocked(
+            link.pocket_id ? { url: link.open_url } : link
+          )
+      );
+    }
+
+    // Limit the results to the requested number and set a type corresponding to
+    // which query selected it
+    return aLinks.slice(0, aOptions.numItems).map(item =>
+      Object.assign(item, {
+        type: aType,
+      })
+    );
+  },
+
+  /**
+   * From an Array of links, if favicons are present, convert to data URIs
+   *
+   * @param {Array} aLinks
+   *          an array containing objects with favicon data and mimeTypes
+   *
+   * @returns {Array} an array of links with favicons as data uri
+   */
+  _faviconBytesToDataURI(aLinks) {
+    return aLinks.map(link => {
+      if (link.favicon) {
+        let encodedData = btoa(String.fromCharCode.apply(null, link.favicon));
+        link.favicon = `data:${link.mimeType};base64,${encodedData}`;
+        delete link.mimeType;
+      }
+
+      if (link.smallFavicon) {
+        let encodedData = btoa(
+          String.fromCharCode.apply(null, link.smallFavicon)
+        );
+        link.smallFavicon = `data:${link.smallFaviconMimeType};base64,${encodedData}`;
+        delete link.smallFaviconMimeType;
+      }
+
+      return link;
+    });
+  },
+
+  /**
+   * Get favicon data (and metadata) for a uri. Fetches both the largest favicon
+   * available, for Activity Stream; and a normal-sized favicon, for the Urlbar.
+   *
+   * @param {nsIURI} aUri Page to check for favicon data
+   * @param {number} preferredFaviconWidth
+   *   The preferred width of the of the normal-sized favicon in pixels.
+   * @returns A promise of an object (possibly empty) containing the data.
+   */
+  async _loadIcons(aUri, preferredFaviconWidth) {
+    let iconData = {};
+    // Fetch the largest icon available.
+    let faviconData;
+    try {
+      faviconData = await lazy.PlacesUtils.promiseFaviconData(
+        aUri,
+        this.THUMB_FAVICON_SIZE
+      );
+      Object.assign(iconData, {
+        favicon: faviconData.data,
+        faviconLength: faviconData.dataLen,
+        faviconRef: faviconData.uri.ref,
+        faviconSize: faviconData.size,
+        mimeType: faviconData.mimeType,
+      });
+    } catch (e) {
+      // Return early because fetching the largest favicon is the primary
+      // purpose of NewTabUtils.
+      return null;
+    }
+
+    // Also fetch a smaller icon.
+    try {
+      faviconData = await lazy.PlacesUtils.promiseFaviconData(
+        aUri,
+        preferredFaviconWidth
+      );
+      Object.assign(iconData, {
+        smallFavicon: faviconData.data,
+        smallFaviconLength: faviconData.dataLen,
+        smallFaviconRef: faviconData.uri.ref,
+        smallFaviconSize: faviconData.size,
+        smallFaviconMimeType: faviconData.mimeType,
+      });
+    } catch (e) {
+      // Do nothing with the error since we still have the large favicon fields.
+    }
+
+    return iconData;
+  },
+
+  /**
+   * Computes favicon data for each url in a set of links
+   *
+   * @param {Array} links
+   *          an array containing objects without favicon data or mimeTypes yet
+   *
+   * @returns {Promise} Returns a promise with the array of links with the largest
+   *                    favicon available (as a byte array), mimeType, byte array
+   *                    length, and favicon size (width)
+   */
+  _addFavicons(aLinks) {
+    let win;
+    if (BrowserWindowTracker) {
+      win = BrowserWindowTracker.getTopWindow();
+    }
+    // We fetch two copies of a page's favicon: the largest available, for
+    // Activity Stream; and a smaller size appropriate for the Urlbar.
+    const preferredFaviconWidth =
+      DEFAULT_SMALL_FAVICON_WIDTH * (win ? win.devicePixelRatio : 2);
+    // Each link in the array needs a favicon for it's page - so we fire off a
+    // promise for each link to compute the favicon data and attach it back to
+    // the original link object. We must wait until all favicons for the array
+    // of links are computed before returning
+    return Promise.all(
+      aLinks.map(
+        link =>
+          // eslint-disable-next-line no-async-promise-executor
+          new Promise(async resolve => {
+            // Never add favicon data for pocket items
+            if (link.type === "pocket") {
+              resolve(link);
+              return;
+            }
+            let iconData;
+            try {
+              let linkUri = Services.io.newURI(link.url);
+              iconData = await this._loadIcons(linkUri, preferredFaviconWidth);
+
+              // Switch the scheme to try again with the other
+              if (!iconData) {
+                linkUri = linkUri
+                  .mutate()
+                  .setScheme(linkUri.scheme === "https" ? "http" : "https")
+                  .finalize();
+                iconData = await this._loadIcons(
+                  linkUri,
+                  preferredFaviconWidth
+                );
+              }
+            } catch (e) {
+              // We just won't put icon data on the link
+            }
+
+            // Add the icon data to the link if we have any
+            resolve(Object.assign(link, iconData));
+          })
+      )
+    );
+  },
+
+  /**
+   * Helper function which makes the call to the Pocket API to fetch the user's
+   * saved Pocket items.
+   */
+  fetchSavedPocketItems(requestData) {
+    const latestSince =
+      Services.prefs.getStringPref(PREF_POCKET_LATEST_SINCE, 0) * 1000;
+
+    // Do not fetch Pocket items for users that have been inactive for too long, or are not logged in
+    if (
+      !lazy.pktApi.isUserLoggedIn() ||
+      Date.now() - latestSince > POCKET_INACTIVE_TIME
+    ) {
+      return Promise.resolve(null);
+    }
+
+    return new Promise((resolve, reject) => {
+      lazy.pktApi.retrieve(requestData, {
+        success(data) {
+          resolve(data);
+        },
+        error(error) {
+          reject(error);
+        },
+      });
+    });
+  },
+
+  /**
+   * Get the most recently Pocket-ed items from a user's Pocket list. See:
+   * https://getpocket.com/developer/docs/v3/retrieve for details
+   *
+   * @param {Object} aOptions
+   *   {int} numItems: The max number of pocket items to fetch
+   */
+  async getRecentlyPocketed(aOptions) {
+    const pocketSecondsAgo =
+      Math.floor(Date.now() / 1000) - ACTIVITY_STREAM_DEFAULT_RECENT;
+    const requestData = {
+      detailType: "complete",
+      count: aOptions.numItems,
+      since: pocketSecondsAgo,
+    };
+    let data;
+    try {
+      data = await this.fetchSavedPocketItems(requestData);
+      if (!data) {
+        return [];
+      }
+    } catch (e) {
+      console.error(e);
+      return [];
+    }
+    /* Extract relevant parts needed to show this card as a highlight:
+     * url, preview image, title, description, and the unique item_id
+     * necessary for Pocket to identify the item
+     */
+    let items = Object.values(data.list)
+      // status "0" means not archived or deleted
+      .filter(item => item.status === "0")
+      .map(item => ({
+        date_added: item.time_added * 1000,
+        description: item.excerpt,
+        preview_image_url: item.image && item.image.src,
+        title: item.resolved_title,
+        url: item.resolved_url,
+        pocket_id: item.item_id,
+        open_url: item.open_url,
+      }));
+
+    // Append the query param to let Pocket know this item came from highlights
+    for (let item of items) {
+      let url = new URL(item.open_url);
+      url.searchParams.append("src", "fx_new_tab");
+      item.open_url = url.href;
+    }
+
+    return this._processHighlights(items, aOptions, "pocket");
+  },
+
+  /**
+   * Get most-recently-created visited bookmarks for Activity Stream.
+   *
+   * @param {Object} aOptions
+   *   {num}  bookmarkSecondsAgo: Maximum age of added bookmark.
+   *   {bool} ignoreBlocked: Do not filter out blocked links.
+   *   {int}  numItems: Maximum number of items to return.
+   */
+  async getRecentBookmarks(aOptions) {
+    const options = Object.assign(
+      {
+        bookmarkSecondsAgo: ACTIVITY_STREAM_DEFAULT_RECENT,
+        ignoreBlocked: false,
+        numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
+      },
+      aOptions || {}
+    );
+
+    const sqlQuery = `
+      SELECT
+        b.guid AS bookmarkGuid,
+        description,
+        h.guid,
+        preview_image_url,
+        b.title,
+        b.dateAdded / 1000 AS date_added,
+        url
+      FROM moz_bookmarks b
+      JOIN moz_bookmarks p
+        ON p.id = b.parent
+      JOIN moz_places h
+        ON h.id = b.fk
+      WHERE b.dateAdded >= :dateAddedThreshold
+        AND b.title NOTNULL
+        AND b.type = :bookmarkType
+        AND p.parent <> :tagsFolderId
+        ${this._commonPlacesWhere}
+      ORDER BY b.dateAdded DESC
+      LIMIT :limit
+    `;
+
+    return this._processHighlights(
+      await this.executePlacesQuery(sqlQuery, {
+        columns: [...this._highlightsColumns, "date_added"],
+        params: this._getCommonParams(options, {
+          dateAddedThreshold:
+            (Date.now() - options.bookmarkSecondsAgo * 1000) * 1000,
+        }),
+      }),
+      options,
+      "bookmark"
+    );
+  },
+
+  /**
+   * Get total count of all bookmarks.
+   * Note: this includes default bookmarks
+   *
+   * @return {int} The number bookmarks in the places DB.
+   */
+  async getTotalBookmarksCount() {
+    let sqlQuery = `
+      SELECT count(*) FROM moz_bookmarks b
+      JOIN moz_bookmarks t ON t.id = b.parent
+      AND t.parent <> :tags_folder
+     WHERE b.type = :type_bookmark
+    `;
+
+    const result = await this.executePlacesQuery(sqlQuery, {
+      params: {
+        tags_folder: lazy.PlacesUtils.tagsFolderId,
+        type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      },
+    });
+
+    return result[0][0];
+  },
+
+  /**
+   * Get most-recently-visited history with metadata for Activity Stream.
+   *
+   * @param {Object} aOptions
+   *   {bool} ignoreBlocked: Do not filter out blocked links.
+   *   {int}  numItems: Maximum number of items to return.
+   */
+  async getRecentHistory(aOptions) {
+    const options = Object.assign(
+      {
+        ignoreBlocked: false,
+        numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
+      },
+      aOptions || {}
+    );
+
+    const sqlQuery = `
+      SELECT
+        ${this._commonBookmarkGuidSelect},
+        description,
+        guid,
+        preview_image_url,
+        title,
+        url
+      FROM moz_places h
+      WHERE description NOTNULL
+        AND preview_image_url NOTNULL
+        ${this._commonPlacesWhere}
+      ORDER BY last_visit_date DESC
+      LIMIT :limit
+    `;
+
+    return this._processHighlights(
+      await this.executePlacesQuery(sqlQuery, {
+        columns: this._highlightsColumns,
+        params: this._getCommonParams(options),
+      }),
+      options,
+      "history"
+    );
+  },
+
+  /*
+   * Gets the top frecent sites for Activity Stream.
+   *
+   * @param {Object} aOptions
+   *   {bool} ignoreBlocked: Do not filter out blocked links.
+   *   {int}  numItems: Maximum number of items to return.
+   *   {int}  topsiteFrecency: Minimum amount of frecency for a site.
+   *   {bool} onePerDomain: Dedupe the resulting list.
+   *   {bool} includeFavicon: Include favicons if available.
+   *   {string} hideWithSearchParam: URLs that contain this search param will be
+   *     excluded from the returned links. This value should be either undefined
+   *     or a string with one of the following forms:
+   *     - undefined: Fall back to the value of pref
+   *       `browser.newtabpage.activity-stream.hideTopSitesWithSearchParam`
+   *     - "" (empty) - Disable this feature
+   *     - "key" - Search param named "key" with any or no value
+   *     - "key=" - Search param named "key" with no value
+   *     - "key=value" - Search param named "key" with value "value"
+   *
+   * @returns {Promise} Returns a promise with the array of links as payload.
+   */
+  async getTopFrecentSites(aOptions) {
+    const options = Object.assign(
+      {
+        ignoreBlocked: false,
+        numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
+        topsiteFrecency: ACTIVITY_STREAM_DEFAULT_FRECENCY,
+        onePerDomain: true,
+        includeFavicon: true,
+        hideWithSearchParam: Services.prefs.getCharPref(
+          "browser.newtabpage.activity-stream.hideTopSitesWithSearchParam",
+          ""
+        ),
+      },
+      aOptions || {}
+    );
+
+    // Double the item count in case the host is deduped between with www or
+    // not-www (i.e., 2 hosts) and an extra buffer for multiple pages per host.
+    const origNumItems = options.numItems;
+    if (options.onePerDomain) {
+      options.numItems *= 2 * 10;
+    }
+
+    // Keep this query fast with frecency-indexed lookups (even with excess
+    // rows) and shift the more complex logic to post-processing afterwards
+    const sqlQuery = `
+      SELECT
+        ${this._commonBookmarkGuidSelect},
+        frecency,
+        guid,
+        last_visit_date / 1000 AS lastVisitDate,
+        rev_host,
+        title,
+        url,
+        "history" as type
+      FROM moz_places h
+      WHERE frecency >= :frecencyThreshold
+        ${this._commonPlacesWhere}
+      ORDER BY frecency DESC
+      LIMIT :limit
+    `;
+
+    let links = await this.executePlacesQuery(sqlQuery, {
+      columns: [
+        "bookmarkGuid",
+        "frecency",
+        "guid",
+        "lastVisitDate",
+        "title",
+        "url",
+        "type",
+      ],
+      params: this._getCommonParams(options, {
+        frecencyThreshold: options.topsiteFrecency,
+      }),
+    });
+
+    // Determine if the other link is "better" (larger frecency, more recent,
+    // lexicographically earlier url)
+    function isOtherBetter(link, other) {
+      if (other.frecency === link.frecency) {
+        if (other.lastVisitDate === link.lastVisitDate) {
+          return other.url < link.url;
+        }
+        return other.lastVisitDate > link.lastVisitDate;
+      }
+      return other.frecency > link.frecency;
+    }
+
+    // Update a host Map with the better link
+    function setBetterLink(map, link, hostMatcher, combiner = () => {}) {
+      const host = hostMatcher(link.url)[1];
+      if (map.has(host)) {
+        const other = map.get(host);
+        if (isOtherBetter(link, other)) {
+          link = other;
+        }
+        combiner(link, other);
+      }
+      map.set(host, link);
+    }
+
+    // Convert all links that are supposed to be a seach shortcut to its canonical URL
+    if (
+      didSuccessfulImport &&
+      Services.prefs.getBoolPref(
+        `browser.newtabpage.activity-stream.${searchShortcuts.SEARCH_SHORTCUTS_EXPERIMENT}`
+      )
+    ) {
+      links.forEach(link => {
+        let searchProvider = searchShortcuts.getSearchProvider(
+          shortURL.shortURL(link)
+        );
+        if (searchProvider) {
+          link.url = searchProvider.url;
+        }
+      });
+    }
+
+    // Remove links that contain the hide-with search param.
+    if (options.hideWithSearchParam) {
+      let [key, value] = options.hideWithSearchParam.split("=");
+      links = links.filter(link => {
+        try {
+          let { searchParams } = new URL(link.url);
+          return value === undefined
+            ? !searchParams.has(key)
+            : !searchParams.getAll(key).includes(value);
+        } catch (error) {}
+        return true;
+      });
+    }
+
+    // Remove any blocked links.
+    if (!options.ignoreBlocked) {
+      links = links.filter(link => !BlockedLinks.isBlocked(link));
+    }
+
+    if (options.onePerDomain) {
+      // De-dup the links.
+      const exactHosts = new Map();
+      for (const link of links) {
+        // First we want to find the best link for an exact host
+        setBetterLink(exactHosts, link, url => url.match(/:\/\/([^\/]+)/));
+      }
+
+      // Clean up exact hosts to dedupe as non-www hosts
+      const hosts = new Map();
+      for (const link of exactHosts.values()) {
+        setBetterLink(
+          hosts,
+          link,
+          url => url.match(/:\/\/(?:www\.)?([^\/]+)/),
+          // Combine frecencies when deduping these links
+          (targetLink, otherLink) => {
+            targetLink.frecency = link.frecency + otherLink.frecency;
+          }
+        );
+      }
+
+      links = [...hosts.values()];
+    }
+    // Pick out the top links using the same comparer as before
+    links = links.sort(isOtherBetter).slice(0, origNumItems);
+
+    if (!options.includeFavicon) {
+      return links;
+    }
+    // Get the favicons as data URI for now (until we use the favicon protocol)
+    return this._faviconBytesToDataURI(await this._addFavicons(links));
+  },
+
+  /**
+   * Gets a specific bookmark given some info about it
+   *
+   * @param {Obj} aInfo
+   *          An object with one and only one of the following properties:
+   *            - url
+   *            - guid
+   *            - parentGuid and index
+   */
+  async getBookmark(aInfo) {
+    let bookmark = await lazy.PlacesUtils.bookmarks.fetch(aInfo);
+    if (!bookmark) {
+      return null;
+    }
+    let result = {};
+    result.bookmarkGuid = bookmark.guid;
+    result.bookmarkTitle = bookmark.title;
+    result.lastModified = bookmark.lastModified.getTime();
+    result.url = bookmark.url.href;
+    return result;
+  },
+
+  /**
+   * Count the number of visited urls grouped by day
+   */
+  getUserMonthlyActivity() {
+    let sqlQuery = `
+      SELECT count(*),
+        strftime('%Y-%m-%d', visit_date/1000000.0, 'unixepoch', 'localtime') as date_format
+      FROM moz_historyvisits
+      WHERE visit_date > 0
+      AND visit_date > strftime('%s','now','localtime','start of day','-27 days','utc') * 1000000
+      GROUP BY date_format
+      ORDER BY date_format ASC
+    `;
+
+    return this.executePlacesQuery(sqlQuery);
+  },
+
+  /**
+   * Executes arbitrary query against places database
+   *
+   * @param {String} aQuery
+   *        SQL query to execute
+   * @param {Object} [optional] aOptions
+   *          aOptions.columns - an array of column names. if supplied the return
+   *          items will consists of objects keyed on column names. Otherwise
+   *          array of raw values is returned in the select order
+   *          aOptions.param - an object of SQL binding parameters
+   *
+   * @returns {Promise} Returns a promise with the array of retrieved items
+   */
+  async executePlacesQuery(aQuery, aOptions = {}) {
+    let { columns, params } = aOptions;
+    let items = [];
+    let queryError = null;
+    let conn = await lazy.PlacesUtils.promiseDBConnection();
+    await conn.executeCached(aQuery, params, (aRow, aCancel) => {
+      try {
+        let item = null;
+        // if columns array is given construct an object
+        if (columns && Array.isArray(columns)) {
+          item = {};
+          columns.forEach(column => {
+            item[column] = aRow.getResultByName(column);
+          });
+        } else {
+          // if no columns - make an array of raw values
+          item = [];
+          for (let i = 0; i < aRow.numEntries; i++) {
+            item.push(aRow.getResultByIndex(i));
+          }
+        }
+        items.push(item);
+      } catch (e) {
+        queryError = e;
+        aCancel();
+      }
+    });
+    if (queryError) {
+      throw new Error(queryError);
+    }
+    return items;
+  },
+};
+
+/**
+ * A set of actions which influence what sites shown on the Activity Stream page
+ */
+var ActivityStreamLinks = {
+  _savedPocketStories: null,
+  _pocketLastUpdated: 0,
+  _pocketLastLatest: 0,
+
+  /**
+   * Block a url
+   *
+   * @param {Object} aLink
+   *          The link which contains a URL to add to the block list
+   */
+  blockURL(aLink) {
+    BlockedLinks.block(aLink);
+    // If we're blocking a pocket item, invalidate the cache too
+    if (aLink.pocket_id) {
+      this._savedPocketStories = null;
+    }
+  },
+
+  onLinkBlocked(aLink) {
+    Services.obs.notifyObservers(null, "newtab-linkBlocked", aLink.url);
+  },
+
+  /**
+   * Adds a bookmark and opens up the Bookmark Dialog to show feedback that
+   * the bookmarking action has been successful
+   *
+   * @param {Object} aData
+   *          aData.url The url to bookmark
+   *          aData.title The title of the page to bookmark
+   * @param {Window} aBrowserWindow
+   *          The current browser chrome window
+   *
+   * @returns {Promise} Returns a promise set to an object representing the bookmark
+   */
+  addBookmark(aData, aBrowserWindow) {
+    const { url, title } = aData;
+    return aBrowserWindow.PlacesCommandHook.bookmarkLink(url, title);
+  },
+
+  /**
+   * Removes a bookmark
+   *
+   * @param {String} aBookmarkGuid
+   *          The bookmark guid associated with the bookmark to remove
+   *
+   * @returns {Promise} Returns a promise at completion.
+   */
+  deleteBookmark(aBookmarkGuid) {
+    return lazy.PlacesUtils.bookmarks.remove(aBookmarkGuid);
+  },
+
+  /**
+   * Removes a history link and unpins the URL if previously pinned
+   *
+   * @param {String} aUrl
+   *           The url to be removed from history
+   *
+   * @returns {Promise} Returns a promise set to true if link was removed
+   */
+  deleteHistoryEntry(aUrl) {
+    const url = aUrl;
+    PinnedLinks.unpin({ url });
+    return lazy.PlacesUtils.history.remove(url);
+  },
+
+  /**
+   * Helper function which makes the call to the Pocket API to delete an item from
+   * a user's saved to Pocket feed. Also, invalidate the Pocket stories cache
+   *
+   * @param {Integer} aItemID
+   *           The unique pocket ID used to find the item to be deleted
+   *
+   *@returns {Promise} Returns a promise at completion
+   */
+  deletePocketEntry(aItemID) {
+    this._savedPocketStories = null;
+    return new Promise((success, error) =>
+      lazy.pktApi.deleteItem(aItemID, { success, error })
+    );
+  },
+
+  /**
+   * Helper function which makes the call to the Pocket API to archive an item from
+   * a user's saved to Pocket feed. Also, invalidate the Pocket stories cache
+   *
+   * @param {Integer} aItemID
+   *           The unique pocket ID used to find the item to be archived
+   *
+   *@returns {Promise} Returns a promise at completion
+   */
+  archivePocketEntry(aItemID) {
+    this._savedPocketStories = null;
+    return new Promise((success, error) =>
+      lazy.pktApi.archiveItem(aItemID, { success, error })
+    );
+  },
+
+  /**
+   * Helper function which makes the call to the Pocket API to save an item to
+   * a user's saved to Pocket feed if they are logged in. Also, invalidate the
+   * Pocket stories cache
+   *
+   * @param {String} aUrl
+   *           The URL belonging to the story being saved
+   * @param {String} aTitle
+   *           The title belonging to the story being saved
+   * @param {Browser} aBrowser
+   *           The target browser to show the doorhanger in
+   *
+   *@returns {Promise} Returns a promise at completion
+   */
+  addPocketEntry(aUrl, aTitle, aBrowser) {
+    // If the user is not logged in, show the panel to prompt them to log in
+    if (!lazy.pktApi.isUserLoggedIn()) {
+      lazy.Pocket.savePage(aBrowser, aUrl, aTitle);
+      return Promise.resolve(null);
+    }
+
+    // If the user is logged in, just save the link to Pocket and Activity Stream
+    // will update the page
+    this._savedPocketStories = null;
+    return new Promise((success, error) => {
+      lazy.pktApi.addLink(aUrl, {
+        title: aTitle,
+        success,
+        error,
+      });
+    });
+  },
+
+  /**
+   * Get the Highlights links to show on Activity Stream
+   *
+   * @param {Object} aOptions
+   *   {bool} excludeBookmarks: Don't add bookmark items.
+   *   {bool} excludeHistory: Don't add history items.
+   *   {bool} excludePocket: Don't add Pocket items.
+   *   {bool} withFavicons: Add favicon data: URIs, when possible.
+   *   {int}  numItems: Maximum number of (bookmark or history) items to return.
+   *
+   * @return {Promise} Returns a promise with the array of links as the payload
+   */
+  async getHighlights(aOptions = {}) {
+    aOptions.numItems = aOptions.numItems || ACTIVITY_STREAM_DEFAULT_LIMIT;
+    const results = [];
+
+    // First get bookmarks if we want them
+    if (!aOptions.excludeBookmarks) {
+      results.push(
+        ...(await ActivityStreamProvider.getRecentBookmarks(aOptions))
+      );
+    }
+
+    // Add the Pocket items if we need more and want them
+    if (aOptions.numItems - results.length > 0 && !aOptions.excludePocket) {
+      const latestSince = ~~Services.prefs.getStringPref(
+        PREF_POCKET_LATEST_SINCE,
+        0
+      );
+      // Invalidate the cache, get new stories, and update timestamps if:
+      //  1. we do not have saved to Pocket stories already cached OR
+      //  2. it has been too long since we last got Pocket stories OR
+      //  3. there has been a paged saved to pocket since we last got new stories
+      if (
+        !this._savedPocketStories ||
+        Date.now() - this._pocketLastUpdated > POCKET_UPDATE_TIME ||
+        this._pocketLastLatest < latestSince
+      ) {
+        this._savedPocketStories =
+          await ActivityStreamProvider.getRecentlyPocketed(aOptions);
+        this._pocketLastUpdated = Date.now();
+        this._pocketLastLatest = latestSince;
+      }
+      results.push(...this._savedPocketStories);
+    }
+
+    // Add in history if we need more and want them
+    if (aOptions.numItems - results.length > 0 && !aOptions.excludeHistory) {
+      // Use the same numItems as bookmarks above in case we remove duplicates
+      const history = await ActivityStreamProvider.getRecentHistory(aOptions);
+
+      // Only include a url once in the result preferring the bookmark
+      const bookmarkUrls = new Set(results.map(({ url }) => url));
+      for (const page of history) {
+        if (!bookmarkUrls.has(page.url)) {
+          results.push(page);
+
+          // Stop adding pages once we reach the desired maximum
+          if (results.length === aOptions.numItems) {
+            break;
+          }
+        }
+      }
+    }
+
+    if (aOptions.withFavicons) {
+      return ActivityStreamProvider._faviconBytesToDataURI(
+        await ActivityStreamProvider._addFavicons(results)
+      );
+    }
+
+    return results;
+  },
+
+  /**
+   * Get the top sites to show on Activity Stream
+   *
+   * @return {Promise} Returns a promise with the array of links as the payload
+   */
+  async getTopSites(aOptions = {}) {
+    return ActivityStreamProvider.getTopFrecentSites(aOptions);
+  },
+};
+
+/**
+ * Singleton that provides access to all links contained in the grid (including
+ * the ones that don't fit on the grid). A link is a plain object that looks
+ * like this:
+ *
+ * {
+ *   url: "http://www.mozilla.org/",
+ *   title: "Mozilla",
+ *   frecency: 1337,
+ *   lastVisitDate: 1394678824766431,
+ * }
+ */
+var Links = {
+  /**
+   * The maximum number of links returned by getLinks.
+   */
+  maxNumLinks: LINKS_GET_LINKS_LIMIT,
+
+  /**
+   * A mapping from each provider to an object { sortedLinks, siteMap, linkMap }.
+   * sortedLinks is the cached, sorted array of links for the provider.
+   * siteMap is a mapping from base domains to URL count associated with the domain.
+   *         The count does not include blocked URLs. siteMap is used to look up a
+   *         user's top sites that can be targeted with a suggested tile.
+   * linkMap is a Map from link URLs to link objects.
+   */
+  _providers: new Map(),
+
+  /**
+   * The properties of link objects used to sort them.
+   */
+  _sortProperties: ["frecency", "lastVisitDate", "url"],
+
+  /**
+   * List of callbacks waiting for the cache to be populated.
+   */
+  _populateCallbacks: [],
+
+  /**
+   * A list of objects that are observing links updates.
+   */
+  _observers: [],
+
+  /**
+   * Registers an object that will be notified when links updates.
+   */
+  addObserver(aObserver) {
+    this._observers.push(aObserver);
+  },
+
+  /**
+   * Adds a link provider.
+   * @param aProvider The link provider.
+   */
+  addProvider: function Links_addProvider(aProvider) {
+    this._providers.set(aProvider, null);
+    aProvider.addObserver(this);
+  },
+
+  /**
+   * Removes a link provider.
+   * @param aProvider The link provider.
+   */
+  removeProvider: function Links_removeProvider(aProvider) {
+    if (!this._providers.delete(aProvider)) {
+      throw new Error("Unknown provider");
+    }
+  },
+
+  /**
+   * Populates the cache with fresh links from the providers.
+   * @param aCallback The callback to call when finished (optional).
+   * @param aForce When true, populates the cache even when it's already filled.
+   */
+  populateCache: function Links_populateCache(aCallback, aForce) {
+    let callbacks = this._populateCallbacks;
+
+    // Enqueue the current callback.
+    callbacks.push(aCallback);
+
+    // There was a callback waiting already, thus the cache has not yet been
+    // populated.
+    if (callbacks.length > 1) {
+      return;
+    }
+
+    function executeCallbacks() {
+      while (callbacks.length) {
+        let callback = callbacks.shift();
+        if (callback) {
+          try {
+            callback();
+          } catch (e) {
+            // We want to proceed even if a callback fails.
+          }
+        }
+      }
+    }
+
+    let numProvidersRemaining = this._providers.size;
+    for (let [provider /* , links */] of this._providers) {
+      this._populateProviderCache(
+        provider,
+        () => {
+          if (--numProvidersRemaining == 0) {
+            executeCallbacks();
+          }
+        },
+        aForce
+      );
+    }
+
+    this._addObserver();
+  },
+
+  /**
+   * Gets the current set of links contained in the grid.
+   * @return The links in the grid.
+   */
+  getLinks: function Links_getLinks() {
+    let pinnedLinks = Array.from(PinnedLinks.links);
+    let links = this._getMergedProviderLinks();
+
+    let sites = new Set();
+    for (let link of pinnedLinks) {
+      if (link) {
+        sites.add(NewTabUtils.extractSite(link.url));
+      }
+    }
+
+    // Filter blocked and pinned links and duplicate base domains.
+    links = links.filter(function (link) {
+      let site = NewTabUtils.extractSite(link.url);
+      if (site == null || sites.has(site)) {
+        return false;
+      }
+      sites.add(site);
+
+      return !BlockedLinks.isBlocked(link) && !PinnedLinks.isPinned(link);
+    });
+
+    // Try to fill the gaps between pinned links.
+    for (let i = 0; i < pinnedLinks.length && links.length; i++) {
+      if (!pinnedLinks[i]) {
+        pinnedLinks[i] = links.shift();
+      }
+    }
+
+    // Append the remaining links if any.
+    if (links.length) {
+      pinnedLinks = pinnedLinks.concat(links);
+    }
+
+    for (let link of pinnedLinks) {
+      if (link) {
+        link.baseDomain = NewTabUtils.extractSite(link.url);
+      }
+    }
+    return pinnedLinks;
+  },
+
+  /**
+   * Resets the links cache.
+   */
+  resetCache: function Links_resetCache() {
+    for (let provider of this._providers.keys()) {
+      this._providers.set(provider, null);
+    }
+  },
+
+  /**
+   * Compares two links.
+   * @param aLink1 The first link.
+   * @param aLink2 The second link.
+   * @return A negative number if aLink1 is ordered before aLink2, zero if
+   *         aLink1 and aLink2 have the same ordering, or a positive number if
+   *         aLink1 is ordered after aLink2.
+   *
+   * @note compareLinks's this object is bound to Links below.
+   */
+  compareLinks: function Links_compareLinks(aLink1, aLink2) {
+    for (let prop of this._sortProperties) {
+      if (!(prop in aLink1) || !(prop in aLink2)) {
+        throw new Error("Comparable link missing required property: " + prop);
+      }
+    }
+    return (
+      aLink2.frecency - aLink1.frecency ||
+      aLink2.lastVisitDate - aLink1.lastVisitDate ||
+      aLink1.url.localeCompare(aLink2.url)
+    );
+  },
+
+  _incrementSiteMap(map, link) {
+    if (NewTabUtils.blockedLinks.isBlocked(link)) {
+      // Don't count blocked URLs.
+      return;
+    }
+    let site = NewTabUtils.extractSite(link.url);
+    map.set(site, (map.get(site) || 0) + 1);
+  },
+
+  _decrementSiteMap(map, link) {
+    if (NewTabUtils.blockedLinks.isBlocked(link)) {
+      // Blocked URLs are not included in map.
+      return;
+    }
+    let site = NewTabUtils.extractSite(link.url);
+    let previousURLCount = map.get(site);
+    if (previousURLCount === 1) {
+      map.delete(site);
+    } else {
+      map.set(site, previousURLCount - 1);
+    }
+  },
+
+  /**
+   * Update the siteMap cache based on the link given and whether we need
+   * to increment or decrement it. We do this by iterating over all stored providers
+   * to find which provider this link already exists in. For providers that
+   * have this link, we will adjust siteMap for them accordingly.
+   *
+   * @param aLink The link that will affect siteMap
+   * @param increment A boolean for whether to increment or decrement siteMap
+   */
+  _adjustSiteMapAndNotify(aLink, increment = true) {
+    for (let [, /* provider */ cache] of this._providers) {
+      // We only update siteMap if aLink is already stored in linkMap.
+      if (cache.linkMap.get(aLink.url)) {
+        if (increment) {
+          this._incrementSiteMap(cache.siteMap, aLink);
+          continue;
+        }
+        this._decrementSiteMap(cache.siteMap, aLink);
+      }
+    }
+    this._callObservers("onLinkChanged", aLink);
+  },
+
+  onLinkBlocked(aLink) {
+    this._adjustSiteMapAndNotify(aLink, false);
+  },
+
+  onLinkUnblocked(aLink) {
+    this._adjustSiteMapAndNotify(aLink);
+  },
+
+  populateProviderCache(provider, callback) {
+    if (!this._providers.has(provider)) {
+      throw new Error(
+        "Can only populate provider cache for existing provider."
+      );
+    }
+
+    return this._populateProviderCache(provider, callback, false);
+  },
+
+  /**
+   * Calls getLinks on the given provider and populates our cache for it.
+   * @param aProvider The provider whose cache will be populated.
+   * @param aCallback The callback to call when finished.
+   * @param aForce When true, populates the provider's cache even when it's
+   *               already filled.
+   */
+  _populateProviderCache(aProvider, aCallback, aForce) {
+    let cache = this._providers.get(aProvider);
+    let createCache = !cache;
+    if (createCache) {
+      cache = {
+        // Start with a resolved promise.
+        populatePromise: new Promise(resolve => resolve()),
+      };
+      this._providers.set(aProvider, cache);
+    }
+    // Chain the populatePromise so that calls are effectively queued.
+    cache.populatePromise = cache.populatePromise.then(() => {
+      return new Promise(resolve => {
+        if (!createCache && !aForce) {
+          aCallback();
+          resolve();
+          return;
+        }
+        aProvider.getLinks(links => {
+          // Filter out null and undefined links so we don't have to deal with
+          // them in getLinks when merging links from providers.
+          links = links.filter(link => !!link);
+          cache.sortedLinks = links;
+          cache.siteMap = links.reduce((map, link) => {
+            this._incrementSiteMap(map, link);
+            return map;
+          }, new Map());
+          cache.linkMap = links.reduce((map, link) => {
+            map.set(link.url, link);
+            return map;
+          }, new Map());
+          aCallback();
+          resolve();
+        });
+      });
+    });
+  },
+
+  /**
+   * Merges the cached lists of links from all providers whose lists are cached.
+   * @return The merged list.
+   */
+  _getMergedProviderLinks: function Links__getMergedProviderLinks() {
+    // Build a list containing a copy of each provider's sortedLinks list.
+    let linkLists = [];
+    for (let provider of this._providers.keys()) {
+      let links = this._providers.get(provider);
+      if (links && links.sortedLinks) {
+        linkLists.push(links.sortedLinks.slice());
+      }
+    }
+
+    return this.mergeLinkLists(linkLists);
+  },
+
+  mergeLinkLists: function Links_mergeLinkLists(linkLists) {
+    if (linkLists.length == 1) {
+      return linkLists[0];
+    }
+
+    function getNextLink() {
+      let minLinks = null;
+      for (let links of linkLists) {
+        if (
+          links.length &&
+          (!minLinks || Links.compareLinks(links[0], minLinks[0]) < 0)
+        ) {
+          minLinks = links;
+        }
+      }
+      return minLinks ? minLinks.shift() : null;
+    }
+
+    let finalLinks = [];
+    for (
+      let nextLink = getNextLink();
+      nextLink && finalLinks.length < this.maxNumLinks;
+      nextLink = getNextLink()
+    ) {
+      finalLinks.push(nextLink);
+    }
+
+    return finalLinks;
+  },
+
+  /**
+   * Called by a provider to notify us when a single link changes.
+   * @param aProvider The provider whose link changed.
+   * @param aLink The link that changed.  If the link is new, it must have all
+   *              of the _sortProperties.  Otherwise, it may have as few or as
+   *              many as is convenient.
+   * @param aIndex The current index of the changed link in the sortedLinks
+                   cache in _providers. Defaults to -1 if the provider doesn't know the index
+   * @param aDeleted Boolean indicating if the provider has deleted the link.
+   */
+  onLinkChanged: function Links_onLinkChanged(
+    aProvider,
+    aLink,
+    aIndex = -1,
+    aDeleted = false
+  ) {
+    if (!("url" in aLink)) {
+      throw new Error("Changed links must have a url property");
+    }
+
+    let links = this._providers.get(aProvider);
+    if (!links) {
+      // This is not an error, it just means that between the time the provider
+      // was added and the future time we call getLinks on it, it notified us of
+      // a change.
+      return;
+    }
+
+    let { sortedLinks, siteMap, linkMap } = links;
+    let existingLink = linkMap.get(aLink.url);
+    let insertionLink = null;
+    let updatePages = false;
+
+    if (existingLink) {
+      // Update our copy's position in O(lg n) by first removing it from its
+      // list.  It's important to do this before modifying its properties.
+      if (this._sortProperties.some(prop => prop in aLink)) {
+        let idx = aIndex;
+        if (idx < 0) {
+          idx = this._indexOf(sortedLinks, existingLink);
+        } else if (this.compareLinks(aLink, sortedLinks[idx]) != 0) {
+          throw new Error("aLink should be the same as sortedLinks[idx]");
+        }
+
+        if (idx < 0) {
+          throw new Error("Link should be in _sortedLinks if in _linkMap");
+        }
+        sortedLinks.splice(idx, 1);
+
+        if (aDeleted) {
+          updatePages = true;
+          linkMap.delete(existingLink.url);
+          this._decrementSiteMap(siteMap, existingLink);
+        } else {
+          // Update our copy's properties.
+          Object.assign(existingLink, aLink);
+
+          // Finally, reinsert our copy below.
+          insertionLink = existingLink;
+        }
+      }
+      // Update our copy's title in O(1).
+      if ("title" in aLink && aLink.title != existingLink.title) {
+        existingLink.title = aLink.title;
+        updatePages = true;
+      }
+    } else if (this._sortProperties.every(prop => prop in aLink)) {
+      // Before doing the O(lg n) insertion below, do an O(1) check for the
+      // common case where the new link is too low-ranked to be in the list.
+      if (sortedLinks.length && sortedLinks.length == aProvider.maxNumLinks) {
+        let lastLink = sortedLinks[sortedLinks.length - 1];
+        if (this.compareLinks(lastLink, aLink) < 0) {
+          return;
+        }
+      }
+      // Copy the link object so that changes later made to it by the caller
+      // don't affect our copy.
+      insertionLink = {};
+      for (let prop in aLink) {
+        insertionLink[prop] = aLink[prop];
+      }
+      linkMap.set(aLink.url, insertionLink);
+      this._incrementSiteMap(siteMap, aLink);
+    }
+
+    if (insertionLink) {
+      let idx = this._insertionIndexOf(sortedLinks, insertionLink);
+      sortedLinks.splice(idx, 0, insertionLink);
+      if (sortedLinks.length > aProvider.maxNumLinks) {
+        let lastLink = sortedLinks.pop();
+        linkMap.delete(lastLink.url);
+        this._decrementSiteMap(siteMap, lastLink);
+      }
+      updatePages = true;
+    }
+
+    if (updatePages) {
+      AllPages.update(null, "links-changed");
+    }
+  },
+
+  /**
+   * Called by a provider to notify us when many links change.
+   */
+  onManyLinksChanged: function Links_onManyLinksChanged(aProvider) {
+    this._populateProviderCache(
+      aProvider,
+      () => {
+        AllPages.update(null, "links-changed");
+      },
+      true
+    );
+  },
+
+  _indexOf: function Links__indexOf(aArray, aLink) {
+    return this._binsearch(aArray, aLink, "indexOf");
+  },
+
+  _insertionIndexOf: function Links__insertionIndexOf(aArray, aLink) {
+    return this._binsearch(aArray, aLink, "insertionIndexOf");
+  },
+
+  _binsearch: function Links__binsearch(aArray, aLink, aMethod) {
+    return lazy.BinarySearch[aMethod](this.compareLinks, aArray, aLink);
+  },
+
+  /**
+   * Implements the nsIObserver interface to get notified about browser history
+   * sanitization.
+   */
+  observe: function Links_observe(aSubject, aTopic, aData) {
+    // Make sure to update open about:newtab instances. If there are no opened
+    // pages we can just wait for the next new tab to populate the cache again.
+    if (AllPages.length && AllPages.enabled) {
+      this.populateCache(function () {
+        AllPages.update();
+      }, true);
+    } else {
+      this.resetCache();
+    }
+  },
+
+  _callObservers(methodName, ...args) {
+    for (let obs of this._observers) {
+      if (typeof obs[methodName] == "function") {
+        try {
+          obs[methodName](this, ...args);
+        } catch (err) {
+          console.error(err);
+        }
+      }
+    }
+  },
+
+  /**
+   * Adds a sanitization observer and turns itself into a no-op after the first
+   * invokation.
+   */
+  _addObserver: function Links_addObserver() {
+    Services.obs.addObserver(this, "browser:purge-session-history", true);
+    this._addObserver = function () {};
+  },
+
+  QueryInterface: ChromeUtils.generateQI([
+    "nsIObserver",
+    "nsISupportsWeakReference",
+  ]),
+};
+
+Links.compareLinks = Links.compareLinks.bind(Links);
+
+/**
+ * Singleton used to collect telemetry data.
+ *
+ */
+var Telemetry = {
+  /**
+   * Initializes object.
+   */
+  init: function Telemetry_init() {
+    Services.obs.addObserver(this, TOPIC_GATHER_TELEMETRY);
+  },
+
+  uninit: function Telemetry_uninit() {
+    Services.obs.removeObserver(this, TOPIC_GATHER_TELEMETRY);
+  },
+
+  /**
+   * Collects data.
+   */
+  _collect: function Telemetry_collect() {
+    let probes = [
+      { histogram: "NEWTAB_PAGE_ENABLED", value: AllPages.enabled },
+      {
+        histogram: "NEWTAB_PAGE_PINNED_SITES_COUNT",
+        value: PinnedLinks.links.length,
+      },
+      {
+        histogram: "NEWTAB_PAGE_BLOCKED_SITES_COUNT",
+        value: Object.keys(BlockedLinks.links).length,
+      },
+    ];
+
+    probes.forEach(function Telemetry_collect_forEach(aProbe) {
+      Services.telemetry.getHistogramById(aProbe.histogram).add(aProbe.value);
+    });
+  },
+
+  /**
+   * Listens for gather telemetry topic.
+   */
+  observe: function Telemetry_observe(aSubject, aTopic, aData) {
+    this._collect();
+  },
+};
+
+/**
+ * Singleton that checks if a given link should be displayed on about:newtab
+ * or if we should rather not do it for security reasons. URIs that inherit
+ * their caller's principal will be filtered.
+ */
+var LinkChecker = {
+  _cache: {},
+
+  get flags() {
+    return (
+      Ci.nsIScriptSecurityManager.DISALLOW_INHERIT_PRINCIPAL |
+      Ci.nsIScriptSecurityManager.DONT_REPORT_ERRORS
+    );
+  },
+
+  checkLoadURI: function LinkChecker_checkLoadURI(aURI) {
+    if (!(aURI in this._cache)) {
+      this._cache[aURI] = this._doCheckLoadURI(aURI);
+    }
+
+    return this._cache[aURI];
+  },
+
+  _doCheckLoadURI: function Links_doCheckLoadURI(aURI) {
+    try {
+      // about:newtab is currently privileged. In any case, it should be
+      // possible for tiles to point to pretty much everything - but not
+      // to stuff that inherits the system principal, so we check:
+      let systemPrincipal = Services.scriptSecurityManager.getSystemPrincipal();
+      Services.scriptSecurityManager.checkLoadURIStrWithPrincipal(
+        systemPrincipal,
+        aURI,
+        this.flags
+      );
+      return true;
+    } catch (e) {
+      // We got a weird URI or one that would inherit the caller's principal.
+      return false;
+    }
+  },
+};
+
+var ExpirationFilter = {
+  init: function ExpirationFilter_init() {
+    lazy.PageThumbs.addExpirationFilter(this);
+  },
+
+  filterForThumbnailExpiration:
+    function ExpirationFilter_filterForThumbnailExpiration(aCallback) {
+      if (!AllPages.enabled) {
+        aCallback([]);
+        return;
+      }
+
+      Links.populateCache(function () {
+        let urls = [];
+
+        // Add all URLs to the list that we want to keep thumbnails for.
+        for (let link of Links.getLinks().slice(0, 25)) {
+          if (link && link.url) {
+            urls.push(link.url);
+          }
+        }
+
+        aCallback(urls);
+      });
+    },
+};
+
+/**
+ * Singleton that provides the public API of this JSM.
+ */
+export var NewTabUtils = {
+  _initialized: false,
+
+  /**
+   * Extract a "site" from a url in a way that multiple urls of a "site" returns
+   * the same "site."
+   * @param aUrl Url spec string
+   * @return The "site" string or null
+   */
+  extractSite: function Links_extractSite(url) {
+    let host;
+    try {
+      // Note that nsIURI.asciiHost throws NS_ERROR_FAILURE for some types of
+      // URIs, including jar and moz-icon URIs.
+      host = Services.io.newURI(url).asciiHost;
+    } catch (ex) {
+      return null;
+    }
+
+    // Strip off common subdomains of the same site (e.g., www, load balancer)
+    return host.replace(/^(m|mobile|www\d*)\./, "");
+  },
+
+  init: function NewTabUtils_init() {
+    if (this.initWithoutProviders()) {
+      PlacesProvider.init();
+      Links.addProvider(PlacesProvider);
+      BlockedLinks.addObserver(Links);
+      BlockedLinks.addObserver(ActivityStreamLinks);
+    }
+  },
+
+  initWithoutProviders: function NewTabUtils_initWithoutProviders() {
+    if (!this._initialized) {
+      this._initialized = true;
+      ExpirationFilter.init();
+      Telemetry.init();
+      return true;
+    }
+    return false;
+  },
+
+  uninit: function NewTabUtils_uninit() {
+    if (this.initialized) {
+      Telemetry.uninit();
+      BlockedLinks.removeObservers();
+    }
+  },
+
+  getProviderLinks(aProvider) {
+    let cache = Links._providers.get(aProvider);
+    if (cache && cache.sortedLinks) {
+      return cache.sortedLinks;
+    }
+    return [];
+  },
+
+  isTopSiteGivenProvider(aSite, aProvider) {
+    let cache = Links._providers.get(aProvider);
+    if (cache && cache.siteMap) {
+      return cache.siteMap.has(aSite);
+    }
+    return false;
+  },
+
+  isTopPlacesSite(aSite) {
+    return this.isTopSiteGivenProvider(aSite, PlacesProvider);
+  },
+
+  /**
+   * Restores all sites that have been removed from the grid.
+   */
+  restore: function NewTabUtils_restore() {
+    lazy.Storage.clear();
+    Links.resetCache();
+    PinnedLinks.resetCache();
+    BlockedLinks.resetCache();
+
+    Links.populateCache(function () {
+      AllPages.update();
+    }, true);
+  },
+
+  /**
+   * Undoes all sites that have been removed from the grid and keep the pinned
+   * tabs.
+   * @param aCallback the callback method.
+   */
+  undoAll: function NewTabUtils_undoAll(aCallback) {
+    lazy.Storage.remove("blockedLinks");
+    Links.resetCache();
+    BlockedLinks.resetCache();
+    Links.populateCache(aCallback, true);
+  },
+
+  links: Links,
+  allPages: AllPages,
+  pinnedLinks: PinnedLinks,
+  blockedLinks: BlockedLinks,
+  activityStreamLinks: ActivityStreamLinks,
+  activityStreamProvider: ActivityStreamProvider,
+};