Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1903 insertions, 0 deletions

diff --git a/browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js b/browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js
new file mode 100644
index 0000000000..f6560d7ab2
--- /dev/null
+++ b/browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js
@@ -0,0 +1,1903 @@
+import { FAKE_GLOBAL_PREFS, FakePrefs, GlobalOverrider } from "test/unit/utils";
+import { actionTypes as at } from "common/Actions.sys.mjs";
+import injector from "inject!lib/TopStoriesFeed.jsm";
+
+describe("Top Stories Feed", () => {
+  let TopStoriesFeed;
+  let STORIES_UPDATE_TIME;
+  let TOPICS_UPDATE_TIME;
+  let SECTION_ID;
+  let SPOC_IMPRESSION_TRACKING_PREF;
+  let REC_IMPRESSION_TRACKING_PREF;
+  let DEFAULT_RECS_EXPIRE_TIME;
+  let instance;
+  let clock;
+  let globals;
+  let sectionsManagerStub;
+  let shortURLStub;
+
+  const FAKE_OPTIONS = {
+    stories_endpoint: "https://somedomain.org/stories?key=$apiKey",
+    stories_referrer: "https://somedomain.org/referrer",
+    topics_endpoint: "https://somedomain.org/topics?key=$apiKey",
+    survey_link: "https://www.surveymonkey.com/r/newtabffx",
+    api_key_pref: "apiKeyPref",
+    provider_name: "test-provider",
+    provider_icon: "provider-icon",
+    provider_description: "provider_desc",
+  };
+
+  beforeEach(() => {
+    FAKE_GLOBAL_PREFS.set("apiKeyPref", "test-api-key");
+    FAKE_GLOBAL_PREFS.set(
+      "pocketCta",
+      JSON.stringify({
+        cta_button: "",
+        cta_text: "",
+        cta_url: "",
+        use_cta: false,
+      })
+    );
+
+    globals = new GlobalOverrider();
+    globals.set("PlacesUtils", { history: {} });
+    globals.set("pktApi", { isUserLoggedIn() {} });
+    clock = sinon.useFakeTimers();
+    shortURLStub = sinon.stub().callsFake(site => site.url);
+    sectionsManagerStub = {
+      onceInitialized: sinon.stub().callsFake(callback => callback()),
+      enableSection: sinon.spy(),
+      disableSection: sinon.spy(),
+      updateSection: sinon.spy(),
+      sections: new Map([["topstories", { options: FAKE_OPTIONS }]]),
+    };
+
+    ({
+      TopStoriesFeed,
+      STORIES_UPDATE_TIME,
+      TOPICS_UPDATE_TIME,
+      SECTION_ID,
+      SPOC_IMPRESSION_TRACKING_PREF,
+      REC_IMPRESSION_TRACKING_PREF,
+      DEFAULT_RECS_EXPIRE_TIME,
+    } = injector({
+      "lib/ActivityStreamPrefs.jsm": { Prefs: FakePrefs },
+      "lib/ShortURL.jsm": { shortURL: shortURLStub },
+      "lib/SectionsManager.jsm": { SectionsManager: sectionsManagerStub },
+    }));
+
+    instance = new TopStoriesFeed();
+    instance.store = {
+      getState() {
+        return {
+          Prefs: {
+            values: {
+              showSponsored: true,
+              "feeds.section.topstories": true,
+            },
+          },
+        };
+      },
+      dispatch: sinon.spy(),
+    };
+    instance.storiesLastUpdated = 0;
+    instance.topicsLastUpdated = 0;
+  });
+  afterEach(() => {
+    globals.restore();
+    clock.restore();
+  });
+
+  describe("#lazyloading TopStories", () => {
+    beforeEach(() => {
+      instance.discoveryStreamEnabled = true;
+    });
+    it("should bind parseOptions to SectionsManager.onceInitialized when discovery stream is true", () => {
+      instance.discoveryStreamEnabled = false;
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: true }),
+            "feeds.section.topstories": true,
+          },
+        },
+      });
+      instance.onAction({ type: at.INIT, data: {} });
+
+      assert.calledOnce(sectionsManagerStub.onceInitialized);
+    });
+    it("should bind parseOptions to SectionsManager.onceInitialized when discovery stream is false", () => {
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: false }),
+            "feeds.section.topstories": true,
+          },
+        },
+      });
+      instance.onAction({ type: at.INIT, data: {} });
+      assert.calledOnce(sectionsManagerStub.onceInitialized);
+    });
+    it("Should initialize properties once while lazy loading if not initialized earlier", () => {
+      instance.discoveryStreamEnabled = false;
+      instance.propertiesInitialized = false;
+      sinon.stub(instance, "initializeProperties");
+      instance.lazyLoadTopStories();
+      assert.calledOnce(instance.initializeProperties);
+    });
+    it("should not re-initialize properties", () => {
+      // For discovery stream experience disabled TopStoriesFeed properties
+      // are initialized in constructor and should not be called again while lazy loading topstories
+      sinon.stub(instance, "initializeProperties");
+      instance.discoveryStreamEnabled = false;
+      instance.propertiesInitialized = true;
+      instance.lazyLoadTopStories();
+      assert.notCalled(instance.initializeProperties);
+    });
+    it("should have early exit onInit when discovery is true", async () => {
+      sinon.stub(instance, "doContentUpdate");
+      await instance.onInit();
+      assert.notCalled(instance.doContentUpdate);
+      assert.isUndefined(instance.storiesLoaded);
+    });
+    it("should complete onInit when discovery is false", async () => {
+      instance.discoveryStreamEnabled = false;
+      sinon.stub(instance, "doContentUpdate");
+      await instance.onInit();
+      assert.calledOnce(instance.doContentUpdate);
+      assert.isTrue(instance.storiesLoaded);
+    });
+    it("should handle limited actions when discoverystream is enabled", async () => {
+      sinon.spy(instance, "handleDisabled");
+      sinon.stub(instance, "getPocketState");
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: true }),
+            "discoverystream.enabled": true,
+            "feeds.section.topstories": true,
+          },
+        },
+      });
+
+      instance.onAction({ type: at.INIT, data: {} });
+
+      assert.calledOnce(instance.handleDisabled);
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.notCalled(instance.getPocketState);
+    });
+    it("should handle NEW_TAB_REHYDRATED when discoverystream is disabled", async () => {
+      instance.discoveryStreamEnabled = false;
+      sinon.spy(instance, "handleDisabled");
+      sinon.stub(instance, "getPocketState");
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: false }),
+            "feeds.section.topstories": true,
+          },
+        },
+      });
+      instance.onAction({ type: at.INIT, data: {} });
+      assert.notCalled(instance.handleDisabled);
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.getPocketState);
+    });
+    it("should handle UNINIT when discoverystream is enabled", async () => {
+      sinon.stub(instance, "uninit");
+      instance.onAction({ type: at.UNINIT });
+      assert.calledOnce(instance.uninit);
+    });
+    it("should fire init on PREF_CHANGED", () => {
+      sinon.stub(instance, "onInit");
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.config", value: {} },
+      });
+      assert.calledOnce(instance.onInit);
+    });
+    it("should fire init on DISCOVERY_STREAM_PREF_ENABLED", () => {
+      sinon.stub(instance, "onInit");
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.enabled", value: true },
+      });
+      assert.calledOnce(instance.onInit);
+    });
+    it("should not fire init on PREF_CHANGED if stories are loaded", () => {
+      sinon.stub(instance, "onInit");
+      sinon.spy(instance, "lazyLoadTopStories");
+      instance.storiesLoaded = true;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.config", value: {} },
+      });
+      assert.calledOnce(instance.lazyLoadTopStories);
+      assert.notCalled(instance.onInit);
+    });
+    it("should fire init on PREF_CHANGED when discoverystream is disabled", () => {
+      instance.discoveryStreamEnabled = false;
+      sinon.stub(instance, "onInit");
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.config", value: {} },
+      });
+      assert.calledOnce(instance.onInit);
+    });
+    it("should not fire init on PREF_CHANGED when discoverystream is disabled and stories are loaded", () => {
+      instance.discoveryStreamEnabled = false;
+      sinon.stub(instance, "onInit");
+      sinon.spy(instance, "lazyLoadTopStories");
+      instance.storiesLoaded = true;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.config", value: {} },
+      });
+      assert.calledOnce(instance.lazyLoadTopStories);
+      assert.notCalled(instance.onInit);
+    });
+    it("should not init props if ds pref is true", () => {
+      sinon.stub(instance, "initializeProperties");
+      instance.propertiesInitialized = false;
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: false }),
+            "discoverystream.enabled": true,
+            "feeds.section.topstories": true,
+          },
+        },
+      });
+      instance.lazyLoadTopStories({
+        dsPref: JSON.stringify({ enabled: true }),
+      });
+      assert.notCalled(instance.initializeProperties);
+    });
+    it("should fire init if user pref is true", () => {
+      sinon.stub(instance, "onInit");
+      instance.store.getState = () => ({
+        Prefs: {
+          values: {
+            "discoverystream.config": JSON.stringify({ enabled: false }),
+            "discoverystream.enabled": false,
+            "feeds.section.topstories": false,
+          },
+        },
+      });
+      instance.lazyLoadTopStories({ userPref: true });
+      assert.calledOnce(instance.onInit);
+    });
+    it("should fire uninit if topstories update to false", () => {
+      sinon.stub(instance, "uninit");
+      instance.discoveryStreamEnabled = false;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          value: false,
+          name: "feeds.section.topstories",
+        },
+      });
+      assert.calledOnce(instance.uninit);
+      instance.discoveryStreamEnabled = true;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          value: false,
+          name: "feeds.section.topstories",
+        },
+      });
+      assert.calledTwice(instance.uninit);
+    });
+    it("should fire lazyLoadTopstories if topstories update to true", () => {
+      sinon.stub(instance, "lazyLoadTopStories");
+      instance.discoveryStreamEnabled = false;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          value: true,
+          name: "feeds.section.topstories",
+        },
+      });
+      assert.calledOnce(instance.lazyLoadTopStories);
+      instance.discoveryStreamEnabled = true;
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          value: true,
+          name: "feeds.section.topstories",
+        },
+      });
+      assert.calledTwice(instance.lazyLoadTopStories);
+    });
+  });
+
+  describe("#init", () => {
+    it("should create a TopStoriesFeed", () => {
+      assert.instanceOf(instance, TopStoriesFeed);
+    });
+    it("should bind parseOptions to SectionsManager.onceInitialized", () => {
+      instance.onAction({ type: at.INIT, data: {} });
+      assert.calledOnce(sectionsManagerStub.onceInitialized);
+    });
+    it("should initialize endpoints based on options", async () => {
+      await instance.onInit();
+      assert.equal(
+        "https://somedomain.org/stories?key=test-api-key",
+        instance.stories_endpoint
+      );
+      assert.equal(
+        "https://somedomain.org/referrer",
+        instance.stories_referrer
+      );
+      assert.equal(
+        "https://somedomain.org/topics?key=test-api-key",
+        instance.topics_endpoint
+      );
+    });
+    it("should enable its section", () => {
+      instance.onAction({ type: at.INIT, data: {} });
+      assert.calledOnce(sectionsManagerStub.enableSection);
+      assert.calledWith(sectionsManagerStub.enableSection, SECTION_ID);
+    });
+    it("init should fire onInit", () => {
+      instance.onInit = sinon.spy();
+      instance.onAction({ type: at.INIT, data: {} });
+      assert.calledOnce(instance.onInit);
+    });
+    it("should fetch stories on init", async () => {
+      instance.fetchStories = sinon.spy();
+      await instance.onInit();
+      assert.calledOnce(instance.fetchStories);
+    });
+    it("should fetch topics on init", async () => {
+      instance.fetchTopics = sinon.spy();
+      await instance.onInit();
+      assert.calledOnce(instance.fetchTopics);
+    });
+    it("should not fetch if endpoint not configured", () => {
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      sectionsManagerStub.sections.set("topstories", { options: {} });
+      instance.init();
+      assert.notCalled(fetchStub);
+    });
+    it("should report error for invalid configuration", () => {
+      globals.sandbox.spy(global.console, "error");
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          api_key_pref: "invalid",
+          stories_endpoint: "https://invalid.com/?apiKey=$apiKey",
+        },
+      });
+      instance.init();
+
+      assert.calledWith(
+        console.error,
+        "Problem initializing top stories feed: An API key was specified but none configured: https://invalid.com/?apiKey=$apiKey"
+      );
+    });
+    it("should report error for missing api key", () => {
+      globals.sandbox.spy(global.console, "error");
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          stories_endpoint: "https://somedomain.org/stories?key=$apiKey",
+          topics_endpoint: "https://somedomain.org/topics?key=$apiKey",
+        },
+      });
+      instance.init();
+
+      assert.called(console.error);
+    });
+    it("should load data from cache on init", async () => {
+      instance.loadCachedData = sinon.spy();
+      await instance.onInit();
+      assert.calledOnce(instance.loadCachedData);
+    });
+  });
+  describe("#uninit", () => {
+    it("should disable its section", () => {
+      instance.onAction({ type: at.UNINIT });
+      assert.calledOnce(sectionsManagerStub.disableSection);
+      assert.calledWith(sectionsManagerStub.disableSection, SECTION_ID);
+    });
+    it("should unload stories on uninit", async () => {
+      sinon.stub(instance.cache, "set").returns(Promise.resolve());
+      await instance.clearCache();
+      assert.calledWith(instance.cache.set.firstCall, "stories", {});
+      assert.calledWith(instance.cache.set.secondCall, "topics", {});
+      assert.calledWith(instance.cache.set.thirdCall, "spocs", {});
+    });
+  });
+  describe("#cache", () => {
+    it("should clear all cache items when calling clearCache", () => {
+      sinon.stub(instance.cache, "set").returns(Promise.resolve());
+      instance.storiesLoaded = true;
+      instance.uninit();
+      assert.equal(instance.storiesLoaded, false);
+    });
+    it("should set spocs cache on fetch", async () => {
+      const response = {
+        recommendations: [{ id: "1" }, { id: "2" }],
+        settings: {},
+        spocs: [{ id: "spoc1" }],
+      };
+
+      instance.show_spocs = true;
+      instance.stories_endpoint = "stories-endpoint";
+
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", { blockedLinks: { isBlocked: () => {} } });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      sinon.spy(instance.cache, "set");
+
+      await instance.fetchStories();
+
+      assert.calledOnce(instance.cache.set);
+      const { args } = instance.cache.set.firstCall;
+      assert.equal(args[0], "stories");
+      assert.equal(args[1].spocs[0].id, "spoc1");
+    });
+    it("should get spocs on cache load", async () => {
+      instance.cache.get = () => ({
+        stories: {
+          recommendations: [{ id: "1" }, { id: "2" }],
+          spocs: [{ id: "spoc1" }],
+        },
+      });
+      instance.storiesLastUpdated = 0;
+      globals.set("NewTabUtils", { blockedLinks: { isBlocked: () => {} } });
+
+      await instance.loadCachedData();
+      assert.equal(instance.spocs[0].guid, "spoc1");
+    });
+  });
+  describe("#fetch", () => {
+    it("should fetch stories, send event and cache results", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          stories_endpoint: "stories-endpoint",
+          stories_referrer: "referrer",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        recommendations: [
+          {
+            id: "1",
+            title: "title",
+            excerpt: "description",
+            image_src: "image-url",
+            url: "rec-url",
+            published_timestamp: "123",
+            context: "trending",
+            icon: "icon",
+          },
+        ],
+      };
+      const stories = [
+        {
+          guid: "1",
+          type: "now",
+          title: "title",
+          context: "trending",
+          icon: "icon",
+          description: "description",
+          image: "image-url",
+          referrer: "referrer",
+          url: "rec-url",
+          hostname: "rec-url",
+          score: 1,
+          spoc_meta: {},
+        },
+      ];
+
+      instance.cache.set = sinon.spy();
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      assert.calledOnce(fetchStub);
+      assert.calledOnce(shortURLStub);
+      assert.calledWithExactly(fetchStub, instance.stories_endpoint, {
+        credentials: "omit",
+      });
+      assert.calledOnce(sectionsManagerStub.updateSection);
+      assert.calledWith(sectionsManagerStub.updateSection, SECTION_ID, {
+        rows: stories,
+      });
+      assert.calledOnce(instance.cache.set);
+      assert.calledWith(
+        instance.cache.set,
+        "stories",
+        Object.assign({}, response, { _timestamp: 0 })
+      );
+    });
+    it("should use domain as hostname, if present", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          stories_endpoint: "stories-endpoint",
+          stories_referrer: "referrer",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        recommendations: [
+          {
+            id: "1",
+            title: "title",
+            excerpt: "description",
+            image_src: "image-url",
+            url: "rec-url",
+            domain: "domain",
+            published_timestamp: "123",
+            context: "trending",
+            icon: "icon",
+          },
+        ],
+      };
+      const stories = [
+        {
+          guid: "1",
+          type: "now",
+          title: "title",
+          context: "trending",
+          icon: "icon",
+          description: "description",
+          image: "image-url",
+          referrer: "referrer",
+          url: "rec-url",
+          hostname: "domain",
+          score: 1,
+          spoc_meta: {},
+        },
+      ];
+
+      instance.cache.set = sinon.spy();
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      assert.calledOnce(fetchStub);
+      assert.notCalled(shortURLStub);
+      assert.calledWith(sectionsManagerStub.updateSection, SECTION_ID, {
+        rows: stories,
+      });
+    });
+    it("should call SectionsManager.updateSection", () => {
+      instance.dispatchUpdateEvent(123, {});
+      assert.calledOnce(sectionsManagerStub.updateSection);
+    });
+    it("should report error for unexpected stories response", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: { stories_endpoint: "stories-endpoint" },
+      });
+      globals.set("fetch", fetchStub);
+      globals.sandbox.spy(global.console, "error");
+
+      fetchStub.resolves({ ok: false, status: 400 });
+      await instance.onInit();
+
+      assert.calledOnce(fetchStub);
+      assert.calledWithExactly(fetchStub, instance.stories_endpoint, {
+        credentials: "omit",
+      });
+      assert.equal(instance.storiesLastUpdated, 0);
+      assert.called(console.error);
+    });
+    it("should exclude blocked (dismissed) URLs", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: { stories_endpoint: "stories-endpoint" },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: site => site.url === "blocked" },
+      });
+
+      const response = {
+        recommendations: [{ url: "blocked" }, { url: "not_blocked" }],
+      };
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      // Issue!
+      // Should actually be fixed when cache is fixed.
+      assert.calledOnce(sectionsManagerStub.updateSection);
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows.length,
+        1
+      );
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows[0].url,
+        "not_blocked"
+      );
+    });
+    it("should mark stories as new", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: { stories_endpoint: "stories-endpoint" },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      clock.restore();
+      const response = {
+        recommendations: [
+          { published_timestamp: Date.now() / 1000 },
+          { published_timestamp: "0" },
+          {
+            published_timestamp: (Date.now() - 2 * 24 * 60 * 60 * 1000) / 1000,
+          },
+        ],
+      };
+
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+
+      await instance.onInit();
+      assert.calledOnce(sectionsManagerStub.updateSection);
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows.length,
+        3
+      );
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows[0].type,
+        "now"
+      );
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows[1].type,
+        "trending"
+      );
+      assert.equal(
+        sectionsManagerStub.updateSection.firstCall.args[1].rows[2].type,
+        "trending"
+      );
+    });
+    it("should fetch topics, send event and cache results", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: { topics_endpoint: "topics-endpoint" },
+      });
+      globals.set("fetch", fetchStub);
+
+      const response = {
+        topics: [
+          { name: "topic1", url: "url-topic1" },
+          { name: "topic2", url: "url-topic2" },
+        ],
+      };
+      const topics = [
+        {
+          name: "topic1",
+          url: "url-topic1",
+        },
+        {
+          name: "topic2",
+          url: "url-topic2",
+        },
+      ];
+
+      instance.cache.set = sinon.spy();
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      assert.calledOnce(fetchStub);
+      assert.calledWithExactly(fetchStub, instance.topics_endpoint, {
+        credentials: "omit",
+      });
+      assert.calledOnce(sectionsManagerStub.updateSection);
+      assert.calledWithMatch(sectionsManagerStub.updateSection, SECTION_ID, {
+        topics,
+      });
+      assert.calledOnce(instance.cache.set);
+      assert.calledWith(
+        instance.cache.set,
+        "topics",
+        Object.assign({}, response, { _timestamp: 0 })
+      );
+    });
+    it("should report error for unexpected topics response", async () => {
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.sandbox.spy(global.console, "error");
+
+      instance.topics_endpoint = "topics-endpoint";
+      fetchStub.resolves({ ok: false, status: 400 });
+      await instance.fetchTopics();
+
+      assert.calledOnce(fetchStub);
+      assert.calledWithExactly(fetchStub, instance.topics_endpoint, {
+        credentials: "omit",
+      });
+      assert.notCalled(instance.store.dispatch);
+      assert.called(console.error);
+    });
+  });
+  describe("#personalization", () => {
+    it("should sort stories", async () => {
+      const response = {
+        recommendations: [{ id: "1" }, { id: "2" }],
+        settings: {},
+      };
+
+      instance.compareScore = sinon.spy();
+      instance.stories_endpoint = "stories-endpoint";
+
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+
+      await instance.fetchStories();
+      assert.calledOnce(instance.compareScore);
+    });
+    it("should sort items based on relevance score", () => {
+      let items = [{ score: 0.1 }, { score: 0.2 }];
+      items = items.sort(instance.compareScore);
+      assert.deepEqual(items, [{ score: 0.2 }, { score: 0.1 }]);
+    });
+    it("should rotate items", () => {
+      let items = [
+        { guid: "g1" },
+        { guid: "g2" },
+        { guid: "g3" },
+        { guid: "g4" },
+        { guid: "g5" },
+        { guid: "g6" },
+      ];
+
+      // No impressions should leave items unchanged
+      let rotated = instance.rotate(items);
+      assert.deepEqual(items, rotated);
+
+      // Recent impression should leave items unchanged
+      instance._prefs.get = pref =>
+        pref === REC_IMPRESSION_TRACKING_PREF &&
+        JSON.stringify({ g1: 1, g2: 1, g3: 1 });
+      rotated = instance.rotate(items);
+      assert.deepEqual(items, rotated);
+
+      // Impression older than expiration time should rotate items
+      clock.tick(DEFAULT_RECS_EXPIRE_TIME + 1);
+      rotated = instance.rotate(items);
+      assert.deepEqual(
+        [
+          { guid: "g4" },
+          { guid: "g5" },
+          { guid: "g6" },
+          { guid: "g1" },
+          { guid: "g2" },
+          { guid: "g3" },
+        ],
+        rotated
+      );
+
+      instance._prefs.get = pref =>
+        pref === REC_IMPRESSION_TRACKING_PREF &&
+        JSON.stringify({
+          g1: 1,
+          g2: 1,
+          g3: 1,
+          g4: DEFAULT_RECS_EXPIRE_TIME + 1,
+        });
+      clock.tick(DEFAULT_RECS_EXPIRE_TIME);
+      rotated = instance.rotate(items);
+      assert.deepEqual(
+        [
+          { guid: "g5" },
+          { guid: "g6" },
+          { guid: "g1" },
+          { guid: "g2" },
+          { guid: "g3" },
+          { guid: "g4" },
+        ],
+        rotated
+      );
+    });
+    it("should record top story impressions", async () => {
+      instance._prefs = { get: pref => undefined, set: sinon.spy() };
+
+      clock.tick(1);
+      let expectedPrefValue = JSON.stringify({ 1: 1, 2: 1, 3: 1 });
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 1 }, { id: 2 }, { id: 3 }],
+        },
+      });
+      assert.calledWith(
+        instance._prefs.set.firstCall,
+        REC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValue
+      );
+
+      // Only need to record first impression, so impression pref shouldn't change
+      instance._prefs.get = pref => expectedPrefValue;
+      clock.tick(1);
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 1 }, { id: 2 }, { id: 3 }],
+        },
+      });
+      assert.calledOnce(instance._prefs.set);
+
+      // New first impressions should be added
+      clock.tick(1);
+      let expectedPrefValueTwo = JSON.stringify({
+        1: 1,
+        2: 1,
+        3: 1,
+        4: 3,
+        5: 3,
+        6: 3,
+      });
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 4 }, { id: 5 }, { id: 6 }],
+        },
+      });
+      assert.calledWith(
+        instance._prefs.set.secondCall,
+        REC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValueTwo
+      );
+    });
+    it("should not record top story impressions for non-view impressions", async () => {
+      instance._prefs = { get: pref => undefined, set: sinon.spy() };
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", click: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", block: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", pocket: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+    });
+    it("should clean up top story impressions", async () => {
+      instance._prefs = {
+        get: pref => JSON.stringify({ 1: 1, 2: 1, 3: 1 }),
+        set: sinon.spy(),
+      };
+
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      instance.stories_endpoint = "stories-endpoint";
+      const response = { recommendations: [{ id: 3 }, { id: 4 }, { id: 5 }] };
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.fetchStories();
+
+      // Should remove impressions for rec 1 and 2 as no longer in the feed
+      assert.calledWith(
+        instance._prefs.set.firstCall,
+        REC_IMPRESSION_TRACKING_PREF,
+        JSON.stringify({ 3: 1 })
+      );
+    });
+    it("should not change provider with badly formed JSON", async () => {
+      sinon.stub(instance, "uninit");
+      sinon.stub(instance, "init");
+      sinon.stub(instance, "clearCache").returns(Promise.resolve());
+      await instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          name: "feeds.section.topstories.options",
+          value: "{version: 2}",
+        },
+      });
+      assert.notCalled(instance.uninit);
+      assert.notCalled(instance.init);
+      assert.notCalled(instance.clearCache);
+    });
+  });
+  describe("#spocs", async () => {
+    it("should not display expired or untimestamped spocs", async () => {
+      clock.tick(441792000000); // 01/01/1984
+
+      instance.spocsPerNewTabs = 1;
+      instance.show_spocs = true;
+      instance.isBelowFrequencyCap = () => true;
+
+      // NOTE: `expiration_timestamp` is seconds since UNIX epoch
+      instance.spocs = [
+        // No timestamp stays visible
+        {
+          id: "spoc1",
+        },
+        // Expired spoc gets filtered out
+        {
+          id: "spoc2",
+          expiration_timestamp: 1,
+        },
+        // Far future expiration spoc stays visible
+        {
+          id: "spoc3",
+          expiration_timestamp: 32503708800, // 01/01/3000
+        },
+      ];
+
+      sinon.spy(instance, "filterSpocs");
+
+      instance.filterSpocs();
+
+      assert.equal(instance.filterSpocs.firstCall.returnValue.length, 2);
+      assert.equal(instance.filterSpocs.firstCall.returnValue[0].id, "spoc1");
+      assert.equal(instance.filterSpocs.firstCall.returnValue[1].id, "spoc3");
+    });
+    it("should insert spoc with provided probability", async () => {
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        recommendations: [{ guid: "rec1" }, { guid: "rec2" }, { guid: "rec3" }],
+        // Include spocs with a expiration in the very distant future
+        spocs: [
+          { id: "spoc1", expiration_timestamp: 9999999999999 },
+          { id: "spoc2", expiration_timestamp: 9999999999999 },
+        ],
+      };
+
+      instance.show_spocs = true;
+      instance.stories_endpoint = "stories-endpoint";
+      instance.storiesLoaded = true;
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.fetchStories();
+
+      instance.store.getState = () => ({
+        Sections: [{ id: "topstories", rows: response.recommendations }],
+        Prefs: { values: { showSponsored: true } },
+      });
+
+      globals.set("Math", {
+        random: () => 0.4,
+        min: Math.min,
+      });
+      instance.dispatchSpocDone = () => {};
+      instance.getPocketState = () => {};
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.store.dispatch);
+      let [action] = instance.store.dispatch.firstCall.args;
+
+      assert.equal(at.SECTION_UPDATE, action.type);
+      assert.equal(true, action.meta.skipMain);
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc1");
+      // Make sure spoc is marked as pinned so it doesn't get removed when preloaded tabs refresh
+      assert.equal(action.data.rows[2].pinned, true);
+
+      // Second new tab shouldn't trigger a section update event (spocsPerNewTab === 0.5)
+      globals.set("Math", {
+        random: () => 0.6,
+        min: Math.min,
+      });
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.store.dispatch);
+
+      globals.set("Math", {
+        random: () => 0.3,
+        min: Math.min,
+      });
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledTwice(instance.store.dispatch);
+      [action] = instance.store.dispatch.secondCall.args;
+      assert.equal(at.SECTION_UPDATE, action.type);
+      assert.equal(true, action.meta.skipMain);
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc1");
+      // Make sure spoc is marked as pinned so it doesn't get removed when preloaded tabs refresh
+      assert.equal(action.data.rows[2].pinned, true);
+    });
+    it("should delay inserting spoc if stories haven't been fetched", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      globals.set("Math", {
+        random: () => 0.4,
+        min: Math.min,
+        floor: Math.floor,
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        recommendations: [{ id: "rec1" }, { id: "rec2" }, { id: "rec3" }],
+        // Include one spoc with a expiration in the very distant future
+        spocs: [
+          { id: "spoc1", expiration_timestamp: 9999999999999 },
+          { id: "spoc2" },
+        ],
+      };
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.notCalled(instance.store.dispatch);
+      assert.equal(instance.contentUpdateQueue.length, 1);
+
+      instance.spocsPerNewTabs = 0.5;
+      instance.store.getState = () => ({
+        Sections: [{ id: "topstories", rows: response.recommendations }],
+        Prefs: { values: { showSponsored: true } },
+      });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+
+      await instance.onInit();
+      assert.equal(instance.contentUpdateQueue.length, 0);
+      assert.calledOnce(instance.store.dispatch);
+      let [action] = instance.store.dispatch.firstCall.args;
+      assert.equal(action.type, at.SECTION_UPDATE);
+    });
+    it("should not insert spoc if preffed off", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: false,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        spocs: [{ id: "spoc1" }, { id: "spoc2" }],
+      };
+      sinon.spy(instance, "maybeAddSpoc");
+      sinon.spy(instance, "shouldShowSpocs");
+
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.maybeAddSpoc);
+      assert.calledOnce(instance.shouldShowSpocs);
+      assert.notCalled(instance.store.dispatch);
+    });
+    it("should call dispatchSpocDone when calling maybeAddSpoc", async () => {
+      instance.dispatchSpocDone = sinon.spy();
+      instance.storiesLoaded = true;
+      await instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.dispatchSpocDone);
+      assert.calledWith(instance.dispatchSpocDone, {});
+    });
+    it("should fire POCKET_WAITING_FOR_SPOC action with false", () => {
+      instance.dispatchSpocDone({});
+      assert.calledOnce(instance.store.dispatch);
+      const [action] = instance.store.dispatch.firstCall.args;
+      assert.equal(action.type, "POCKET_WAITING_FOR_SPOC");
+      assert.equal(action.data, false);
+    });
+    it("should not insert spoc if user opted out", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        spocs: [{ id: "spoc1" }, { id: "spoc2" }],
+      };
+
+      instance.store.getState = () => ({
+        Sections: [{ id: "topstories", rows: response.recommendations }],
+        Prefs: { values: { showSponsored: false } },
+      });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.notCalled(instance.store.dispatch);
+    });
+    it("should not fail if there is no spoc", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      globals.set("Math", {
+        random: () => 0.4,
+        min: Math.min,
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        recommendations: [{ id: "rec1" }, { id: "rec2" }, { id: "rec3" }],
+      };
+
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.notCalled(instance.store.dispatch);
+    });
+    it("should record spoc/campaign impressions for frequency capping", async () => {
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      globals.set("Math", {
+        random: () => 0.4,
+        min: Math.min,
+        floor: Math.floor,
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        spocs: [
+          { id: 1, campaign_id: 5 },
+          { id: 4, campaign_id: 6 },
+        ],
+      };
+
+      instance._prefs = { get: pref => undefined, set: sinon.spy() };
+      instance.show_spocs = true;
+      instance.stories_endpoint = "stories-endpoint";
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.fetchStories();
+
+      let expectedPrefValue = JSON.stringify({ 5: [0] });
+      let expectedPrefValueCallTwo = JSON.stringify({ 2: 0, 3: 0 });
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 3 }, { id: 2 }, { id: 1 }],
+        },
+      });
+      assert.calledWith(
+        instance._prefs.set.firstCall,
+        SPOC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValue
+      );
+      assert.calledWith(
+        instance._prefs.set.secondCall,
+        REC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValueCallTwo
+      );
+
+      clock.tick(1);
+      instance._prefs.get = pref => expectedPrefValue;
+      let expectedPrefValueCallThree = JSON.stringify({ 5: [0, 1] });
+      let expectedPrefValueCallFour = JSON.stringify({ 2: 1, 3: 1, 5: [0] });
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 3 }, { id: 2 }, { id: 1 }],
+        },
+      });
+      assert.calledWith(
+        instance._prefs.set.thirdCall,
+        SPOC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValueCallThree
+      );
+      assert.calledWith(
+        instance._prefs.set.getCall(3),
+        REC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValueCallFour
+      );
+
+      clock.tick(1);
+      instance._prefs.get = pref => expectedPrefValueCallThree;
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 3 }, { id: 2 }, { id: 4 }],
+        },
+      });
+      assert.calledWith(
+        instance._prefs.set.getCall(4),
+        SPOC_IMPRESSION_TRACKING_PREF,
+        JSON.stringify({ 5: [0, 1], 6: [2] })
+      );
+      assert.calledWith(
+        instance._prefs.set.getCall(5),
+        REC_IMPRESSION_TRACKING_PREF,
+        JSON.stringify({ 2: 2, 3: 2, 5: [0, 1] })
+      );
+    });
+    it("should not record spoc/campaign impressions for non-view impressions", async () => {
+      let fetchStub = globals.sandbox.stub();
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        spocs: [
+          { id: 1, campaign_id: 5 },
+          { id: 4, campaign_id: 6 },
+        ],
+      };
+
+      instance._prefs = { get: pref => undefined, set: sinon.spy() };
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", click: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", block: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: { source: "TOP_STORIES", pocket: 0, tiles: [{ id: 1 }] },
+      });
+      assert.notCalled(instance._prefs.set);
+    });
+    it("should clean up spoc/campaign impressions", async () => {
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      instance._prefs = { get: pref => undefined, set: sinon.spy() };
+      instance.show_spocs = true;
+      instance.stories_endpoint = "stories-endpoint";
+
+      const response = {
+        settings: { spocsPerNewTabs: 0.5 },
+        spocs: [
+          { id: 1, campaign_id: 5 },
+          { id: 4, campaign_id: 6 },
+        ],
+      };
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.fetchStories();
+
+      // simulate impressions for campaign 5 and 6
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 3 }, { id: 2 }, { id: 1 }],
+        },
+      });
+      instance._prefs.get = pref =>
+        pref === SPOC_IMPRESSION_TRACKING_PREF && JSON.stringify({ 5: [0] });
+      instance.onAction({
+        type: at.TELEMETRY_IMPRESSION_STATS,
+        data: {
+          source: "TOP_STORIES",
+          tiles: [{ id: 3 }, { id: 2 }, { id: 4 }],
+        },
+      });
+
+      let expectedPrefValue = JSON.stringify({ 5: [0], 6: [0] });
+      assert.calledWith(
+        instance._prefs.set.thirdCall,
+        SPOC_IMPRESSION_TRACKING_PREF,
+        expectedPrefValue
+      );
+      instance._prefs.get = pref =>
+        pref === SPOC_IMPRESSION_TRACKING_PREF && expectedPrefValue;
+
+      // remove campaign 5 from response
+      const updatedResponse = {
+        settings: { spocsPerNewTabs: 1 },
+        spocs: [{ id: 4, campaign_id: 6 }],
+      };
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(updatedResponse),
+      });
+      await instance.fetchStories();
+
+      // should remove campaign 5 from pref as no longer active
+      assert.calledWith(
+        instance._prefs.set.getCall(4),
+        SPOC_IMPRESSION_TRACKING_PREF,
+        JSON.stringify({ 6: [0] })
+      );
+    });
+    it("should maintain frequency caps when inserting spocs", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      const response = {
+        settings: { spocsPerNewTabs: 1 },
+        recommendations: [{ guid: "rec1" }, { guid: "rec2" }, { guid: "rec3" }],
+        spocs: [
+          // Set spoc `expiration_timestamp`s in the very distant future to ensure they show up
+          {
+            id: "spoc1",
+            campaign_id: 1,
+            caps: { lifetime: 3, campaign: { count: 2, period: 3600 } },
+            expiration_timestamp: 999999999999,
+          },
+          {
+            id: "spoc2",
+            campaign_id: 2,
+            caps: { lifetime: 1 },
+            expiration_timestamp: 999999999999,
+          },
+        ],
+      };
+
+      instance.store.getState = () => ({
+        Sections: [{ id: "topstories", rows: response.recommendations }],
+        Prefs: { values: { showSponsored: true } },
+      });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+      instance.spocsPerNewTabs = 1;
+
+      clock.tick();
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      let [action] = instance.store.dispatch.firstCall.args;
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc1");
+      instance._prefs.get = pref => JSON.stringify({ 1: [1] });
+
+      clock.tick();
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      [action] = instance.store.dispatch.secondCall.args;
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc1");
+      instance._prefs.get = pref => JSON.stringify({ 1: [1, 2] });
+
+      // campaign 1 period frequency cap now reached (spoc 2 should be shown)
+      clock.tick();
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      [action] = instance.store.dispatch.thirdCall.args;
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc2");
+      instance._prefs.get = pref => JSON.stringify({ 1: [1, 2], 2: [3] });
+
+      // new campaign 1 period starting (spoc 1 sohuld be shown again)
+      clock.tick(2 * 60 * 60 * 1000);
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      [action] = instance.store.dispatch.lastCall.args;
+      assert.equal(action.data.rows[0].guid, "rec1");
+      assert.equal(action.data.rows[1].guid, "rec2");
+      assert.equal(action.data.rows[2].guid, "spoc1");
+      instance._prefs.get = pref =>
+        JSON.stringify({ 1: [1, 2, 7200003], 2: [3] });
+
+      // campaign 1 lifetime cap now reached (no spoc should be sent)
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.callCount(instance.store.dispatch, 4);
+    });
+    it("should maintain client-side MAX_LIFETIME_CAP", async () => {
+      let fetchStub = globals.sandbox.stub();
+      instance.dispatchSpocDone = () => {};
+      sectionsManagerStub.sections.set("topstories", {
+        options: {
+          show_spocs: true,
+          stories_endpoint: "stories-endpoint",
+        },
+      });
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      instance.getPocketState = () => {};
+      instance.dispatchPocketCta = () => {};
+
+      const response = {
+        settings: { spocsPerNewTabs: 1 },
+        recommendations: [{ guid: "rec1" }, { guid: "rec2" }, { guid: "rec3" }],
+        spocs: [{ id: "spoc1", campaign_id: 1, caps: { lifetime: 501 } }],
+      };
+
+      instance.store.getState = () => ({
+        Sections: [{ id: "topstories", rows: response.recommendations }],
+        Prefs: { values: { showSponsored: true } },
+      });
+      fetchStub.resolves({
+        ok: true,
+        status: 200,
+        json: () => Promise.resolve(response),
+      });
+      await instance.onInit();
+
+      instance._prefs.get = pref =>
+        JSON.stringify({ 1: [...Array(500).keys()] });
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.notCalled(instance.store.dispatch);
+    });
+  });
+  describe("#update", () => {
+    it("should fetch stories after update interval", async () => {
+      await instance.onInit();
+      sinon.spy(instance, "fetchStories");
+      await instance.onAction({ type: at.SYSTEM_TICK });
+      assert.notCalled(instance.fetchStories);
+
+      clock.tick(STORIES_UPDATE_TIME);
+      await instance.onAction({ type: at.SYSTEM_TICK });
+      assert.calledOnce(instance.fetchStories);
+    });
+    it("should fetch topics after update interval", async () => {
+      await instance.onInit();
+      sinon.spy(instance, "fetchTopics");
+      await instance.onAction({ type: at.SYSTEM_TICK });
+      assert.notCalled(instance.fetchTopics);
+
+      clock.tick(TOPICS_UPDATE_TIME);
+      await instance.onAction({ type: at.SYSTEM_TICK });
+      assert.calledOnce(instance.fetchTopics);
+    });
+    it("should return updated stories and topics on system tick", async () => {
+      await instance.onInit();
+      sinon.spy(instance, "dispatchUpdateEvent");
+      const stories = [{ guid: "rec1" }, { guid: "rec2" }, { guid: "rec3" }];
+      const topics = [
+        { name: "topic1", url: "url-topic1" },
+        { name: "topic2", url: "url-topic2" },
+      ];
+      clock.tick(TOPICS_UPDATE_TIME);
+      globals.sandbox.stub(instance, "fetchStories").resolves(stories);
+      globals.sandbox.stub(instance, "fetchTopics").resolves(topics);
+
+      await instance.onAction({ type: at.SYSTEM_TICK });
+
+      assert.calledOnce(instance.dispatchUpdateEvent);
+      assert.calledWith(instance.dispatchUpdateEvent, false, {
+        rows: [{ guid: "rec1" }, { guid: "rec2" }, { guid: "rec3" }],
+        topics: [
+          { name: "topic1", url: "url-topic1" },
+          { name: "topic2", url: "url-topic2" },
+        ],
+        read_more_endpoint: undefined,
+      });
+    });
+    it("should not call init and uninit if data doesn't match on options change ", () => {
+      sinon.spy(instance, "init");
+      sinon.spy(instance, "uninit");
+      instance.onAction({ type: at.SECTION_OPTIONS_CHANGED, data: "foo" });
+      assert.notCalled(sectionsManagerStub.disableSection);
+      assert.notCalled(sectionsManagerStub.enableSection);
+      assert.notCalled(instance.init);
+      assert.notCalled(instance.uninit);
+    });
+    it("should call init and uninit on options change", async () => {
+      sinon.stub(instance, "clearCache").returns(Promise.resolve());
+      sinon.spy(instance, "init");
+      sinon.spy(instance, "uninit");
+      await instance.onAction({
+        type: at.SECTION_OPTIONS_CHANGED,
+        data: "topstories",
+      });
+      assert.calledOnce(sectionsManagerStub.disableSection);
+      assert.calledOnce(sectionsManagerStub.enableSection);
+      assert.calledOnce(instance.clearCache);
+      assert.calledOnce(instance.init);
+      assert.calledOnce(instance.uninit);
+    });
+    it("should set LastUpdated to 0 on init", async () => {
+      instance.storiesLastUpdated = 1;
+      instance.topicsLastUpdated = 1;
+
+      await instance.onInit();
+      assert.equal(instance.storiesLastUpdated, 0);
+      assert.equal(instance.topicsLastUpdated, 0);
+    });
+    it("should filter spocs when link is blocked", async () => {
+      instance.spocs = [{ url: "not_blocked" }, { url: "blocked" }];
+      await instance.onAction({
+        type: at.PLACES_LINK_BLOCKED,
+        data: { url: "blocked" },
+      });
+
+      assert.deepEqual(instance.spocs, [{ url: "not_blocked" }]);
+    });
+  });
+  describe("#loadCachedData", () => {
+    it("should update section with cached stories and topics if available", async () => {
+      sectionsManagerStub.sections.set("topstories", {
+        options: { stories_referrer: "referrer" },
+      });
+      const stories = {
+        _timestamp: 123,
+        recommendations: [
+          {
+            id: "1",
+            title: "title",
+            excerpt: "description",
+            image_src: "image-url",
+            url: "rec-url",
+            published_timestamp: "123",
+            context: "trending",
+            icon: "icon",
+            item_score: 0.98,
+          },
+        ],
+      };
+      const transformedStories = [
+        {
+          guid: "1",
+          type: "now",
+          title: "title",
+          context: "trending",
+          icon: "icon",
+          description: "description",
+          image: "image-url",
+          referrer: "referrer",
+          url: "rec-url",
+          hostname: "rec-url",
+          score: 0.98,
+          spoc_meta: {},
+        },
+      ];
+      const topics = {
+        _timestamp: 123,
+        topics: [
+          { name: "topic1", url: "url-topic1" },
+          { name: "topic2", url: "url-topic2" },
+        ],
+      };
+      instance.cache.get = () => ({ stories, topics });
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+
+      await instance.onInit();
+      assert.calledOnce(sectionsManagerStub.updateSection);
+      assert.calledWith(sectionsManagerStub.updateSection, SECTION_ID, {
+        rows: transformedStories,
+        topics: topics.topics,
+        read_more_endpoint: undefined,
+      });
+    });
+    it("should NOT update section if there is no cached data", async () => {
+      instance.cache.get = () => ({});
+      globals.set("NewTabUtils", {
+        blockedLinks: { isBlocked: globals.sandbox.spy() },
+      });
+      await instance.loadCachedData();
+      assert.notCalled(sectionsManagerStub.updateSection);
+    });
+    it("should use store rows if no stories sent to doContentUpdate", async () => {
+      instance.store = {
+        getState() {
+          return {
+            Sections: [{ id: "topstories", rows: [1, 2, 3] }],
+          };
+        },
+      };
+      sinon.spy(instance, "dispatchUpdateEvent");
+
+      instance.doContentUpdate({}, false);
+
+      assert.calledOnce(instance.dispatchUpdateEvent);
+      assert.calledWith(instance.dispatchUpdateEvent, false, {
+        rows: [1, 2, 3],
+      });
+    });
+    it("should broadcast in doContentUpdate when updating from cache", async () => {
+      sectionsManagerStub.sections.set("topstories", {
+        options: { stories_referrer: "referrer" },
+      });
+      globals.set("NewTabUtils", { blockedLinks: { isBlocked: () => {} } });
+      const stories = { recommendations: [{}] };
+      const topics = { topics: [{}] };
+      sinon.spy(instance, "doContentUpdate");
+      instance.cache.get = () => ({ stories, topics });
+      await instance.onInit();
+      assert.calledOnce(instance.doContentUpdate);
+      assert.calledWith(
+        instance.doContentUpdate,
+        {
+          stories: [
+            {
+              context: undefined,
+              description: undefined,
+              guid: undefined,
+              hostname: undefined,
+              icon: undefined,
+              image: undefined,
+              referrer: "referrer",
+              score: 1,
+              spoc_meta: {},
+              title: undefined,
+              type: "trending",
+              url: undefined,
+            },
+          ],
+          topics: [{}],
+        },
+        true
+      );
+    });
+  });
+  describe("#pocket", () => {
+    it("should call getPocketState when hitting NEW_TAB_REHYDRATED", () => {
+      instance.getPocketState = sinon.spy();
+      instance.onAction({
+        type: at.NEW_TAB_REHYDRATED,
+        meta: { fromTarget: {} },
+      });
+      assert.calledOnce(instance.getPocketState);
+      assert.calledWith(instance.getPocketState, {});
+    });
+    it("should call dispatch in getPocketState", () => {
+      const isUserLoggedIn = sinon.spy();
+      globals.set("pktApi", { isUserLoggedIn });
+      instance.getPocketState({});
+      assert.calledOnce(instance.store.dispatch);
+      const [action] = instance.store.dispatch.firstCall.args;
+      assert.equal(action.type, "POCKET_LOGGED_IN");
+      assert.calledOnce(isUserLoggedIn);
+    });
+    it("should call dispatchPocketCta when hitting onInit", async () => {
+      instance.dispatchPocketCta = sinon.spy();
+      await instance.onInit();
+      assert.calledOnce(instance.dispatchPocketCta);
+      assert.calledWith(
+        instance.dispatchPocketCta,
+        JSON.stringify({
+          cta_button: "",
+          cta_text: "",
+          cta_url: "",
+          use_cta: false,
+        }),
+        false
+      );
+    });
+    it("should call dispatch in dispatchPocketCta", () => {
+      instance.dispatchPocketCta(JSON.stringify({ use_cta: true }), false);
+      assert.calledOnce(instance.store.dispatch);
+      const [action] = instance.store.dispatch.firstCall.args;
+      assert.equal(action.type, "POCKET_CTA");
+      assert.equal(action.data.use_cta, true);
+    });
+    it("should call dispatchPocketCta with a pocketCta pref change", () => {
+      instance.dispatchPocketCta = sinon.spy();
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: {
+          name: "pocketCta",
+          value: JSON.stringify({
+            cta_button: "",
+            cta_text: "",
+            cta_url: "",
+            use_cta: false,
+          }),
+        },
+      });
+      assert.calledOnce(instance.dispatchPocketCta);
+      assert.calledWith(
+        instance.dispatchPocketCta,
+        JSON.stringify({
+          cta_button: "",
+          cta_text: "",
+          cta_url: "",
+          use_cta: false,
+        }),
+        true
+      );
+    });
+  });
+  it("should call uninit and init on disabling of showSponsored pref", async () => {
+    sinon.stub(instance, "clearCache").returns(Promise.resolve());
+    sinon.stub(instance, "uninit");
+    sinon.stub(instance, "init");
+    await instance.onAction({
+      type: at.PREF_CHANGED,
+      data: { name: "showSponsored", value: false },
+    });
+    assert.calledOnce(instance.clearCache);
+    assert.calledOnce(instance.uninit);
+    assert.calledOnce(instance.init);
+  });
+});