diff options
Diffstat (limited to 'browser/components/topsites')
| -rw-r--r-- | browser/components/topsites/TopSites.sys.mjs | 2039 | ||||
| -rw-r--r-- | browser/components/topsites/moz.build | 12 | ||||
| -rw-r--r-- | browser/components/topsites/test/unit/test_top_sites.js | 3571 | ||||
| -rw-r--r-- | browser/components/topsites/test/unit/xpcshell.toml | 4 |
4 files changed, 5626 insertions, 0 deletions
diff --git a/browser/components/topsites/TopSites.sys.mjs b/browser/components/topsites/TopSites.sys.mjs new file mode 100644 index 0000000000..736a079f34 --- /dev/null +++ b/browser/components/topsites/TopSites.sys.mjs @@ -0,0 +1,2039 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.mjs"; +import { TippyTopProvider } from "resource://activity-stream/lib/TippyTopProvider.sys.mjs"; +import { + insertPinned, + TOP_SITES_MAX_SITES_PER_ROW, +} from "resource://activity-stream/common/Reducers.sys.mjs"; +import { Dedupe } from "resource://activity-stream/common/Dedupe.sys.mjs"; +import { shortURL } from "resource://activity-stream/lib/ShortURL.sys.mjs"; +import { getDefaultOptions } from "resource://activity-stream/lib/ActivityStreamStorage.sys.mjs"; + +import { + CUSTOM_SEARCH_SHORTCUTS, + SEARCH_SHORTCUTS_EXPERIMENT, + SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF, + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + checkHasSearchEngine, + getSearchProvider, + getSearchFormURL, +} from "resource://activity-stream/lib/SearchShortcuts.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + FilterAdult: "resource://activity-stream/lib/FilterAdult.sys.mjs", + LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs", + Screenshots: "resource://activity-stream/lib/Screenshots.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("TopSites"); +}); + +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +ChromeUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +const DEFAULT_SITES_PREF = "default.sites"; +const SHOWN_ON_NEWTAB_PREF = "feeds.topsites"; +export const DEFAULT_TOP_SITES = []; +const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages) +const MIN_FAVICON_SIZE = 96; +const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"]; +const PINNED_FAVICON_PROPS_TO_MIGRATE = [ + "favicon", + "faviconRef", + "faviconSize", +]; +const SECTION_ID = "topsites"; +const ROWS_PREF = "topSitesRows"; +const SHOW_SPONSORED_PREF = "showSponsoredTopSites"; +// The default total number of sponsored top sites to fetch from Contile +// and Pocket. +const MAX_NUM_SPONSORED = 2; +// Nimbus variable for the total number of sponsored top sites including +// both Contile and Pocket sources. +// The default will be `MAX_NUM_SPONSORED` if this variable is unspecified. +const NIMBUS_VARIABLE_MAX_SPONSORED = "topSitesMaxSponsored"; +// Nimbus variable to allow more than two sponsored tiles from Contile to be +//considered for Top Sites. +const NIMBUS_VARIABLE_ADDITIONAL_TILES = + "topSitesUseAdditionalTilesFromContile"; +// Nimbus variable to enable the SOV feature for sponsored tiles. +const NIMBUS_VARIABLE_CONTILE_SOV_ENABLED = "topSitesContileSovEnabled"; +// Nimbu variable for the total number of sponsor topsite that come from Contile +// The default will be `CONTILE_MAX_NUM_SPONSORED` if variable is unspecified. +const NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED = "topSitesContileMaxSponsored"; + +// Search experiment stuff +const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile"; +const SEARCH_FILTERS = [ + "google", + "search.yahoo", + "yahoo", + "bing", + "ask", + "duckduckgo", +]; + +const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting"; +const DEFAULT_SITES_OVERRIDE_PREF = + "browser.newtabpage.activity-stream.default.sites"; +const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment."; + +// Mozilla Tiles Service (Contile) prefs +// Nimbus variable for the Contile integration. It falls back to the pref: +// `browser.topsites.contile.enabled`. +const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled"; +const NIMBUS_VARIABLE_CONTILE_POSITIONS = "contileTopsitesPositions"; +const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint"; +const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes +// The maximum number of sponsored top sites to fetch from Contile. +const CONTILE_MAX_NUM_SPONSORED = 2; +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; +const CONTILE_CACHE_PREF = "browser.topsites.contile.cachedTiles"; +const CONTILE_CACHE_VALID_FOR_PREF = "browser.topsites.contile.cacheValidFor"; +const CONTILE_CACHE_LAST_FETCH_PREF = "browser.topsites.contile.lastFetch"; + +// Partners of sponsored tiles. +const SPONSORED_TILE_PARTNER_AMP = "amp"; +const SPONSORED_TILE_PARTNER_MOZ_SALES = "moz-sales"; +const SPONSORED_TILE_PARTNERS = new Set([ + SPONSORED_TILE_PARTNER_AMP, + SPONSORED_TILE_PARTNER_MOZ_SALES, +]); + +const DISPLAY_FAIL_REASON_OVERSOLD = "oversold"; +const DISPLAY_FAIL_REASON_DISMISSED = "dismissed"; +const DISPLAY_FAIL_REASON_UNRESOLVED = "unresolved"; + +function getShortURLForCurrentSearch() { + const url = shortURL({ url: Services.search.defaultEngine.searchForm }); + return url; +} + +class TopSitesTelemetry { + constructor() { + this.allSponsoredTiles = {}; + this.sponsoredTilesConfigured = 0; + } + + _tileProviderForTiles(tiles) { + // Assumption: the list of tiles is from a single provider + return tiles && tiles.length ? this._tileProvider(tiles[0]) : null; + } + + _tileProvider(tile) { + return tile.partner || SPONSORED_TILE_PARTNER_AMP; + } + + _buildPropertyKey(tile) { + let provider = this._tileProvider(tile); + return provider + shortURL(tile); + } + + // Returns an array of strings indicating the property name (based on the + // provider and brand) of tiles that have been filtered e.g. ["moz-salesbrand1"] + // currentTiles: The list of tiles remaining and may be displayed in new tab. + // this.allSponsoredTiles: The original list of tiles set via setTiles prior to any filtering + // The returned list indicated the difference between these two lists (excluding any previously filtered tiles). + _getFilteredTiles(currentTiles) { + let notPreviouslyFilteredTiles = Object.assign( + {}, + ...Object.entries(this.allSponsoredTiles) + .filter( + ([, v]) => + v.display_fail_reason === null || + v.display_fail_reason === undefined + ) + .map(([k, v]) => ({ [k]: v })) + ); + + // Get the property names of the newly filtered list. + let remainingTiles = currentTiles.map(el => { + return this._buildPropertyKey(el); + }); + + // Get the property names of the tiles that were filtered. + let tilesToUpdate = Object.keys(notPreviouslyFilteredTiles).filter( + element => !remainingTiles.includes(element) + ); + return tilesToUpdate; + } + + setSponsoredTilesConfigured() { + const maxSponsored = + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_MAX_SPONSORED + ) ?? MAX_NUM_SPONSORED; + + this.sponsoredTilesConfigured = maxSponsored; + Glean.topsites.sponsoredTilesConfigured.set(maxSponsored); + } + + clearTilesForProvider(provider) { + Object.entries(this.allSponsoredTiles) + .filter(([k]) => k.startsWith(provider)) + .map(([k]) => delete this.allSponsoredTiles[k]); + } + + _getAdvertiser(tile) { + let label = tile.label || null; + let title = tile.title || null; + + return label ?? title ?? shortURL(tile); + } + + setTiles(tiles) { + // Assumption: the list of tiles is from a single provider, + // should be called once per tile source. + if (tiles && tiles.length) { + let tile_provider = this._tileProviderForTiles(tiles); + this.clearTilesForProvider(tile_provider); + + for (let sponsoredTile of tiles) { + this.allSponsoredTiles[this._buildPropertyKey(sponsoredTile)] = { + advertiser: this._getAdvertiser(sponsoredTile).toLowerCase(), + provider: tile_provider, + display_position: null, + display_fail_reason: null, + }; + } + } + } + + _setDisplayFailReason(filteredTiles, reason) { + for (let tile of filteredTiles) { + if (tile in this.allSponsoredTiles) { + let tileToUpdate = this.allSponsoredTiles[tile]; + tileToUpdate.display_position = null; + tileToUpdate.display_fail_reason = reason; + } + } + } + + determineFilteredTilesAndSetToOversold(nonOversoldTiles) { + let filteredTiles = this._getFilteredTiles(nonOversoldTiles); + this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_OVERSOLD); + } + + determineFilteredTilesAndSetToDismissed(nonDismissedTiles) { + let filteredTiles = this._getFilteredTiles(nonDismissedTiles); + this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_DISMISSED); + } + + _setTilePositions(currentTiles) { + // This function performs many loops over a small dataset. The size of + // dataset is limited by the number of sponsored tiles displayed on + // the newtab instance. + if (this.allSponsoredTiles) { + let tilePositionsAssigned = []; + // processing the currentTiles parameter, assigns a position to the + // corresponding property in this.allSponsoredTiles + currentTiles.forEach(item => { + let tile = this.allSponsoredTiles[this._buildPropertyKey(item)]; + if ( + tile && + (tile.display_fail_reason === undefined || + tile.display_fail_reason === null) + ) { + tile.display_position = item.sponsored_position; + // Track assigned tile slots. + tilePositionsAssigned.push(item.sponsored_position); + } + }); + + // Need to check if any objects in this.allSponsoredTiles do not + // have either a display_fail_reason or a display_position set. + // This can happen if the tiles list was updated before the + // metric is written to Glean. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1877197 + let tilesMissingPosition = []; + Object.keys(this.allSponsoredTiles).forEach(property => { + let tile = this.allSponsoredTiles[property]; + if (!tile.display_fail_reason && !tile.display_position) { + tilesMissingPosition.push(property); + } + }); + + if (tilesMissingPosition.length) { + // Determine if any available slots exist based on max number of tiles + // and the list of tiles already used and assign to a tile with missing + // value. + for (let i = 1; i <= this.sponsoredTilesConfigured; i++) { + if (!tilePositionsAssigned.includes(i)) { + let tileProperty = tilesMissingPosition.shift(); + this.allSponsoredTiles[tileProperty].display_position = i; + } + } + } + + // At this point we might still have a few unresolved states. These + // rows will be tagged with a display_fail_reason `unresolved`. + this._detectErrorConditionAndSetUnresolved(); + } + } + + // Checks the data for inconsistent state and updates the display_fail_reason + _detectErrorConditionAndSetUnresolved() { + Object.keys(this.allSponsoredTiles).forEach(property => { + let tile = this.allSponsoredTiles[property]; + if ( + (!tile.display_fail_reason && !tile.display_position) || + (tile.display_fail_reason && tile.display_position) + ) { + tile.display_position = null; + tile.display_fail_reason = DISPLAY_FAIL_REASON_UNRESOLVED; + } + }); + } + + finalizeNewtabPingFields(currentTiles) { + this._setTilePositions(currentTiles); + Glean.topsites.sponsoredTilesReceived.set( + JSON.stringify({ + sponsoredTilesReceived: Object.values(this.allSponsoredTiles), + }) + ); + } +} + +export class ContileIntegration { + constructor(topSitesFeed) { + this._topSitesFeed = topSitesFeed; + this._lastPeriodicUpdate = 0; + this._sites = []; + // The Share-of-Voice object managed by Shepherd and sent via Contile. + this._sov = null; + } + + get sites() { + return this._sites; + } + + get sov() { + return this._sov; + } + + periodicUpdate() { + let now = Date.now(); + if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) { + this._lastPeriodicUpdate = now; + this.refresh(); + } + } + + async refresh() { + let updateDefaultSites = await this._fetchSites(); + await this._topSitesFeed.allocatePositions(); + if (updateDefaultSites) { + this._topSitesFeed._readDefaults(); + } + } + + /** + * Clear Contile Cache Prefs. + */ + _resetContileCachePrefs() { + Services.prefs.clearUserPref(CONTILE_CACHE_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_LAST_FETCH_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_VALID_FOR_PREF); + } + + /** + * Filter the tiles whose sponsor is on the Top Sites sponsor blocklist. + * + * @param {Array} tiles + * An array of the tile objects + */ + _filterBlockedSponsors(tiles) { + const blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + return tiles.filter(tile => !blocklist.includes(shortURL(tile))); + } + + /** + * Calculate the time Contile response is valid for based on cache-control header + * + * @param {string} cacheHeader + * string value of the Contile resposne cache-control header + */ + _extractCacheValidFor(cacheHeader) { + if (!cacheHeader) { + lazy.log.warn("Contile response cache control header is empty"); + return 0; + } + const [, staleIfError] = cacheHeader.match(/stale-if-error=\s*([0-9]+)/i); + const [, maxAge] = cacheHeader.match(/max-age=\s*([0-9]+)/i); + const validFor = + Number.parseInt(staleIfError, 10) + Number.parseInt(maxAge, 10); + return isNaN(validFor) ? 0 : validFor; + } + + /** + * Load Tiles from Contile Cache Prefs + */ + _loadTilesFromCache() { + lazy.log.info("Contile client is trying to load tiles from local cache."); + const now = Math.round(Date.now() / 1000); + const lastFetch = Services.prefs.getIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + 0 + ); + const validFor = Services.prefs.getIntPref(CONTILE_CACHE_VALID_FOR_PREF, 0); + this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured(); + if (now <= lastFetch + validFor) { + try { + let cachedTiles = JSON.parse( + Services.prefs.getStringPref(CONTILE_CACHE_PREF) + ); + this._topSitesFeed._telemetryUtility.setTiles(cachedTiles); + cachedTiles = this._filterBlockedSponsors(cachedTiles); + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed( + cachedTiles + ); + this._sites = cachedTiles; + lazy.log.info("Local cache loaded."); + return true; + } catch (error) { + lazy.log.warn(`Failed to load tiles from local cache: ${error}.`); + return false; + } + } + + return false; + } + + /** + * Determine number of Tiles to get from Contile + */ + _getMaxNumFromContile() { + return ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) ?? CONTILE_MAX_NUM_SPONSORED + ); + } + + async _fetchSites() { + if ( + !lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) || + !this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF] + ) { + if (this._sites.length) { + this._sites = []; + return true; + } + return false; + } + try { + let url = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF); + const response = await this._topSitesFeed.fetch(url, { + credentials: "omit", + }); + if (!response.ok) { + lazy.log.warn( + `Contile endpoint returned unexpected status: ${response.status}` + ); + if (response.status === 304 || response.status >= 500) { + return this._loadTilesFromCache(); + } + } + + const lastFetch = Math.round(Date.now() / 1000); + Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, lastFetch); + this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured(); + + // Contile returns 204 indicating there is no content at the moment. + // If this happens, it will clear `this._sites` reset the cached tiles + // to an empty array. + if (response.status === 204) { + this._topSitesFeed._telemetryUtility.clearTilesForProvider( + SPONSORED_TILE_PARTNER_AMP + ); + if (this._sites.length) { + this._sites = []; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + return true; + } + return false; + } + const body = await response.json(); + + if (body?.sov) { + this._sov = JSON.parse(atob(body.sov)); + } + if (body?.tiles && Array.isArray(body.tiles)) { + const useAdditionalTiles = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_ADDITIONAL_TILES + ); + + const maxNumFromContile = this._getMaxNumFromContile(); + + let { tiles } = body; + this._topSitesFeed._telemetryUtility.setTiles(tiles); + if ( + useAdditionalTiles !== undefined && + !useAdditionalTiles && + tiles.length > maxNumFromContile + ) { + tiles.length = maxNumFromContile; + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold( + tiles + ); + } + tiles = this._filterBlockedSponsors(tiles); + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed( + tiles + ); + if (tiles.length > maxNumFromContile) { + lazy.log.info("Remove unused links from Contile"); + tiles.length = maxNumFromContile; + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold( + tiles + ); + } + this._sites = tiles; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + Services.prefs.setIntPref( + CONTILE_CACHE_VALID_FOR_PREF, + this._extractCacheValidFor( + response.headers.get("cache-control") || + response.headers.get("Cache-Control") + ) + ); + + return true; + } + } catch (error) { + lazy.log.warn( + `Failed to fetch data from Contile server: ${error.message}` + ); + return this._loadTilesFromCache(); + } + return false; + } +} + +class _TopSites { + constructor() { + this._telemetryUtility = new TopSitesTelemetry(); + this._contile = new ContileIntegration(this); + this._tippyTopProvider = new TippyTopProvider(); + ChromeUtils.defineLazyGetter( + this, + "_currentSearchHostname", + getShortURLForCurrentSearch + ); + this.dedupe = new Dedupe(this._dedupeKey); + this.frecentCache = new lazy.LinksCache( + lazy.NewTabUtils.activityStreamLinks, + "getTopSites", + CACHED_LINK_PROPS_TO_MIGRATE, + (oldOptions, newOptions) => + // Refresh if no old options or requesting more items + !(oldOptions.numItems >= newOptions.numItems) + ); + this.pinnedCache = new lazy.LinksCache( + lazy.NewTabUtils.pinnedLinks, + "links", + [...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE] + ); + lazy.PageThumbs.addExpirationFilter(this); + this._nimbusChangeListener = this._nimbusChangeListener.bind(this); + } + + _nimbusChangeListener(event, reason) { + // The Nimbus API current doesn't specify the changed variable(s) in the + // listener callback, so we have to refresh unconditionally on every change + // of the `newtab` feature. It should be a manageable overhead given the + // current update cadence (6 hours) of Nimbus. + // + // Skip the experiment and rollout loading reasons since this feature has + // `isEarlyStartup` enabled, the feature variables are already available + // before the experiment or rollout loads. + if ( + !["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason) + ) { + this._contile.refresh(); + } + } + + init() { + // If the feed was previously disabled PREFS_INITIAL_VALUES was never received + this._readDefaults({ isStartup: true }); + this._storage = this.store.dbStorage.getDbTable("sectionPrefs"); + this._contile.refresh(); + Services.obs.addObserver(this, "browser-search-engine-modified"); + Services.obs.addObserver(this, "browser-region-updated"); + Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener); + } + + uninit() { + lazy.PageThumbs.removeExpirationFilter(this); + Services.obs.removeObserver(this, "browser-search-engine-modified"); + Services.obs.removeObserver(this, "browser-region-updated"); + Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.offUpdate(this._nimbusChangeListener); + } + + observe(subj, topic, data) { + switch (topic) { + case "browser-search-engine-modified": + // We should update the current top sites if the search engine has been changed since + // the search engine that gets filtered out of top sites has changed. + // We also need to drop search shortcuts when their engine gets removed / hidden. + if ( + data === "engine-default" && + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] + ) { + delete this._currentSearchHostname; + this._currentSearchHostname = getShortURLForCurrentSearch(); + } + this.refresh({ broadcast: true }); + break; + case "browser-region-updated": + this._readDefaults(); + break; + case "nsPref:changed": + if ( + data === REMOTE_SETTING_DEFAULTS_PREF || + data === DEFAULT_SITES_OVERRIDE_PREF || + data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH) + ) { + this._readDefaults(); + } + break; + } + } + + _dedupeKey(site) { + return site && site.hostname; + } + + /** + * _readDefaults - sets DEFAULT_TOP_SITES + */ + async _readDefaults({ isStartup = false } = {}) { + this._useRemoteSetting = false; + + if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) { + this.refreshDefaults( + this.store.getState().Prefs.values[DEFAULT_SITES_PREF], + { isStartup } + ); + return; + } + + // Try using default top sites from enterprise policies or tests. The pref + // is locked when set via enterprise policy. Tests have no default sites + // unless they set them via this pref. + if ( + Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) || + Cu.isInAutomation + ) { + let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, ""); + this.refreshDefaults(sites, { isStartup }); + return; + } + + // Clear out the array of any previous defaults. + DEFAULT_TOP_SITES.length = 0; + + // Read defaults from contile. + const contileEnabled = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ); + + // Keep the number of positions in the array in sync with CONTILE_MAX_NUM_SPONSORED. + // sponsored_position is a 1-based index, and contilePositions is a 0-based index, + // so we need to add 1 to each of these. + // Also currently this does not work with SOV. + let contilePositions = lazy.NimbusFeatures.pocketNewtab + .getVariable(NIMBUS_VARIABLE_CONTILE_POSITIONS) + ?.split(",") + .map(item => parseInt(item, 10) + 1) + .filter(item => !Number.isNaN(item)); + if (!contilePositions || contilePositions.length === 0) { + contilePositions = [1, 2]; + } + + let hasContileTiles = false; + if (contileEnabled) { + let contilePositionIndex = 0; + // We need to loop through potential spocs and set their positions. + // If we run out of spocs or positions, we stop. + // First, we need to know which array is shortest. This is our exit condition. + const minLength = Math.min( + contilePositions.length, + this._contile.sites.length + ); + // Loop until we run out of spocs or positions. + for (let i = 0; i < minLength; i++) { + let site = this._contile.sites[i]; + let hostname = shortURL(site); + let link = { + isDefault: true, + url: site.url, + hostname, + sendAttributionRequest: false, + label: site.name, + show_sponsored_label: hostname !== "yandex", + sponsored_position: contilePositions[contilePositionIndex++], + sponsored_click_url: site.click_url, + sponsored_impression_url: site.impression_url, + sponsored_tile_id: site.id, + partner: SPONSORED_TILE_PARTNER_AMP, + }; + if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) { + // Only use the image from Contile if it's hi-res, otherwise, fallback + // to the built-in favicons. + link.favicon = site.image_url; + link.faviconSize = site.image_size; + } + DEFAULT_TOP_SITES.push(link); + } + hasContileTiles = contilePositionIndex > 0; + //This is to catch where we receive 3 tiles but reduce to 2 early in the filtering, before blocked list applied. + this._telemetryUtility.determineFilteredTilesAndSetToOversold( + DEFAULT_TOP_SITES + ); + } + + // Read defaults from remote settings. + this._useRemoteSetting = true; + let remoteSettingData = await this._getRemoteConfig(); + + const sponsoredBlocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + + for (let siteData of remoteSettingData) { + let hostname = shortURL(siteData); + // Drop default sites when Contile already provided a sponsored one with + // the same host name. + if ( + contileEnabled && + DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1 + ) { + continue; + } + // Also drop those sponsored sites that were blocked by the user before + // with the same hostname. + if ( + siteData.sponsored_position && + sponsoredBlocklist.includes(hostname) + ) { + continue; + } + let link = { + isDefault: true, + url: siteData.url, + hostname, + sendAttributionRequest: !!siteData.send_attribution_request, + }; + if (siteData.url_urlbar_override) { + link.url_urlbar = siteData.url_urlbar_override; + } + if (siteData.title) { + link.label = siteData.title; + } + if (siteData.search_shortcut) { + link = await this.topSiteToSearchTopSite(link); + } else if (siteData.sponsored_position) { + if (contileEnabled && hasContileTiles) { + continue; + } + const { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + } = siteData; + link = { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + show_sponsored_label: link.hostname !== "yandex", + ...link, + }; + } + DEFAULT_TOP_SITES.push(link); + } + + this.refresh({ broadcast: true, isStartup }); + } + + refreshDefaults(sites, { isStartup = false } = {}) { + // Clear out the array of any previous defaults + DEFAULT_TOP_SITES.length = 0; + + // Add default sites if any based on the pref + if (sites) { + for (const url of sites.split(",")) { + const site = { + isDefault: true, + url, + }; + site.hostname = shortURL(site); + DEFAULT_TOP_SITES.push(site); + } + } + + this.refresh({ broadcast: true, isStartup }); + } + + async _getRemoteConfig(firstTime = true) { + if (!this._remoteConfig) { + this._remoteConfig = await lazy.RemoteSettings("top-sites"); + this._remoteConfig.on("sync", () => { + this._readDefaults(); + }); + } + + let result = []; + let failed = false; + try { + result = await this._remoteConfig.get(); + } catch (ex) { + console.error(ex); + failed = true; + } + if (!result.length) { + console.error("Received empty top sites configuration!"); + failed = true; + } + // If we failed, or the result is empty, try loading from the local dump. + if (firstTime && failed) { + await this._remoteConfig.db.clear(); + // Now call this again. + return this._getRemoteConfig(false); + } + + // Sort sites based on the "order" attribute. + result.sort((a, b) => a.order - b.order); + + result = result.filter(topsite => { + // Filter by region. + if (topsite.exclude_regions?.includes(lazy.Region.home)) { + return false; + } + if ( + topsite.include_regions?.length && + !topsite.include_regions.includes(lazy.Region.home) + ) { + return false; + } + + // Filter by locale. + if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) { + return false; + } + if ( + topsite.include_locales?.length && + !topsite.include_locales.includes(Services.locale.appLocaleAsBCP47) + ) { + return false; + } + + // Filter by experiment. + // Exclude this top site if any of the specified experiments are running. + if ( + topsite.exclude_experiments?.some(experimentID => + Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + // Exclude this top site if none of the specified experiments are running. + if ( + topsite.include_experiments?.length && + topsite.include_experiments.every( + experimentID => + !Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + + return true; + }); + + return result; + } + + filterForThumbnailExpiration(callback) { + const { rows } = this.store.getState().TopSites; + callback( + rows.reduce((acc, site) => { + acc.push(site.url); + if (site.customScreenshotURL) { + acc.push(site.customScreenshotURL); + } + return acc; + }, []) + ); + } + + /** + * shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine? + * + * @param {string} hostname a top site hostname, such as "amazon" or "foo" + * @returns {bool} + */ + shouldFilterSearchTile(hostname) { + if ( + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] && + (SEARCH_FILTERS.includes(hostname) || + hostname === this._currentSearchHostname) + ) { + return true; + } + return false; + } + + /** + * _maybeInsertSearchShortcuts - if the search shortcuts experiment is running, + * insert search shortcuts if needed + * + * @param {Array} plainPinnedSites (from the pinnedSitesCache) + * @returns {boolean} Did we insert any search shortcuts? + */ + async _maybeInsertSearchShortcuts(plainPinnedSites) { + // Only insert shortcuts if the experiment is running + if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + // We don't want to insert shortcuts we've previously inserted + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",") + .filter(s => s); // Filter out empty strings + const newInsertedShortcuts = []; + + let shouldPin = this._useRemoteSetting + ? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname) + : this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(","); + shouldPin = shouldPin + .map(getSearchProvider) + .filter(s => s && s.shortURL !== this._currentSearchHostname); + + // If we've previously inserted all search shortcuts return early + if ( + shouldPin.every(shortcut => + prevInsertedShortcuts.includes(shortcut.shortURL) + ) + ) { + return false; + } + + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + + // The plainPinnedSites array is populated with pinned sites at their + // respective indices, and null everywhere else, but is not always the + // right length + const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0); + const pinnedSites = [...plainPinnedSites].concat( + Array(emptySlots).fill(null) + ); + + const tryToInsertSearchShortcut = async shortcut => { + const nextAvailable = pinnedSites.indexOf(null); + // Only add a search shortcut if the site isn't already pinned, we + // haven't previously inserted it, there's space to pin it, and the + // search engine is available in Firefox + if ( + !pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) && + !prevInsertedShortcuts.includes(shortcut.shortURL) && + nextAvailable > -1 && + (await checkHasSearchEngine(shortcut.keyword)) + ) { + const site = await this.topSiteToSearchTopSite({ url: shortcut.url }); + this._pinSiteAt(site, nextAvailable); + pinnedSites[nextAvailable] = site; + newInsertedShortcuts.push(shortcut.shortURL); + } + }; + + for (let shortcut of shouldPin) { + await tryToInsertSearchShortcut(shortcut); + } + + if (newInsertedShortcuts.length) { + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.concat(newInsertedShortcuts).join(",") + ) + ); + return true; + } + } + + return false; + } + + /** + * This thin wrapper around global.fetch makes it easier for us to write + * automated tests that simulate responses from this fetch. + */ + fetch(...args) { + return fetch(...args); + } + + /** + * Fetch topsites spocs from the DiscoveryStream feed. + * + * @returns {Array} An array of sponsored tile objects. + */ + fetchDiscoveryStreamSpocs() { + let sponsored = []; + const { DiscoveryStream } = this.store.getState(); + if (DiscoveryStream) { + const discoveryStreamSpocs = + DiscoveryStream.spocs.data["sponsored-topsites"]?.items || []; + // Find the first component of a type and remove it from layout + const findSponsoredTopsitesPositions = name => { + for (const row of DiscoveryStream.layout) { + for (const component of row.components) { + if (component.placement?.name === name) { + return component.spocs.positions; + } + } + } + return null; + }; + + // Get positions from layout for now. This could be improved if we store position data in state. + const discoveryStreamSpocPositions = + findSponsoredTopsitesPositions("sponsored-topsites"); + + if (discoveryStreamSpocPositions?.length) { + function reformatImageURL(url, width, height) { + // Change the image URL to request a size tailored for the parent container width + // Also: force JPEG, quality 60, no upscaling, no EXIF data + // Uses Thumbor: https://thumbor.readthedocs.io/en/latest/usage.html + // For now we wrap this in single quotes because this is being used in a url() css rule, and otherwise would cause a parsing error. + return `'https://img-getpocket.cdn.mozilla.net/${width}x${height}/filters:format(jpeg):quality(60):no_upscale():strip_exif()/${encodeURIComponent( + url + )}'`; + } + + // We need to loop through potential spocs and set their positions. + // If we run out of spocs or positions, we stop. + // First, we need to know which array is shortest. This is our exit condition. + const minLength = Math.min( + discoveryStreamSpocPositions.length, + discoveryStreamSpocs.length + ); + // Loop until we run out of spocs or positions. + for (let i = 0; i < minLength; i++) { + const positionIndex = discoveryStreamSpocPositions[i].index; + const spoc = discoveryStreamSpocs[i]; + const link = { + favicon: reformatImageURL(spoc.raw_image_src, 96, 96), + faviconSize: 96, + type: "SPOC", + label: spoc.title || spoc.sponsor, + title: spoc.title || spoc.sponsor, + url: spoc.url, + flightId: spoc.flight_id, + id: spoc.id, + guid: spoc.id, + shim: spoc.shim, + // For now we are assuming position based on intended position. + // Actual position can shift based on other content. + // We send the intended position in the ping. + pos: positionIndex, + // Set this so that SPOC topsites won't be shown in the URL bar. + // See Bug 1822027. Note that `sponsored_position` is 1-based. + sponsored_position: positionIndex + 1, + // This is used for topsites deduping. + hostname: shortURL({ url: spoc.url }), + partner: SPONSORED_TILE_PARTNER_MOZ_SALES, + }; + sponsored.push(link); + } + } + } + return sponsored; + } + + // eslint-disable-next-line max-statements + async getLinksWithDefaults(isStartup = false) { + const prefValues = this.store.getState().Prefs.values; + const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW; + const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT]; + // We must wait for search services to initialize in order to access default + // search engine properties without triggering a synchronous initialization + try { + await Services.search.init(); + } catch { + // We continue anyway because we want the user to see their sponsored, + // saved, or visited shortcut tiles even if search engines are not + // available. + } + + // Get all frecent sites from history. + let frecent = []; + let cache; + try { + // Request can throw if executing the linkGetter inside LinksCache returns + // a null object. + cache = await this.frecentCache.request({ + // We need to overquery due to the top 5 alexa search + default search possibly being removed + numItems: numItems + SEARCH_FILTERS.length + 1, + topsiteFrecency: FRECENCY_THRESHOLD, + }); + } catch (ex) { + cache = []; + } + + for (let link of cache) { + // The cache can contain null values. + if (!link) { + continue; + } + const hostname = shortURL(link); + if (!this.shouldFilterSearchTile(hostname)) { + frecent.push({ + ...(searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link), + hostname, + }); + } + } + + // Get defaults. + let contileSponsored = []; + let notBlockedDefaultSites = []; + for (let link of DEFAULT_TOP_SITES) { + // For sponsored Yandex links, default filtering is reversed: we only + // show them if Yandex is the default search engine. + if (link.sponsored_position && link.hostname === "yandex") { + if (link.hostname !== this._currentSearchHostname) { + continue; + } + } else if (this.shouldFilterSearchTile(link.hostname)) { + continue; + } + // Drop blocked default sites. + if ( + lazy.NewTabUtils.blockedLinks.isBlocked({ + url: link.url, + }) + ) { + continue; + } + // If we've previously blocked a search shortcut, remove the default top site + // that matches the hostname + const searchProvider = getSearchProvider(shortURL(link)); + if ( + searchProvider && + lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url }) + ) { + continue; + } + if (link.sponsored_position) { + if (!prefValues[SHOW_SPONSORED_PREF]) { + continue; + } + contileSponsored[link.sponsored_position - 1] = link; + + // Unpin search shortcut if present for the sponsored link to be shown + // instead. + this._unpinSearchShortcut(link.hostname); + } else { + notBlockedDefaultSites.push( + searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link + ); + } + } + this._telemetryUtility.determineFilteredTilesAndSetToDismissed( + contileSponsored + ); + + const discoverySponsored = this.fetchDiscoveryStreamSpocs(); + this._telemetryUtility.setTiles(discoverySponsored); + + const sponsored = this._mergeSponsoredLinks({ + [SPONSORED_TILE_PARTNER_AMP]: contileSponsored, + [SPONSORED_TILE_PARTNER_MOZ_SALES]: discoverySponsored, + }); + + this._maybeCapSponsoredLinks(sponsored); + + // This will set all extra tiles to oversold, including moz-sales. + this._telemetryUtility.determineFilteredTilesAndSetToOversold(sponsored); + + // Get pinned links augmented with desired properties + let plainPinned = await this.pinnedCache.request(); + + // Insert search shortcuts if we need to. + // _maybeInsertSearchShortcuts returns true if any search shortcuts are + // inserted, meaning we need to expire and refresh the pinnedCache + if (await this._maybeInsertSearchShortcuts(plainPinned)) { + this.pinnedCache.expire(); + plainPinned = await this.pinnedCache.request(); + } + + const pinned = await Promise.all( + plainPinned.map(async link => { + if (!link) { + return link; + } + + // Drop pinned search shortcuts when their engine has been removed / hidden. + if (link.searchTopSite) { + const searchProvider = getSearchProvider(shortURL(link)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return null; + } + } + + // Copy all properties from a frecent link and add more + const finder = other => other.url === link.url; + + // Remove frecent link's screenshot if pinned link has a custom one + const frecentSite = frecent.find(finder); + if (frecentSite && link.customScreenshotURL) { + delete frecentSite.screenshot; + } + // If the link is a frecent site, do not copy over 'isDefault', else check + // if the site is a default site + const copy = Object.assign( + {}, + frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) }, + link, + { hostname: shortURL(link) }, + { searchTopSite: !!link.searchTopSite } + ); + + // Add in favicons if we don't already have it + if (!copy.favicon) { + try { + lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI( + await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy]) + ); + + for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) { + copy.__sharedCache.updateLink(prop, copy[prop]); + } + } catch (e) { + // Some issue with favicon, so just continue without one + } + } + + return copy; + }) + ); + + // Remove any duplicates from frecent and default sites + const [, dedupedSponsored, dedupedFrecent, dedupedDefaults] = + this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites); + const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults]; + + // Remove adult sites if we need to + const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned); + + // Insert the original pinned sites into the deduped frecent and defaults. + let withPinned = insertPinned(checkedAdult, pinned); + // Insert sponsored sites at their desired position. + dedupedSponsored.forEach(link => { + if (!link) { + return; + } + let index = link.sponsored_position - 1; + if (index >= withPinned.length) { + withPinned[index] = link; + } else if (withPinned[index]?.sponsored_position) { + // We currently want DiscoveryStream spocs to replace existing spocs. + withPinned[index] = link; + } else { + withPinned.splice(index, 0, link); + } + }); + // Remove excess items after we inserted sponsored ones. + withPinned = withPinned.slice(0, numItems); + + // Now, get a tippy top icon, a rich icon, or screenshot for every item + for (const link of withPinned) { + if (link) { + // If there is a custom screenshot this is the only image we display + if (link.customScreenshotURL) { + this._fetchScreenshot(link, link.customScreenshotURL, isStartup); + } else if (link.searchTopSite && !link.isDefault) { + await this._attachTippyTopIconForSearchShortcut(link, link.label); + } else { + this._fetchIcon(link, isStartup); + } + + // Remove internal properties that might be updated after dispatch + delete link.__sharedCache; + + // Indicate that these links should get a frecency bonus when clicked + link.typedBonus = true; + } + } + + this._linksWithDefaults = withPinned; + + this._telemetryUtility.finalizeNewtabPingFields(dedupedSponsored); + return withPinned; + } + + /** + * Cap sponsored links if they're more than the specified maximum. + * + * @param {Array} links An array of sponsored links. Capping will be performed in-place. + */ + _maybeCapSponsoredLinks(links) { + // Set maximum sponsored top sites + const maxSponsored = + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_MAX_SPONSORED + ) ?? MAX_NUM_SPONSORED; + if (links.length > maxSponsored) { + links.length = maxSponsored; + } + } + + /** + * Merge sponsored links from all the partners using SOV if present. + * For each tile position, the user is assigned to one partner via stable sampling. + * If the chosen partner doesn't have a tile to serve, another tile from a different + * partner is used as the replacement. + * + * @param {object} sponsoredLinks An object with sponsored links from all the partners. + * @returns {Array} An array of merged sponsored links. + */ + _mergeSponsoredLinks(sponsoredLinks) { + const { positions: allocatedPositions, ready: sovReady } = + this.store.getState().TopSites.sov || {}; + if ( + !this._contile.sov || + !sovReady || + !lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_SOV_ENABLED + ) + ) { + return Object.values(sponsoredLinks).flat(); + } + + // AMP links might have empty slots, remove them as SOV doesn't need those. + sponsoredLinks[SPONSORED_TILE_PARTNER_AMP] = + sponsoredLinks[SPONSORED_TILE_PARTNER_AMP].filter(Boolean); + + let sponsored = []; + let chosenPartners = []; + + for (const allocation of allocatedPositions) { + let link = null; + const { assignedPartner } = allocation; + if (assignedPartner) { + // Unknown partners are allowed so that new parters can be added to Shepherd + // sooner without waiting for client changes. + link = sponsoredLinks[assignedPartner]?.shift(); + } + + if (!link) { + // If the chosen partner doesn't have a tile for this postion, choose any + // one from another group. For simplicity, we do _not_ do resampling here + // against the remaining partners. + for (const partner of SPONSORED_TILE_PARTNERS) { + if ( + partner === assignedPartner || + sponsoredLinks[partner].length === 0 + ) { + continue; + } + link = sponsoredLinks[partner].shift(); + break; + } + + if (!link) { + // No more links to be added across all the partners, just return. + if (chosenPartners.length) { + Glean.newtab.sovAllocation.set( + chosenPartners.map(entry => JSON.stringify(entry)) + ); + } + return sponsored; + } + } + + // Update the position fields. Note that postion is also 1-based in SOV. + link.sponsored_position = allocation.position; + if (link.pos !== undefined) { + // Pocket `pos` is 0-based. + link.pos = allocation.position - 1; + } + sponsored.push(link); + + chosenPartners.push({ + pos: allocation.position, + assigned: assignedPartner, // The assigned partner based on SOV + chosen: link.partner, + }); + } + // Record chosen partners to glean + if (chosenPartners.length) { + Glean.newtab.sovAllocation.set( + chosenPartners.map(entry => JSON.stringify(entry)) + ); + } + + // add the remaining contile sponsoredLinks when nimbus variable present + if ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) + ) { + return sponsored.concat(sponsoredLinks[SPONSORED_TILE_PARTNER_AMP]); + } + + return sponsored; + } + + /** + * Attach TippyTop icon to the given search shortcut + * + * Note that it queries the search form URL from search service For Yandex, + * and uses it to choose the best icon for its shortcut variants. + * + * @param {object} link A link object with a `url` property + * @param {string} keyword Search keyword + */ + async _attachTippyTopIconForSearchShortcut(link, keyword) { + if ( + ["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"].includes(keyword) + ) { + let site = { url: link.url }; + site.url = (await getSearchFormURL(keyword)) || site.url; + this._tippyTopProvider.processSite(site); + link.tippyTopIcon = site.tippyTopIcon; + link.smallFavicon = site.smallFavicon; + link.backgroundColor = site.backgroundColor; + } else { + this._tippyTopProvider.processSite(link); + } + } + + /** + * Refresh the top sites data for content. + * + * @param {object} options + * @param {bool} options.broadcast Should the update be broadcasted. + * @param {bool} options.isStartup Being called while TopSitesFeed is initting. + */ + async refresh(options = {}) { + // Avoiding refreshing if it's already happening. + if (this._refreshing) { + return; + } + if (!this._startedUp && !options.isStartup) { + // Initial refresh still pending. + return; + } + this._refreshing = true; + this._startedUp = true; + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + const links = await this.getLinksWithDefaults({ + isStartup: options.isStartup, + }); + const newAction = { type: at.TOP_SITES_UPDATED, data: { links } }; + let storedPrefs; + try { + storedPrefs = (await this._storage.get(SECTION_ID)) || {}; + } catch (e) { + storedPrefs = {}; + console.error("Problem getting stored prefs for TopSites"); + } + newAction.data.pref = getDefaultOptions(storedPrefs); + + if (options.isStartup) { + newAction.meta = { + isStartup: true, + }; + } + + if (options.broadcast) { + // Broadcast an update to all open content pages + this.store.dispatch(ac.BroadcastToContent(newAction)); + } else { + // Don't broadcast only update the state and update the preloaded tab. + this.store.dispatch(ac.AlsoToPreloaded(newAction)); + } + this._refreshing = false; + if (Cu.isInAutomation) { + Services.obs.notifyObservers(null, "topsites-refreshed"); + } + } + + // Allocate ad positions to partners based on SOV via stable randomization. + async allocatePositions() { + // If the fetch to get sov fails for whatever reason, we can just return here. + // Code that uses this falls back to flattening allocations instead if this has failed. + if (!this._contile.sov) { + return; + } + // This sample input should ensure we return the same result for this allocation, + // even if called from other parts of the code. + const sampleInput = `${lazy.contextId}-${this._contile.sov.name}`; + const allocatedPositions = []; + for (const allocation of this._contile.sov.allocations) { + const allocatedPosition = { + position: allocation.position, + }; + allocatedPositions.push(allocatedPosition); + const ratios = allocation.allocation.map(alloc => alloc.percentage); + if (ratios.length) { + const index = await lazy.Sampling.ratioSample(sampleInput, ratios); + allocatedPosition.assignedPartner = + allocation.allocation[index].partner; + } + } + + this.store.dispatch( + ac.OnlyToMain({ + type: at.SOV_UPDATED, + data: { + ready: !!allocatedPositions.length, + positions: allocatedPositions, + }, + }) + ); + } + + async updateCustomSearchShortcuts(isStartup = false) { + if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + return; + } + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + // Populate the state with available search shortcuts + let searchShortcuts = []; + for (const engine of await Services.search.getAppProvidedEngines()) { + const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s => + engine.aliases.includes(s.keyword) + ); + if (shortcut) { + let clone = { ...shortcut }; + await this._attachTippyTopIconForSearchShortcut(clone, clone.keyword); + searchShortcuts.push(clone); + } + } + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.UPDATE_SEARCH_SHORTCUTS, + data: { searchShortcuts }, + meta: { + isStartup, + }, + }) + ); + } + + async topSiteToSearchTopSite(site) { + const searchProvider = getSearchProvider(shortURL(site)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return site; + } + return { + ...site, + searchTopSite: true, + label: searchProvider.keyword, + }; + } + + /** + * Get an image for the link preferring tippy top, rich favicon, screenshots. + */ + async _fetchIcon(link, isStartup = false) { + // Nothing to do if we already have a rich icon from the page + if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) { + return; + } + + // Nothing more to do if we can use a default tippy top icon + this._tippyTopProvider.processSite(link); + if (link.tippyTopIcon) { + return; + } + + // Make a request for a better icon + this._requestRichIcon(link.url); + + // Also request a screenshot if we don't have one yet + await this._fetchScreenshot(link, link.url, isStartup); + } + + /** + * Fetch, cache and broadcast a screenshot for a specific topsite. + * + * @param {object} link cached topsite object + * @param {string} url where to fetch the image from + * @param {boolean} isStartup Whether the screenshot is fetched while + * initting TopSitesFeed. + */ + async _fetchScreenshot(link, url, isStartup = false) { + // We shouldn't bother caching screenshots if they won't be shown. + if ( + link.screenshot || + !this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF] + ) { + return; + } + await lazy.Screenshots.maybeCacheScreenshot( + link, + url, + "screenshot", + screenshot => + this.store.dispatch( + ac.BroadcastToContent({ + data: { screenshot, url: link.url }, + type: at.SCREENSHOT_UPDATED, + meta: { + isStartup, + }, + }) + ) + ); + } + + /** + * Dispatch screenshot preview to target or notify if request failed. + * + * @param {string} url The URL used to capture the screenshot + * @param {string} target Id of content process where to dispatch the result + */ + async getScreenshotPreview(url, target) { + const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || ""; + this.store.dispatch( + ac.OnlyToOneContent( + { + data: { url, preview }, + type: at.PREVIEW_RESPONSE, + }, + target + ) + ); + } + + _requestRichIcon(url) { + this.store.dispatch({ + type: at.RICH_ICON_MISSING, + data: { url }, + }); + } + + updateSectionPrefs(collapsed) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.TOP_SITES_PREFS_UPDATED, + data: { pref: collapsed }, + }) + ); + } + + /** + * Inform others that top sites data has been updated due to pinned changes. + */ + _broadcastPinnedSitesUpdated() { + // Pinned data changed, so make sure we get latest + this.pinnedCache.expire(); + + // Refresh to update pinned sites with screenshots, trigger deduping, etc. + this.refresh({ broadcast: true }); + } + + /** + * Pin a site at a specific position saving only the desired keys. + * + * @param customScreenshotURL {string} User set URL of preview image for site + * @param label {string} User set string of custom site name + */ + // To refactor in Bug 1891997 + /* eslint-enable jsdoc/check-param-names */ + async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) { + const toPin = { url }; + if (label) { + toPin.label = label; + } + if (customScreenshotURL) { + toPin.customScreenshotURL = customScreenshotURL; + } + if (searchTopSite) { + toPin.searchTopSite = searchTopSite; + } + lazy.NewTabUtils.pinnedLinks.pin(toPin, index); + + await this._clearLinkCustomScreenshot({ customScreenshotURL, url }); + } + + async _clearLinkCustomScreenshot(site) { + // If screenshot url changed or was removed we need to update the cached link obj + if (site.customScreenshotURL !== undefined) { + const pinned = await this.pinnedCache.request(); + const link = pinned.find(pin => pin && pin.url === site.url); + if (link && link.customScreenshotURL !== site.customScreenshotURL) { + link.__sharedCache.updateLink("screenshot", undefined); + } + } + } + + /** + * Handle a pin action of a site to a position. + */ + async pin(action) { + let { site, index } = action.data; + index = this._adjustPinIndexForSponsoredLinks(site, index); + // If valid index provided, pin at that position + if (index >= 0) { + await this._pinSiteAt(site, index); + this._broadcastPinnedSitesUpdated(); + } else { + // Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option, + // then we want to make sure to unblock that link if it has previously been + // blocked. We know if the site has been added because the index will be -1. + if (index === -1) { + lazy.NewTabUtils.blockedLinks.unblock({ url: site.url }); + this.frecentCache.expire(); + } + this.insert(action); + } + } + + /** + * Handle an unpin action of a site. + */ + unpin(action) { + const { site } = action.data; + lazy.NewTabUtils.pinnedLinks.unpin(site); + this._broadcastPinnedSitesUpdated(); + } + + unpinAllSearchShortcuts() { + Services.prefs.clearUserPref( + `browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}` + ); + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if (pinnedLink && pinnedLink.searchTopSite) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + } + } + this.pinnedCache.expire(); + } + + _unpinSearchShortcut(vendor) { + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if ( + pinnedLink && + pinnedLink.searchTopSite && + shortURL(pinnedLink) === vendor + ) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + this.pinnedCache.expire(); + + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(","); + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.filter(s => s !== vendor).join(",") + ) + ); + break; + } + } + } + + /** + * Reduces the given pinning index by the number of preceding sponsored + * sites, to accomodate for sponsored sites pushing pinned ones to the side, + * effectively increasing their index again. + */ + _adjustPinIndexForSponsoredLinks(site, index) { + if (!this._linksWithDefaults) { + return index; + } + // Adjust insertion index for sponsored sites since their position is + // fixed. + let adjustedIndex = index; + for (let i = 0; i < index; i++) { + const link = this._linksWithDefaults[i]; + if ( + link && + link.sponsored_position && + this._linksWithDefaults[i]?.url !== site.url + ) { + adjustedIndex--; + } + } + return adjustedIndex; + } + + /** + * Insert a site to pin at a position shifting over any other pinned sites. + */ + _insertPin(site, originalIndex, draggedFromIndex) { + let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex); + + // Don't insert any pins past the end of the visible top sites. Otherwise, + // we can end up with a bunch of pinned sites that can never be unpinned again + // from the UI. + const topSitesCount = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + if (index >= topSitesCount) { + return; + } + + let pinned = lazy.NewTabUtils.pinnedLinks.links; + if (!pinned[index]) { + this._pinSiteAt(site, index); + } else { + pinned[draggedFromIndex] = null; + // Find the hole to shift the pinned site(s) towards. We shift towards the + // hole left by the site being dragged. + let holeIndex = index; + const indexStep = index > draggedFromIndex ? -1 : 1; + while (pinned[holeIndex]) { + holeIndex += indexStep; + } + if (holeIndex >= topSitesCount || holeIndex < 0) { + // There are no holes, so we will effectively unpin the last slot and shifting + // towards it. This only happens when adding a new top site to an already + // fully pinned grid. + holeIndex = topSitesCount - 1; + } + + // Shift towards the hole. + const shiftingStep = holeIndex > index ? -1 : 1; + while (holeIndex !== index) { + const nextIndex = holeIndex + shiftingStep; + this._pinSiteAt(pinned[nextIndex], holeIndex); + holeIndex = nextIndex; + } + this._pinSiteAt(site, index); + } + } + + /** + * Handle an insert (drop/add) action of a site. + */ + async insert(action) { + let { index } = action.data; + // Treat invalid pin index values (e.g., -1, undefined) as insert in the first position + if (!(index > 0)) { + index = 0; + } + + // Inserting a top site pins it in the specified slot, pushing over any link already + // pinned in the slot (unless it's the last slot, then it replaces). + this._insertPin( + action.data.site, + index, + action.data.draggedFromIndex !== undefined + ? action.data.draggedFromIndex + : this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW + ); + + await this._clearLinkCustomScreenshot(action.data.site); + this._broadcastPinnedSitesUpdated(); + } + + updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) { + // Unpin the deletedShortcuts. + deletedShortcuts.forEach(({ url }) => { + lazy.NewTabUtils.pinnedLinks.unpin({ url }); + }); + + // Pin the addedShortcuts. + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + addedShortcuts.forEach(shortcut => { + // Find first hole in pinnedLinks. + let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link); + if ( + index < 0 && + lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots + ) { + // pinnedLinks can have less slots than the total available. + index = lazy.NewTabUtils.pinnedLinks.links.length; + } + if (index >= 0) { + lazy.NewTabUtils.pinnedLinks.pin(shortcut, index); + } else { + // No slots available, we need to do an insert in first slot and push over other pinned links. + this._insertPin(shortcut, 0, numberOfSlots); + } + }); + + this._broadcastPinnedSitesUpdated(); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.updateCustomSearchShortcuts(true /* isStartup */); + break; + case at.SYSTEM_TICK: + this.refresh({ broadcast: false }); + this._contile.periodicUpdate(); + break; + // All these actions mean we need new top sites + case at.PLACES_HISTORY_CLEARED: + case at.PLACES_LINKS_DELETED: + this.frecentCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PLACES_LINKS_CHANGED: + this.frecentCache.expire(); + this.refresh({ broadcast: false }); + break; + case at.PLACES_LINK_BLOCKED: + this.frecentCache.expire(); + this.pinnedCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PREF_CHANGED: + switch (action.data.name) { + case DEFAULT_SITES_PREF: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data.value); + } + break; + case ROWS_PREF: + case FILTER_DEFAULT_SEARCH_PREF: + case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF: + this.refresh({ broadcast: true }); + break; + case SHOW_SPONSORED_PREF: + if ( + lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) + ) { + this._contile.refresh(); + } else { + this.refresh({ broadcast: true }); + } + if (!action.data.value) { + this._contile._resetContileCachePrefs(); + } + + break; + case SEARCH_SHORTCUTS_EXPERIMENT: + if (action.data.value) { + this.updateCustomSearchShortcuts(); + } else { + this.unpinAllSearchShortcuts(); + } + this.refresh({ broadcast: true }); + } + break; + case at.UPDATE_SECTION_PREFS: + if (action.data.id === SECTION_ID) { + this.updateSectionPrefs(action.data.value); + } + break; + case at.PREFS_INITIAL_VALUES: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data[DEFAULT_SITES_PREF]); + } + break; + case at.TOP_SITES_PIN: + this.pin(action); + break; + case at.TOP_SITES_UNPIN: + this.unpin(action); + break; + case at.TOP_SITES_INSERT: + this.insert(action); + break; + case at.PREVIEW_REQUEST: + this.getScreenshotPreview(action.data.url, action.meta.fromTarget); + break; + case at.UPDATE_PINNED_SEARCH_SHORTCUTS: + this.updatePinnedSearchShortcuts(action.data); + break; + case at.DISCOVERY_STREAM_SPOCS_UPDATE: + // Refresh to update sponsored topsites. + this.refresh({ broadcast: true, isStartup: action.meta.isStartup }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} + +export const TopSites = new _TopSites(); diff --git a/browser/components/topsites/moz.build b/browser/components/topsites/moz.build new file mode 100644 index 0000000000..f8c7a96fa2 --- /dev/null +++ b/browser/components/topsites/moz.build @@ -0,0 +1,12 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXTRA_JS_MODULES += ["TopSites.sys.mjs"] + +XPCSHELL_TESTS_MANIFESTS += ["test/unit/xpcshell.toml"] + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Top Sites") diff --git a/browser/components/topsites/test/unit/test_top_sites.js b/browser/components/topsites/test/unit/test_top_sites.js new file mode 100644 index 0000000000..3de5f43262 --- /dev/null +++ b/browser/components/topsites/test/unit/test_top_sites.js @@ -0,0 +1,3571 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { TopSites, DEFAULT_TOP_SITES } = ChromeUtils.importESModule( + "resource:///modules/TopSites.sys.mjs" +); + +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.mjs" +); + +ChromeUtils.defineESModuleGetters(this, { + FilterAdult: "resource://activity-stream/lib/FilterAdult.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + shortURL: "resource://activity-stream/lib/ShortURL.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", + Screenshots: "resource://activity-stream/lib/Screenshots.sys.mjs", + Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs", + SearchService: "resource://gre/modules/SearchService.sys.mjs", + TestUtils: "resource://testing-common/TestUtils.sys.mjs", + TOP_SITES_DEFAULT_ROWS: "resource://activity-stream/common/Reducers.sys.mjs", + TOP_SITES_MAX_SITES_PER_ROW: + "resource://activity-stream/common/Reducers.sys.mjs", +}); + +const FAKE_FAVICON = "data987"; +const FAKE_FAVICON_SIZE = 128; +const FAKE_FRECENCY = 200; +const FAKE_LINKS = new Array(2 * TOP_SITES_MAX_SITES_PER_ROW) + .fill(null) + .map((v, i) => ({ + frecency: FAKE_FRECENCY, + url: `http://www.site${i}.com`, + })); +const FAKE_SCREENSHOT = "data123"; +const SEARCH_SHORTCUTS_EXPERIMENT_PREF = "improvesearch.topSiteSearchShortcuts"; +const SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF = + "improvesearch.topSiteSearchShortcuts.searchEngines"; +const SEARCH_SHORTCUTS_HAVE_PINNED_PREF = + "improvesearch.topSiteSearchShortcuts.havePinned"; +const SHOWN_ON_NEWTAB_PREF = "feeds.topsites"; +const SHOW_SPONSORED_PREF = "showSponsoredTopSites"; +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; +const CONTILE_CACHE_PREF = "browser.topsites.contile.cachedTiles"; + +// This pref controls how long the contile cache is valid for in seconds. +const CONTILE_CACHE_VALID_FOR_SECONDS_PREF = + "browser.topsites.contile.cacheValidFor"; +// This pref records when the last contile fetch occurred, as a UNIX timestamp +// in seconds. +const CONTILE_CACHE_LAST_FETCH_PREF = "browser.topsites.contile.lastFetch"; + +function FakeTippyTopProvider() {} +FakeTippyTopProvider.prototype = { + async init() { + this.initialized = true; + }, + processSite(site) { + return site; + }, +}; + +let gSearchServiceInitStub; +let gGetTopSitesStub; + +function stubTopSites(sandbox) { + let cachedStorage = TopSites._storage; + let cachedStore = TopSites.store; + + async function cleanup() { + if (TopSites._refreshing) { + info("Wait for refresh to finish."); + // Wait for refresh to finish or else removing the store while a process + // is running will result in errors. + await TestUtils.topicObserved("topsites-refreshed"); + } + TopSites._tippyTopProvider.initialized = false; + TopSites._storage = cachedStorage; + TopSites.store = cachedStore; + TopSites.pinnedCache.clear(); + TopSites.frecentCache.clear(); + info("Finished cleaning up TopSites."); + } + + const storage = { + init: sandbox.stub().resolves(), + get: sandbox.stub().resolves(), + set: sandbox.stub().resolves(), + }; + + // Setup for tests that don't call `init` but require feed.storage + TopSites._storage = storage; + TopSites.store = { + dispatch: sinon.spy(), + getState() { + return this.state; + }, + state: { + Prefs: { values: { topSitesRows: 2 } }, + TopSites: { rows: Array(12).fill("site") }, + }, + dbStorage: { getDbTable: sandbox.stub().returns(storage) }, + }; + info("Created mock store for TopSites."); + return cleanup; +} + +add_setup(async () => { + let sandbox = sinon.createSandbox(); + sandbox.stub(SearchService.prototype, "defaultEngine").get(() => { + return { identifier: "ddg", searchForm: "https://duckduckgo.com" }; + }); + + gGetTopSitesStub = sandbox + .stub(NewTabUtils.activityStreamLinks, "getTopSites") + .resolves(FAKE_LINKS); + + gSearchServiceInitStub = sandbox + .stub(SearchService.prototype, "init") + .resolves(); + + sandbox.stub(NewTabUtils.activityStreamProvider, "_faviconBytesToDataURI"); + + sandbox + .stub(NewTabUtils.activityStreamProvider, "_addFavicons") + .callsFake(l => { + return Promise.resolve( + l.map(link => { + link.favicon = FAKE_FAVICON; + link.faviconSize = FAKE_FAVICON_SIZE; + return link; + }) + ); + }); + + sandbox.stub(Screenshots, "getScreenshotForURL").resolves(FAKE_SCREENSHOT); + sandbox.spy(Screenshots, "maybeCacheScreenshot"); + sandbox.stub(Screenshots, "_shouldGetScreenshots").returns(true); + + registerCleanupFunction(() => { + sandbox.restore(); + }); +}); + +add_task(async function test_construction() { + Assert.ok(TopSites._currentSearchHostname, "_currentSearchHostname defined"); +}); + +add_task(async function test_refreshDefaults() { + let sandbox = sinon.createSandbox(); + let cleanup = stubTopSites(sandbox); + Assert.ok( + !DEFAULT_TOP_SITES.length, + "Should have 0 DEFAULT_TOP_SITES initially." + ); + + info("refreshDefaults should add defaults on PREFS_INITIAL_VALUES"); + TopSites.onAction({ + type: at.PREFS_INITIAL_VALUES, + data: { "default.sites": "https://foo.com" }, + }); + + Assert.equal( + DEFAULT_TOP_SITES.length, + 1, + "Should have 1 DEFAULT_TOP_SITES now." + ); + + // Reset the DEFAULT_TOP_SITES; + DEFAULT_TOP_SITES.length = 0; + + info("refreshDefaults should add defaults on default.sites PREF_CHANGED"); + TopSites.onAction({ + type: at.PREF_CHANGED, + data: { name: "default.sites", value: "https://foo.com" }, + }); + + Assert.equal( + DEFAULT_TOP_SITES.length, + 1, + "Should have 1 DEFAULT_TOP_SITES now." + ); + + // Reset the DEFAULT_TOP_SITES; + DEFAULT_TOP_SITES.length = 0; + + info("refreshDefaults should refresh on topSiteRows PREF_CHANGED"); + let refreshStub = sandbox.stub(TopSites, "refresh"); + TopSites.onAction({ type: at.PREF_CHANGED, data: { name: "topSitesRows" } }); + Assert.ok(TopSites.refresh.calledOnce, "refresh called"); + refreshStub.restore(); + + // Reset the DEFAULT_TOP_SITES; + DEFAULT_TOP_SITES.length = 0; + + info("refreshDefaults should have default sites with .isDefault = true"); + TopSites.refreshDefaults("https://foo.com"); + Assert.equal( + DEFAULT_TOP_SITES.length, + 1, + "Should have a DEFAULT_TOP_SITES now." + ); + Assert.ok( + DEFAULT_TOP_SITES[0].isDefault, + "Lone top site should be the default." + ); + + // Reset the DEFAULT_TOP_SITES; + DEFAULT_TOP_SITES.length = 0; + + info("refreshDefaults should have default sites with appropriate hostname"); + TopSites.refreshDefaults("https://foo.com"); + Assert.equal( + DEFAULT_TOP_SITES.length, + 1, + "Should have a DEFAULT_TOP_SITES now." + ); + let [site] = DEFAULT_TOP_SITES; + Assert.equal( + site.hostname, + shortURL(site), + "Lone top site should have the right hostname." + ); + + // Reset the DEFAULT_TOP_SITES; + DEFAULT_TOP_SITES.length = 0; + + info("refreshDefaults should add no defaults on empty pref"); + TopSites.refreshDefaults(""); + Assert.equal( + DEFAULT_TOP_SITES.length, + 0, + "Should have 0 DEFAULT_TOP_SITES now." + ); + + info("refreshDefaults should be able to clear defaults"); + TopSites.refreshDefaults("https://foo.com"); + TopSites.refreshDefaults(""); + + Assert.equal( + DEFAULT_TOP_SITES.length, + 0, + "Should have 0 DEFAULT_TOP_SITES now." + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_filterForThumbnailExpiration() { + let sandbox = sinon.createSandbox(); + let cleanup = stubTopSites(sandbox); + + info( + "filterForThumbnailExpiration should pass rows.urls to the callback provided" + ); + const rows = [ + { url: "foo.com" }, + { url: "bar.com", customScreenshotURL: "custom" }, + ]; + TopSites.store.state.TopSites = { rows }; + const stub = sandbox.stub(); + TopSites.filterForThumbnailExpiration(stub); + Assert.ok(stub.calledOnce); + Assert.ok(stub.calledWithExactly(["foo.com", "bar.com", "custom"])); + + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_getLinksWithDefaults_on_SearchService_init_failure() { + let sandbox = sinon.createSandbox(); + let cleanup = stubTopSites(sandbox); + + TopSites.refreshDefaults("https://foo.com"); + + gSearchServiceInitStub.rejects( + new Error("Simulating search init failures") + ); + + const result = await TopSites.getLinksWithDefaults(); + Assert.ok(result); + + gSearchServiceInitStub.resolves(); + + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_getLinksWithDefaults() { + NewTabUtils.activityStreamLinks.getTopSites.resetHistory(); + + let sandbox = sinon.createSandbox(); + let cleanup = stubTopSites(sandbox); + + TopSites.refreshDefaults("https://foo.com"); + + info("getLinksWithDefaults should get the links from NewTabUtils"); + let result = await TopSites.getLinksWithDefaults(); + + const reference = FAKE_LINKS.map(site => + Object.assign({}, site, { + hostname: shortURL(site), + typedBonus: true, + }) + ); + + Assert.deepEqual(result, reference); + Assert.ok(NewTabUtils.activityStreamLinks.getTopSites.calledOnce); + + info("getLinksWithDefaults should indicate the links get typed bonus"); + Assert.ok(result[0].typedBonus, "Expected typed bonus property to be true."); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_filterAdult() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should filter out non-pinned adult sites"); + + sandbox.stub(FilterAdult, "filter").returns([]); + const TEST_URL = "https://foo.com/"; + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [{ url: TEST_URL }]); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const result = await TopSites.getLinksWithDefaults(); + Assert.ok(FilterAdult.filter.calledOnce); + Assert.equal(result.length, 1); + Assert.equal(result[0].url, TEST_URL); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_caching() { + let sandbox = sinon.createSandbox(); + + info( + "getLinksWithDefaults should filter out the defaults that have been blocked" + ); + // make sure we only have one top site, and we block the only default site we have to show + const url = "www.myonlytopsite.com"; + const topsite = { + frecency: FAKE_FRECENCY, + hostname: shortURL({ url }), + typedBonus: true, + url, + }; + + const blockedDefaultSite = { url: "https://foo.com" }; + gGetTopSitesStub.resolves([topsite]); + sandbox.stub(NewTabUtils.blockedLinks, "isBlocked").callsFake(site => { + return site.url === blockedDefaultSite.url; + }); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + const result = await TopSites.getLinksWithDefaults(); + + // what we should be left with is just the top site we added, and not the default site we blocked + Assert.equal(result.length, 1); + Assert.deepEqual(result[0], topsite); + let foundBlocked = result.find(site => site.url === blockedDefaultSite.url); + Assert.ok(!foundBlocked, "Should not have found blocked site."); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_dedupe() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should call dedupe.group on the links"); + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + let stub = sandbox.stub(TopSites.dedupe, "group").callsFake((...id) => id); + await TopSites.getLinksWithDefaults(); + + Assert.ok(stub.calledOnce, "dedupe.group was called once"); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test__dedupe_key() { + let sandbox = sinon.createSandbox(); + + info("_dedupeKey should dedupe on hostname instead of url"); + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + let site = { url: "foo", hostname: "bar" }; + let result = TopSites._dedupeKey(site); + + Assert.equal(result, site.hostname, "deduped on hostname"); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_adds_defaults() { + let sandbox = sinon.createSandbox(); + + info( + "getLinksWithDefaults should add defaults if there are are not enough links" + ); + const TEST_LINKS = [{ frecency: FAKE_FRECENCY, url: "foo.com" }]; + gGetTopSitesStub.resolves(TEST_LINKS); + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + let result = await TopSites.getLinksWithDefaults(); + + let reference = [...TEST_LINKS, ...DEFAULT_TOP_SITES].map(s => + Object.assign({}, s, { + hostname: shortURL(s), + typedBonus: true, + }) + ); + + Assert.deepEqual(result, reference); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_getLinksWithDefaults_adds_defaults_for_visible_slots() { + let sandbox = sinon.createSandbox(); + + info( + "getLinksWithDefaults should only add defaults up to the number of visible slots" + ); + const numVisible = TOP_SITES_DEFAULT_ROWS * TOP_SITES_MAX_SITES_PER_ROW; + let testLinks = []; + for (let i = 0; i < numVisible - 1; i++) { + testLinks.push({ frecency: FAKE_FRECENCY, url: `foo${i}.com` }); + } + gGetTopSitesStub.resolves(testLinks); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + let result = await TopSites.getLinksWithDefaults(); + + let reference = [...testLinks, DEFAULT_TOP_SITES[0]].map(s => + Object.assign({}, s, { + hostname: shortURL(s), + typedBonus: true, + }) + ); + + Assert.equal(result.length, numVisible); + Assert.deepEqual(result, reference); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_getLinksWithDefaults_no_throw_on_no_links() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should not throw if NewTabUtils returns null"); + gGetTopSitesStub.resolves(null); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + await TopSites.getLinksWithDefaults(); + Assert.ok(true, "getLinksWithDefaults did not throw"); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_get_more_on_request() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should get more if the user has asked for more"); + let testLinks = new Array(4 * TOP_SITES_MAX_SITES_PER_ROW) + .fill(null) + .map((v, i) => ({ + frecency: FAKE_FRECENCY, + url: `http://www.site${i}.com`, + })); + gGetTopSitesStub.resolves(testLinks); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const TEST_ROWS = 3; + TopSites.store.state.Prefs.values.topSitesRows = TEST_ROWS; + + let result = await TopSites.getLinksWithDefaults(); + Assert.equal(result.length, TEST_ROWS * TOP_SITES_MAX_SITES_PER_ROW); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_reuse_cache() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should reuse the cache on subsequent calls"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + gGetTopSitesStub.resetHistory(); + + await TopSites.getLinksWithDefaults(); + await TopSites.getLinksWithDefaults(); + + Assert.ok( + NewTabUtils.activityStreamLinks.getTopSites.calledOnce, + "getTopSites only called once" + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_getLinksWithDefaults_ignore_cache_on_requesting_more() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should ignore the cache when requesting more"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + gGetTopSitesStub.resetHistory(); + + await TopSites.getLinksWithDefaults(); + TopSites.store.state.Prefs.values.topSitesRows *= 3; + await TopSites.getLinksWithDefaults(); + + Assert.ok( + NewTabUtils.activityStreamLinks.getTopSites.calledTwice, + "getTopSites called twice" + ); + + sandbox.restore(); + await cleanup(); + } +); + +add_task( + async function test_getLinksWithDefaults_migrate_frecent_screenshot_data() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should migrate frecent screenshot data without getting screenshots again" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + gGetTopSitesStub.resetHistory(); + + TopSites.store.state.Prefs.values[SHOWN_ON_NEWTAB_PREF] = true; + await TopSites.getLinksWithDefaults(); + + let originalCallCount = Screenshots.getScreenshotForURL.callCount; + TopSites.frecentCache.expire(); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.ok( + NewTabUtils.activityStreamLinks.getTopSites.calledTwice, + "getTopSites called twice" + ); + Assert.equal( + Screenshots.getScreenshotForURL.callCount, + originalCallCount, + "getScreenshotForURL was not called again." + ); + Assert.equal(result[0].screenshot, FAKE_SCREENSHOT); + + sandbox.restore(); + await cleanup(); + } +); + +add_task( + async function test_getLinksWithDefaults_migrate_pinned_favicon_data() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should migrate pinned favicon data without getting favicons again" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + gGetTopSitesStub.resetHistory(); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/" }]); + + await TopSites.getLinksWithDefaults(); + + let originalCallCount = + NewTabUtils.activityStreamProvider._addFavicons.callCount; + TopSites.pinnedCache.expire(); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.equal( + NewTabUtils.activityStreamProvider._addFavicons.callCount, + originalCallCount, + "_addFavicons was not called again." + ); + Assert.equal(result[0].favicon, FAKE_FAVICON); + Assert.equal(result[0].faviconSize, FAKE_FAVICON_SIZE); + + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_getLinksWithDefaults_no_internal_properties() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should not expose internal link properties"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + let result = await TopSites.getLinksWithDefaults(); + + let internal = Object.keys(result[0]).filter(key => key.startsWith("__")); + Assert.equal(internal.join(""), ""); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_copy_frecent_screenshot() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should copy the screenshot of the frecent site if " + + "pinned site doesn't have customScreenshotURL" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const TEST_SCREENSHOT = "screenshot"; + + gGetTopSitesStub.resolves([ + { url: "https://foo.com/", screenshot: TEST_SCREENSHOT }, + ]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/" }]); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.equal(result[0].screenshot, TEST_SCREENSHOT); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_no_copy_frecent_screenshot() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should not copy the frecent screenshot if " + + "customScreenshotURL is set" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + gGetTopSitesStub.resolves([ + { url: "https://foo.com/", screenshot: "screenshot" }, + ]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/", customScreenshotURL: "custom" }]); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.equal(result[0].screenshot, undefined); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_persist_screenshot() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should keep the same screenshot if no frecent site is found" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const CUSTOM_SCREENSHOT = "custom"; + + gGetTopSitesStub.resolves([]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/", screenshot: CUSTOM_SCREENSHOT }]); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.equal(result[0].screenshot, CUSTOM_SCREENSHOT); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_getLinksWithDefaults_no_overwrite_pinned_screenshot() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should not overwrite pinned site screenshot"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const EXISTING_SCREENSHOT = "some-screenshot"; + + gGetTopSitesStub.resolves([{ url: "https://foo.com/", screenshot: "foo" }]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [ + { url: "https://foo.com/", screenshot: EXISTING_SCREENSHOT }, + ]); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.equal(result[0].screenshot, EXISTING_SCREENSHOT); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); + } +); + +add_task( + async function test_getLinksWithDefaults_no_searchTopSite_from_frecent() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should not set searchTopSite from frecent site"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + const EXISTING_SCREENSHOT = "some-screenshot"; + + gGetTopSitesStub.resolves([ + { + url: "https://foo.com/", + searchTopSite: true, + screenshot: EXISTING_SCREENSHOT, + }, + ]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/" }]); + + let result = await TopSites.getLinksWithDefaults(); + + Assert.ok(!result[0].searchTopSite); + // But it should copy over other properties + Assert.equal(result[0].screenshot, EXISTING_SCREENSHOT); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_getLinksWithDefaults_concurrency_getTopSites() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults concurrent calls should call the backing data once" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + NewTabUtils.activityStreamLinks.getTopSites.resetHistory(); + + await Promise.all([ + TopSites.getLinksWithDefaults(), + TopSites.getLinksWithDefaults(), + ]); + + Assert.ok( + NewTabUtils.activityStreamLinks.getTopSites.calledOnce, + "getTopSites only called once" + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_getLinksWithDefaults_concurrency_getScreenshotForURL() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults concurrent calls should call the backing data once" + ); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + TopSites.store.state.Prefs.values[SHOWN_ON_NEWTAB_PREF] = true; + + NewTabUtils.activityStreamLinks.getTopSites.resetHistory(); + Screenshots.getScreenshotForURL.resetHistory(); + + await Promise.all([ + TopSites.getLinksWithDefaults(), + TopSites.getLinksWithDefaults(), + ]); + + Assert.ok( + NewTabUtils.activityStreamLinks.getTopSites.calledOnce, + "getTopSites only called once" + ); + + Assert.equal( + Screenshots.getScreenshotForURL.callCount, + FAKE_LINKS.length, + "getLinksWithDefaults concurrent calls should get screenshots once per link" + ); + await cleanup(); + + cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[SHOWN_ON_NEWTAB_PREF] = true; + + TopSites.refreshDefaults("https://foo.com"); + + sandbox.stub(TopSites, "_requestRichIcon"); + await Promise.all([ + TopSites.getLinksWithDefaults(), + TopSites.getLinksWithDefaults(), + ]); + + Assert.equal( + TopSites.store.dispatch.callCount, + FAKE_LINKS.length, + "getLinksWithDefaults concurrent calls should dispatch once per link screenshot fetched" + ); + + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_getLinksWithDefaults_deduping_no_dedupe_pinned() { + let sandbox = sinon.createSandbox(); + info("getLinksWithDefaults should not dedupe pinned sites"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults("https://foo.com"); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [ + { url: "https://developer.mozilla.org/en-US/docs/Web" }, + { url: "https://developer.mozilla.org/en-US/docs/Learn" }, + ]); + + let sites = await TopSites.getLinksWithDefaults(); + Assert.equal(sites.length, 2 * TOP_SITES_MAX_SITES_PER_ROW); + Assert.equal(sites[0].url, NewTabUtils.pinnedLinks.links[0].url); + Assert.equal(sites[1].url, NewTabUtils.pinnedLinks.links[1].url); + Assert.equal(sites[0].hostname, sites[1].hostname); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_prefer_pinned_sites() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should prefer pinned sites over links"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults(); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [ + { url: "https://developer.mozilla.org/en-US/docs/Web" }, + { url: "https://developer.mozilla.org/en-US/docs/Learn" }, + ]); + + const SECOND_TOP_SITE_URL = "https://www.mozilla.org/"; + + gGetTopSitesStub.resolves([ + { frecency: FAKE_FRECENCY, url: "https://developer.mozilla.org/" }, + { frecency: FAKE_FRECENCY, url: SECOND_TOP_SITE_URL }, + ]); + + let sites = await TopSites.getLinksWithDefaults(); + + // Expecting 3 links where there's 2 pinned and 1 www.mozilla.org, so + // the frecent with matching hostname as pinned is removed. + Assert.equal(sites.length, 3); + Assert.equal(sites[0].url, NewTabUtils.pinnedLinks.links[0].url); + Assert.equal(sites[1].url, NewTabUtils.pinnedLinks.links[1].url); + Assert.equal(sites[2].url, SECOND_TOP_SITE_URL); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_title_and_null() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should return sites that have a title"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults(); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://github.com/mozilla/activity-stream" }]); + + let sites = await TopSites.getLinksWithDefaults(); + for (let site of sites) { + Assert.ok(site.hostname); + } + + info("getLinksWithDefaults should not throw for null entries"); + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [null]); + await TopSites.getLinksWithDefaults(); + Assert.ok(true, "getLinksWithDefaults didn't throw"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_calls__fetchIcon() { + let sandbox = sinon.createSandbox(); + + info("getLinksWithDefaults should return sites that have a title"); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults(); + + sandbox.spy(TopSites, "_fetchIcon"); + let results = await TopSites.getLinksWithDefaults(); + Assert.ok(results.length, "Got back some results"); + Assert.equal(TopSites._fetchIcon.callCount, results.length); + for (let result of results) { + Assert.ok(TopSites._fetchIcon.calledWith(result)); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_calls__fetchScreenshot() { + let sandbox = sinon.createSandbox(); + + info( + "getLinksWithDefaults should call _fetchScreenshot when customScreenshotURL is set" + ); + + gGetTopSitesStub.resolves([]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com", customScreenshotURL: "custom" }]); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults(); + + sandbox.stub(TopSites, "_fetchScreenshot"); + await TopSites.getLinksWithDefaults(); + + Assert.ok(TopSites._fetchScreenshot.calledWith(sinon.match.object, "custom")); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getLinksWithDefaults_with_DiscoveryStream() { + let sandbox = sinon.createSandbox(); + info( + "getLinksWithDefaults should add a sponsored topsite from discoverystream to all the valid indices" + ); + + let makeStreamData = index => ({ + layout: [ + { + components: [ + { + placement: { + name: "sponsored-topsites", + }, + spocs: { + positions: [{ index }], + }, + }, + ], + }, + ], + spocs: { + data: { + "sponsored-topsites": { + items: [{ title: "test spoc", url: "https://test-spoc.com" }], + }, + }, + }, + }); + + let cleanup = stubTopSites(sandbox); + TopSites.refreshDefaults(); + + for (let i = 0; i < FAKE_LINKS.length; i++) { + TopSites.store.state.DiscoveryStream = makeStreamData(i); + const result = await TopSites.getLinksWithDefaults(); + const link = result[i]; + + Assert.equal(link.type, "SPOC"); + Assert.equal(link.title, "test spoc"); + Assert.equal(link.sponsored_position, i + 1); + Assert.equal(link.hostname, "test-spoc"); + Assert.equal(link.url, "https://test-spoc.com"); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_init() { + let sandbox = sinon.createSandbox(); + + sandbox.stub(NimbusFeatures.newtab, "onUpdate"); + + let cleanup = stubTopSites(sandbox); + + sandbox.stub(TopSites, "refresh"); + await TopSites.init(); + + info("TopSites.init should call refresh (broadcast: true)"); + Assert.ok(TopSites.refresh.calledOnce, "refresh called once"); + Assert.ok( + TopSites.refresh.calledWithExactly({ + broadcast: true, + isStartup: true, + }) + ); + + info("TopSites.init should initialise the storage"); + Assert.ok( + TopSites.store.dbStorage.getDbTable.calledOnce, + "getDbTable called once" + ); + Assert.ok( + TopSites.store.dbStorage.getDbTable.calledWithExactly("sectionPrefs") + ); + + info("TopSites.init should call onUpdate to set up Nimbus update listener"); + + Assert.ok( + NimbusFeatures.newtab.onUpdate.calledOnce, + "NimbusFeatures.newtab.onUpdate called once" + ); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh() { + let sandbox = sinon.createSandbox(); + + sandbox.stub(NimbusFeatures.newtab, "onUpdate"); + + let cleanup = stubTopSites(sandbox); + + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + info("TopSites.refresh should wait for tippytop to initialize"); + TopSites._tippyTopProvider.initialized = false; + sandbox.stub(TopSites._tippyTopProvider, "init").resolves(); + + await TopSites.refresh(); + + Assert.ok( + TopSites._tippyTopProvider.init.calledOnce, + "TopSites._tippyTopProvider.init called once" + ); + + info( + "TopSites.refresh should not init the tippyTopProvider if already initialized" + ); + TopSites._tippyTopProvider.initialized = true; + TopSites._tippyTopProvider.init.resetHistory(); + + await TopSites.refresh(); + + Assert.ok( + TopSites._tippyTopProvider.init.notCalled, + "tippyTopProvider not initted again" + ); + + info("TopSites.refresh should broadcast TOP_SITES_UPDATED"); + TopSites.store.dispatch.resetHistory(); + sandbox.stub(TopSites, "getLinksWithDefaults").resolves([]); + + await TopSites.refresh({ broadcast: true }); + + Assert.ok(TopSites.store.dispatch.calledOnce, "dispatch called once"); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.BroadcastToContent({ + type: at.TOP_SITES_UPDATED, + data: { links: [], pref: { collapsed: false } }, + }) + ) + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh_dispatch() { + let sandbox = sinon.createSandbox(); + + info("TopSites.refresh should dispatch an action with the links returned"); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + await TopSites.refresh({ broadcast: true }); + let reference = FAKE_LINKS.map(site => + Object.assign({}, site, { + hostname: shortURL(site), + typedBonus: true, + }) + ); + + Assert.ok(TopSites.store.dispatch.calledOnce, "Store.dispatch called once"); + Assert.equal( + TopSites.store.dispatch.firstCall.args[0].type, + at.TOP_SITES_UPDATED + ); + Assert.deepEqual( + TopSites.store.dispatch.firstCall.args[0].data.links, + reference + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh_empty_slots() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.refresh should handle empty slots in the resulting top sites array" + ); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + gGetTopSitesStub.resolves([FAKE_LINKS[0]]); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [ + null, + null, + FAKE_LINKS[1], + null, + null, + null, + null, + null, + FAKE_LINKS[2], + ]); + + await TopSites.refresh({ broadcast: true }); + + Assert.ok(TopSites.store.dispatch.calledOnce, "Store.dispatch called once"); + + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh_to_preloaded() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.refresh should dispatch AlsoToPreloaded when broadcast is false" + ); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + gGetTopSitesStub.resolves([]); + await TopSites.refresh({ broadcast: false }); + + Assert.ok(TopSites.store.dispatch.calledOnce, "Store.dispatch called once"); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.AlsoToPreloaded({ + type: at.TOP_SITES_UPDATED, + data: { links: [], pref: { collapsed: false } }, + }) + ) + ); + gGetTopSitesStub.resolves(FAKE_LINKS); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh_init_storage() { + let sandbox = sinon.createSandbox(); + + info("TopSites.refresh should not init storage of it's already initialized"); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + TopSites._storage.initialized = true; + + await TopSites.refresh({ broadcast: false }); + + Assert.ok( + TopSites._storage.init.notCalled, + "TopSites._storage.init was not called." + ); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_refresh_handles_indexedDB_errors() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.refresh should dispatch AlsoToPreloaded when broadcast is false" + ); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "_fetchIcon"); + TopSites._startedUp = true; + + TopSites._storage.get.throws(new Error()); + + try { + await TopSites.refresh({ broadcast: false }); + Assert.ok(true, "refresh should have succeeded"); + } catch (e) { + Assert.ok(false, "Should not have thrown"); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_updateSectionPrefs_on_UPDATE_SECTION_PREFS() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.onAction should call updateSectionPrefs on UPDATE_SECTION_PREFS" + ); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "updateSectionPrefs"); + TopSites.onAction({ + type: at.UPDATE_SECTION_PREFS, + data: { id: "topsites" }, + }); + + Assert.ok( + TopSites.updateSectionPrefs.calledOnce, + "TopSites.updateSectionPrefs called once" + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task( + async function test_updateSectionPrefs_dispatch_TOP_SITES_PREFS_UPDATED() { + let sandbox = sinon.createSandbox(); + + info("TopSites.updateSectionPrefs should dispatch TOP_SITES_PREFS_UPDATED"); + + let cleanup = stubTopSites(sandbox); + await TopSites.updateSectionPrefs({ collapsed: true }); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.BroadcastToContent({ + type: at.TOP_SITES_PREFS_UPDATED, + data: { pref: { collapsed: true } }, + }) + ) + ); + + sandbox.restore(); + await cleanup(); + } +); + +add_task(async function test_allocatePositions() { + let sandbox = sinon.createSandbox(); + + info("TopSites.allocationPositions should allocate positions and dispatch"); + + let cleanup = stubTopSites(sandbox); + + let sov = { + name: "SOV-20230518215316", + allocations: [ + { + position: 1, + allocation: [ + { + partner: "amp", + percentage: 100, + }, + { + partner: "moz-sales", + percentage: 0, + }, + ], + }, + { + position: 2, + allocation: [ + { + partner: "amp", + percentage: 80, + }, + { + partner: "moz-sales", + percentage: 20, + }, + ], + }, + ], + }; + + sandbox.stub(TopSites._contile, "sov").get(() => sov); + + sandbox.stub(Sampling, "ratioSample"); + Sampling.ratioSample.onCall(0).resolves(0); + Sampling.ratioSample.onCall(1).resolves(1); + + await TopSites.allocatePositions(); + + Assert.ok( + TopSites.store.dispatch.calledOnce, + "TopSites.store.dispatch called once" + ); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.OnlyToMain({ + type: at.SOV_UPDATED, + data: { + ready: true, + positions: [ + { position: 1, assignedPartner: "amp" }, + { position: 2, assignedPartner: "moz-sales" }, + ], + }, + }) + ) + ); + + Sampling.ratioSample.onCall(2).resolves(0); + Sampling.ratioSample.onCall(3).resolves(0); + + await TopSites.allocatePositions(); + + Assert.ok( + TopSites.store.dispatch.calledTwice, + "TopSites.store.dispatch called twice" + ); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.OnlyToMain({ + type: at.SOV_UPDATED, + data: { + ready: true, + positions: [ + { position: 1, assignedPartner: "amp" }, + { position: 2, assignedPartner: "amp" }, + ], + }, + }) + ) + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getScreenshotPreview() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.getScreenshotPreview should dispatch preview if request is succesful" + ); + + let cleanup = stubTopSites(sandbox); + await TopSites.getScreenshotPreview("custom", 1234); + + Assert.ok(TopSites.store.dispatch.calledOnce); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.OnlyToOneContent( + { + data: { preview: FAKE_SCREENSHOT, url: "custom" }, + type: at.PREVIEW_RESPONSE, + }, + 1234 + ) + ) + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_getScreenshotPreview() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.getScreenshotPreview should return empty string if request fails" + ); + + let cleanup = stubTopSites(sandbox); + Screenshots.getScreenshotForURL.resolves(Promise.resolve(null)); + await TopSites.getScreenshotPreview("custom", 1234); + + Assert.ok(TopSites.store.dispatch.calledOnce); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.OnlyToOneContent( + { + data: { preview: "", url: "custom" }, + type: at.PREVIEW_RESPONSE, + }, + 1234 + ) + ) + ); + + Screenshots.getScreenshotForURL.resolves(FAKE_SCREENSHOT); + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_onAction_part_1() { + let sandbox = sinon.createSandbox(); + + info("TopSites.onAction should call getScreenshotPreview on PREVIEW_REQUEST"); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "getScreenshotPreview"); + TopSites.onAction({ + type: at.PREVIEW_REQUEST, + data: { url: "foo" }, + meta: { fromTarget: 1234 }, + }); + + Assert.ok( + TopSites.getScreenshotPreview.calledOnce, + "TopSites.getScreenshotPreview called once" + ); + Assert.ok(TopSites.getScreenshotPreview.calledWithExactly("foo", 1234)); + + info("TopSites.onAction should refresh on SYSTEM_TICK"); + sandbox.stub(TopSites, "refresh"); + TopSites.onAction({ type: at.SYSTEM_TICK }); + + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + Assert.ok(TopSites.refresh.calledWithExactly({ broadcast: false })); + + info( + "TopSites.onAction should call with correct parameters on TOP_SITES_PIN" + ); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + sandbox.spy(TopSites, "pin"); + + let pinAction = { + type: at.TOP_SITES_PIN, + data: { site: { url: "foo.com" }, index: 7 }, + }; + TopSites.onAction(pinAction); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledWithExactly( + pinAction.data.site, + pinAction.data.index + ) + ); + Assert.ok( + TopSites.pin.calledOnce, + "TopSites.onAction should call pin on TOP_SITES_PIN" + ); + + info( + "TopSites.onAction should unblock a previously blocked top site if " + + "we are now adding it manually via 'Add a Top Site' option" + ); + sandbox.stub(NewTabUtils.blockedLinks, "unblock"); + pinAction = { + type: at.TOP_SITES_PIN, + data: { site: { url: "foo.com" }, index: -1 }, + }; + TopSites.onAction(pinAction); + Assert.ok( + NewTabUtils.blockedLinks.unblock.calledWith({ + url: pinAction.data.site.url, + }) + ); + + info("TopSites.onAction should call insert on TOP_SITES_INSERT"); + sandbox.stub(TopSites, "insert"); + let addAction = { + type: at.TOP_SITES_INSERT, + data: { site: { url: "foo.com" } }, + }; + + TopSites.onAction(addAction); + Assert.ok(TopSites.insert.calledOnce, "TopSites.insert called once"); + + info( + "TopSites.onAction should call unpin with correct parameters " + + "on TOP_SITES_UNPIN" + ); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [ + null, + null, + { url: "foo.com" }, + null, + null, + null, + null, + null, + FAKE_LINKS[0], + ]); + sandbox.stub(NewTabUtils.pinnedLinks, "unpin"); + + let unpinAction = { + type: at.TOP_SITES_UNPIN, + data: { site: { url: "foo.com" } }, + }; + TopSites.onAction(unpinAction); + Assert.ok( + NewTabUtils.pinnedLinks.unpin.calledOnce, + "NewTabUtils.pinnedLinks.unpin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.unpin.calledWith(unpinAction.data.site)); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_onAction_part_2() { + let sandbox = sinon.createSandbox(); + + info( + "TopSites.onAction should call refresh without a target if we clear " + + "history with PLACES_HISTORY_CLEARED" + ); + + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "refresh"); + TopSites.onAction({ type: at.PLACES_HISTORY_CLEARED }); + + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + Assert.ok(TopSites.refresh.calledWithExactly({ broadcast: true })); + + TopSites.refresh.resetHistory(); + + info( + "TopSites.onAction should call refresh without a target " + + "if we remove a Topsite from history" + ); + TopSites.onAction({ type: at.PLACES_LINKS_DELETED }); + + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + Assert.ok(TopSites.refresh.calledWithExactly({ broadcast: true })); + + info("TopSites.onAction should call init on INIT action"); + TopSites.onAction({ type: at.PLACES_LINKS_DELETED }); + sandbox.stub(TopSites, "init"); + TopSites.onAction({ type: at.INIT }); + Assert.ok(TopSites.init.calledOnce, "TopSites.init called once"); + + info("TopSites.onAction should call refresh on PLACES_LINK_BLOCKED action"); + TopSites.refresh.resetHistory(); + await TopSites.onAction({ type: at.PLACES_LINK_BLOCKED }); + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + Assert.ok(TopSites.refresh.calledWithExactly({ broadcast: true })); + + info("TopSites.onAction should call refresh on PLACES_LINKS_CHANGED action"); + TopSites.refresh.resetHistory(); + await TopSites.onAction({ type: at.PLACES_LINKS_CHANGED }); + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + Assert.ok(TopSites.refresh.calledWithExactly({ broadcast: false })); + + info( + "TopSites.onAction should call pin with correct args on " + + "TOP_SITES_INSERT without an index specified" + ); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + + let addAction = { + type: at.TOP_SITES_INSERT, + data: { site: { url: "foo.bar", label: "foo" } }, + }; + TopSites.onAction(addAction); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(addAction.data.site, 0)); + + info( + "TopSites.onAction should call pin with correct args on " + + "TOP_SITES_INSERT" + ); + NewTabUtils.pinnedLinks.pin.resetHistory(); + let dropAction = { + type: at.TOP_SITES_INSERT, + data: { site: { url: "foo.bar", label: "foo" }, index: 3 }, + }; + TopSites.onAction(dropAction); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(dropAction.data.site, 3)); + + // TopSites.init needs to actually run in order to register the observers that'll + // be removed in the following UNINIT test, otherwise uninit will throw. + TopSites.init.restore(); + TopSites.init(); + + info("TopSites.onAction should remove the expiration filter on UNINIT"); + sandbox.stub(PageThumbs, "removeExpirationFilter"); + TopSites.onAction({ type: "UNINIT" }); + Assert.ok( + PageThumbs.removeExpirationFilter.calledOnce, + "PageThumbs.removeExpirationFilter called once" + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_onAction_part_3() { + let sandbox = sinon.createSandbox(); + + let cleanup = stubTopSites(sandbox); + + info( + "TopSites.onAction should call updatePinnedSearchShortcuts " + + "on UPDATE_PINNED_SEARCH_SHORTCUTS action" + ); + sandbox.stub(TopSites, "updatePinnedSearchShortcuts"); + let addedShortcuts = [ + { + url: "https://google.com", + searchVendor: "google", + label: "google", + searchTopSite: true, + }, + ]; + await TopSites.onAction({ + type: at.UPDATE_PINNED_SEARCH_SHORTCUTS, + data: { addedShortcuts }, + }); + Assert.ok( + TopSites.updatePinnedSearchShortcuts.calledOnce, + "TopSites.updatePinnedSearchShortcuts called once" + ); + + info( + "TopSites.onAction should refresh from Contile on " + + "SHOW_SPONSORED_PREF if Contile is enabled" + ); + sandbox.spy(TopSites._contile, "refresh"); + let prefChangeAction = { + type: at.PREF_CHANGED, + data: { name: SHOW_SPONSORED_PREF }, + }; + sandbox.stub(NimbusFeatures.newtab, "getVariable").returns(true); + TopSites.onAction(prefChangeAction); + + Assert.ok( + TopSites._contile.refresh.calledOnce, + "TopSites._contile.refresh called once" + ); + + info( + "TopSites.onAction should not refresh from Contile on " + + "SHOW_SPONSORED_PREF if Contile is disabled" + ); + NimbusFeatures.newtab.getVariable.returns(false); + TopSites._contile.refresh.resetHistory(); + TopSites.onAction(prefChangeAction); + + Assert.ok( + !TopSites._contile.refresh.calledOnce, + "TopSites._contile.refresh never called" + ); + + info( + "TopSites.onAction should reset Contile cache prefs " + + "when SHOW_SPONSORED_PREF is false" + ); + Services.prefs.setStringPref(CONTILE_CACHE_PREF, "[]"); + Services.prefs.setIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + Math.round(Date.now() / 1000) + ); + Services.prefs.setIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF, 15 * 60); + prefChangeAction = { + type: at.PREF_CHANGED, + data: { name: SHOW_SPONSORED_PREF, value: false }, + }; + NimbusFeatures.newtab.getVariable.returns(true); + TopSites._contile.refresh.resetHistory(); + + TopSites.onAction(prefChangeAction); + Assert.ok(!Services.prefs.prefHasUserValue(CONTILE_CACHE_PREF)); + Assert.ok(!Services.prefs.prefHasUserValue(CONTILE_CACHE_LAST_FETCH_PREF)); + Assert.ok( + !Services.prefs.prefHasUserValue(CONTILE_CACHE_VALID_FOR_SECONDS_PREF) + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_insert_part_1() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + let cleanup = stubTopSites(sandbox); + + info("TopSites.insert should pin site in first slot of empty pinned list"); + Screenshots.getScreenshotForURL.resolves(Promise.resolve(null)); + await TopSites.getScreenshotPreview("custom", 1234); + + Assert.ok(TopSites.store.dispatch.calledOnce); + Assert.ok( + TopSites.store.dispatch.calledWithExactly( + ac.OnlyToOneContent( + { + data: { preview: "", url: "custom" }, + type: at.PREVIEW_RESPONSE, + }, + 1234 + ) + ) + ); + + Screenshots.getScreenshotForURL.resolves(FAKE_SCREENSHOT); + + { + info( + "TopSites.insert should pin site in first slot of pinned list with " + + "empty first slot" + ); + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, { url: "example.com" }]); + let site = { url: "foo.bar", label: "foo" }; + await TopSites.insert({ data: { site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info( + "TopSites.insert should move a pinned site in first slot to the " + + "next slot: part 1" + ); + let site1 = { url: "example.com" }; + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [site1]); + let site = { url: "foo.bar", label: "foo" }; + + await TopSites.insert({ data: { site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site1, 1)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info( + "TopSites.insert should move a pinned site in first slot to the " + + "next slot: part 2" + ); + let site1 = { url: "example.com" }; + let site2 = { url: "example.org" }; + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [site1, null, site2]); + let site = { url: "foo.bar", label: "foo" }; + await TopSites.insert({ data: { site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site1, 1)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_insert_part_2() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + let cleanup = stubTopSites(sandbox); + + { + info( + "TopSites.insert should unpin the last site if all slots are " + + "already pinned" + ); + let site1 = { url: "example.com" }; + let site2 = { url: "example.org" }; + let site3 = { url: "example.net" }; + let site4 = { url: "example.biz" }; + let site5 = { url: "example.info" }; + let site6 = { url: "example.news" }; + let site7 = { url: "example.lol" }; + let site8 = { url: "example.golf" }; + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [site1, site2, site3, site4, site5, site6, site7, site8]); + TopSites.store.state.Prefs.values.topSitesRows = 1; + let site = { url: "foo.bar", label: "foo" }; + await TopSites.insert({ data: { site } }); + Assert.equal( + NewTabUtils.pinnedLinks.pin.callCount, + 8, + "NewTabUtils.pinnedLinks.pin called 8 times" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site1, 1)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site2, 2)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site3, 3)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site4, 4)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site5, 5)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site6, 6)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site7, 7)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info("TopSites.insert should trigger refresh on TOP_SITES_INSERT"); + sandbox.stub(TopSites, "refresh"); + let addAction = { + type: at.TOP_SITES_INSERT, + data: { site: { url: "foo.com" } }, + }; + + await TopSites.insert(addAction); + + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + } + + { + info("TopSites.insert should correctly handle different index values"); + let index = -1; + let site = { url: "foo.bar", label: "foo" }; + let action = { data: { index, site } }; + + await TopSites.insert(action); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + + index = undefined; + await TopSites.insert(action); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 0)); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_insert_part_3() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + let cleanup = stubTopSites(sandbox); + + { + info("TopSites.insert should pin site in specified slot that is free"); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, { url: "example.com" }]); + + let site = { url: "foo.bar", label: "foo" }; + + await TopSites.insert({ data: { index: 2, site, draggedFromIndex: 0 } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 2)); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info( + "TopSites.insert should move a pinned site in specified slot " + + "to the next slot" + ); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, null, { url: "example.com" }]); + + let site = { url: "foo.bar", label: "foo" }; + + await TopSites.insert({ data: { index: 2, site, draggedFromIndex: 3 } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 2)); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledWith({ url: "example.com" }, 3) + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info( + "TopSites.insert should move pinned sites in the direction " + + "of the dragged site" + ); + + let site1 = { url: "foo.bar", label: "foo" }; + let site2 = { url: "example.com", label: "example" }; + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, null, site2]); + + await TopSites.insert({ + data: { index: 2, site: site1, draggedFromIndex: 0 }, + }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site1, 2)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site2, 1)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + + await TopSites.insert({ + data: { index: 2, site: site1, draggedFromIndex: 5 }, + }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site1, 2)); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site2, 3)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + { + info("TopSites.insert should not insert past the visible top sites"); + let site1 = { url: "foo.bar", label: "foo" }; + await TopSites.insert({ + data: { index: 42, site: site1, draggedFromIndex: 0 }, + }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.notCalled, + "NewTabUtils.pinnedLinks.pin wasn't called" + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_pin_part_1() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + sandbox.spy(TopSites.pinnedCache, "request"); + let cleanup = stubTopSites(sandbox); + + { + info("TopSites.pin should pin site in specified slot empty pinned list"); + let site = { + url: "foo.bar", + label: "foo", + customScreenshotURL: "screenshot", + }; + Assert.ok( + TopSites.pinnedCache.request.notCalled, + "TopSites.pinnedCache.request not called" + ); + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.called, + "NewTabUtils.pinnedLinks.pin called" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 2)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + TopSites.pinnedCache.request.resetHistory(); + } + + { + info( + "TopSites.pin should lookup the link object to update the custom " + + "screenshot" + ); + let site = { + url: "foo.bar", + label: "foo", + customScreenshotURL: "screenshot", + }; + Assert.ok( + TopSites.pinnedCache.request.notCalled, + "TopSites.pinnedCache.request not called" + ); + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + TopSites.pinnedCache.request.called, + "TopSites.pinnedCache.request called" + ); + NewTabUtils.pinnedLinks.pin.resetHistory(); + TopSites.pinnedCache.request.resetHistory(); + } + + { + info( + "TopSites.pin should lookup the link object to update the custom " + + "screenshot when the custom screenshot is initially null" + ); + let site = { + url: "foo.bar", + label: "foo", + customScreenshotURL: null, + }; + Assert.ok( + TopSites.pinnedCache.request.notCalled, + "TopSites.pinnedCache.request not called" + ); + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + TopSites.pinnedCache.request.called, + "TopSites.pinnedCache.request called" + ); + NewTabUtils.pinnedLinks.pin.resetHistory(); + TopSites.pinnedCache.request.resetHistory(); + } + + { + info( + "TopSites.pin should not do a link object lookup if custom " + + "screenshot field is not set" + ); + let site = { url: "foo.bar", label: "foo" }; + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + !TopSites.pinnedCache.request.called, + "TopSites.pinnedCache.request never called" + ); + NewTabUtils.pinnedLinks.pin.resetHistory(); + TopSites.pinnedCache.request.resetHistory(); + } + + { + info( + "TopSites.pin should pin site in specified slot of pinned " + + "list that is free" + ); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, { url: "example.com" }]); + + let site = { url: "foo.bar", label: "foo" }; + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 2)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + } + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_pin_part_2() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + + { + info("TopSites.pin should save the searchTopSite attribute if set"); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, { url: "example.com" }]); + + let site = { url: "foo.bar", label: "foo", searchTopSite: true }; + let cleanup = stubTopSites(sandbox); + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.firstCall.args[0].searchTopSite); + NewTabUtils.pinnedLinks.pin.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.pin should NOT move a pinned site in specified " + + "slot to the next slot" + ); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [null, null, { url: "example.com" }]); + + let site = { url: "foo.bar", label: "foo" }; + let cleanup = stubTopSites(sandbox); + await TopSites.pin({ data: { index: 2, site } }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok(NewTabUtils.pinnedLinks.pin.calledWith(site, 2)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.pin should properly update LinksCache object " + + "properties between migrations" + ); + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "https://foo.com/" }]); + + let cleanup = stubTopSites(sandbox); + let pinnedLinks = await TopSites.pinnedCache.request(); + Assert.equal(pinnedLinks.length, 1); + TopSites.pinnedCache.expire(); + + pinnedLinks[0].__sharedCache.updateLink("screenshot", "foo"); + + pinnedLinks = await TopSites.pinnedCache.request(); + Assert.equal(pinnedLinks[0].screenshot, "foo"); + + // Force cache expiration in order to trigger a migration of objects + TopSites.pinnedCache.expire(); + pinnedLinks[0].__sharedCache.updateLink("screenshot", "bar"); + + pinnedLinks = await TopSites.pinnedCache.request(); + Assert.equal(pinnedLinks[0].screenshot, "bar"); + await cleanup(); + } + + sandbox.restore(); +}); + +add_task(async function test_pin_part_3() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + sandbox.spy(TopSites, "insert"); + + { + info("TopSites.pin should call insert if index < 0"); + let site = { url: "foo.bar", label: "foo" }; + let action = { data: { index: -1, site } }; + let cleanup = stubTopSites(sandbox); + await TopSites.pin(action); + + Assert.ok(TopSites.insert.calledOnce, "TopSites.insert called once"); + Assert.ok(TopSites.insert.calledWithExactly(action)); + NewTabUtils.pinnedLinks.pin.resetHistory(); + TopSites.insert.resetHistory(); + await cleanup(); + } + + { + info("TopSites.pin should not call insert if index == 0"); + let site = { url: "foo.bar", label: "foo" }; + let action = { data: { index: 0, site } }; + let cleanup = stubTopSites(sandbox); + await TopSites.pin(action); + + Assert.ok(!TopSites.insert.called, "TopSites.insert not called"); + NewTabUtils.pinnedLinks.pin.resetHistory(); + await cleanup(); + } + + { + info("TopSites.pin should trigger refresh on TOP_SITES_PIN"); + let cleanup = stubTopSites(sandbox); + sandbox.stub(TopSites, "refresh"); + let pinExistingAction = { + type: at.TOP_SITES_PIN, + data: { site: FAKE_LINKS[4], index: 4 }, + }; + + await TopSites.pin(pinExistingAction); + + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + NewTabUtils.pinnedLinks.pin.resetHistory(); + await cleanup(); + } + + sandbox.restore(); +}); + +add_task(async function test_integration() { + let sandbox = sinon.createSandbox(); + + info("Test adding a pinned site and removing it with actions"); + let cleanup = stubTopSites(sandbox); + + let resolvers = []; + TopSites.store.dispatch = sandbox.stub().callsFake(() => { + resolvers.shift()(); + }); + TopSites._startedUp = true; + sandbox.stub(TopSites, "_fetchScreenshot"); + + let forDispatch = action => + new Promise(resolve => { + resolvers.push(resolve); + TopSites.onAction(action); + }); + + TopSites._requestRichIcon = sandbox.stub(); + let url = "https://pin.me"; + sandbox.stub(NewTabUtils.pinnedLinks, "pin").callsFake(link => { + NewTabUtils.pinnedLinks.links.push(link); + }); + + await forDispatch({ type: at.TOP_SITES_INSERT, data: { site: { url } } }); + NewTabUtils.pinnedLinks.links.pop(); + await forDispatch({ type: at.PLACES_LINK_BLOCKED }); + + Assert.ok( + TopSites.store.dispatch.calledTwice, + "TopSites.store.dispatch called twice" + ); + Assert.equal( + TopSites.store.dispatch.firstCall.args[0].data.links[0].url, + url + ); + Assert.equal( + TopSites.store.dispatch.secondCall.args[0].data.links[0].url, + FAKE_LINKS[0].url + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_improvesearch_noDefaultSearchTile_experiment() { + let sandbox = sinon.createSandbox(); + const NO_DEFAULT_SEARCH_TILE_PREF = "improvesearch.noDefaultSearchTile"; + + sandbox.stub(SearchService.prototype, "getDefault").resolves({ + identifier: "google", + searchForm: "google.com", + }); + + { + info( + "TopSites.getLinksWithDefaults should filter out alexa top 5 " + + "search from the default sites" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = true; + let top5Test = [ + "https://google.com", + "https://search.yahoo.com", + "https://yahoo.com", + "https://bing.com", + "https://ask.com", + "https://duckduckgo.com", + ]; + + gGetTopSitesStub.resolves([ + { url: "https://amazon.com" }, + ...top5Test.map(url => ({ url })), + ]); + + const urlsReturned = (await TopSites.getLinksWithDefaults()).map( + link => link.url + ); + Assert.ok( + urlsReturned.includes("https://amazon.com"), + "amazon included in default links" + ); + top5Test.forEach(url => + Assert.ok(!urlsReturned.includes(url), `Should not include ${url}`) + ); + + gGetTopSitesStub.resolves(FAKE_LINKS); + await cleanup(); + } + + { + info( + "TopSites.getLinksWithDefaults should not filter out alexa, default " + + "search from the query results if the experiment pref is off" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = false; + + gGetTopSitesStub.resolves([ + { url: "https://google.com" }, + { url: "https://foo.com" }, + { url: "https://duckduckgo" }, + ]); + let urlsReturned = (await TopSites.getLinksWithDefaults()).map( + link => link.url + ); + + Assert.ok(urlsReturned.includes("https://google.com")); + gGetTopSitesStub.resolves(FAKE_LINKS); + await cleanup(); + } + + { + info( + "TopSites.getLinksWithDefaults should filter out the current " + + "default search from the default sites" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = true; + + sandbox.stub(TopSites, "_currentSearchHostname").get(() => "amazon"); + TopSites.onAction({ + type: at.PREFS_INITIAL_VALUES, + data: { "default.sites": "google.com,amazon.com" }, + }); + gGetTopSitesStub.resolves([{ url: "https://foo.com" }]); + + let urlsReturned = (await TopSites.getLinksWithDefaults()).map( + link => link.url + ); + Assert.ok(!urlsReturned.includes("https://amazon.com")); + + gGetTopSitesStub.resolves(FAKE_LINKS); + await cleanup(); + } + + { + info( + "TopSites.getLinksWithDefaults should not filter out current " + + "default search from pinned sites even if it matches the current " + + "default search" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = true; + + sandbox + .stub(NewTabUtils.pinnedLinks, "links") + .get(() => [{ url: "google.com" }]); + gGetTopSitesStub.resolves([{ url: "https://foo.com" }]); + + let urlsReturned = (await TopSites.getLinksWithDefaults()).map( + link => link.url + ); + Assert.ok(urlsReturned.includes("google.com")); + + gGetTopSitesStub.resolves(FAKE_LINKS); + await cleanup(); + } + + sandbox.restore(); +}); + +add_task( + async function test_improvesearch_noDefaultSearchTile_experiment_part_2() { + let sandbox = sinon.createSandbox(); + const NO_DEFAULT_SEARCH_TILE_PREF = "improvesearch.noDefaultSearchTile"; + + sandbox.stub(SearchService.prototype, "getDefault").resolves({ + identifier: "google", + searchForm: "google.com", + }); + + sandbox.stub(TopSites, "refresh"); + + { + info( + "TopSites.getLinksWithDefaults should call refresh and set " + + "._currentSearchHostname to the new engine hostname when the " + + "default search engine has been set" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = true; + + TopSites.observe( + null, + "browser-search-engine-modified", + "engine-default" + ); + Assert.equal(TopSites._currentSearchHostname, "duckduckgo"); + Assert.ok(TopSites.refresh.calledOnce, "TopSites.refresh called once"); + + gGetTopSitesStub.resolves(FAKE_LINKS); + TopSites.refresh.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.getLinksWithDefaults should call refresh when the " + + "experiment pref has changed" + ); + let cleanup = stubTopSites(sandbox); + TopSites.store.state.Prefs.values[NO_DEFAULT_SEARCH_TILE_PREF] = true; + + TopSites.onAction({ + type: at.PREF_CHANGED, + data: { name: NO_DEFAULT_SEARCH_TILE_PREF, value: true }, + }); + Assert.ok( + TopSites.refresh.calledOnce, + "TopSites.refresh was called once" + ); + + TopSites.onAction({ + type: at.PREF_CHANGED, + data: { name: NO_DEFAULT_SEARCH_TILE_PREF, value: false }, + }); + Assert.ok( + TopSites.refresh.calledTwice, + "TopSites.refresh was called twice" + ); + + gGetTopSitesStub.resolves(FAKE_LINKS); + TopSites.refresh.resetHistory(); + await cleanup(); + } + + sandbox.restore(); + } +); + +// eslint-disable-next-line max-statements +add_task(async function test_improvesearch_topSitesSearchShortcuts() { + let sandbox = sinon.createSandbox(); + let searchEngines = [{ aliases: ["@google"] }, { aliases: ["@amazon"] }]; + sandbox + .stub(SearchService.prototype, "getAppProvidedEngines") + .resolves(searchEngines); + sandbox.stub(NewTabUtils.pinnedLinks, "pin").callsFake((site, index) => { + NewTabUtils.pinnedLinks.links[index] = site; + }); + + let prepTopSites = () => { + TopSites.store.state.Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT_PREF] = true; + TopSites.store.state.Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF] = + "google,amazon"; + TopSites.store.state.Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF] = ""; + }; + + { + info( + "TopSites should updateCustomSearchShortcuts when experiment " + + "pref is turned on" + ); + let cleanup = stubTopSites(sandbox); + prepTopSites(); + TopSites.store.state.Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT_PREF] = false; + sandbox.spy(TopSites, "updateCustomSearchShortcuts"); + + // turn the experiment on + TopSites.onAction({ + type: at.PREF_CHANGED, + data: { name: SEARCH_SHORTCUTS_EXPERIMENT_PREF, value: true }, + }); + + Assert.ok( + TopSites.updateCustomSearchShortcuts.calledOnce, + "TopSites.updateCustomSearchShortcuts called once" + ); + TopSites.updateCustomSearchShortcuts.restore(); + await cleanup(); + } + + { + info( + "TopSites should filter out default top sites that match a " + + "hostname of a search shortcut if previously blocked" + ); + let cleanup = stubTopSites(sandbox); + prepTopSites(); + TopSites.refreshDefaults("https://amazon.ca"); + sandbox + .stub(NewTabUtils.blockedLinks, "links") + .value([{ url: "https://amazon.com" }]); + sandbox.stub(NewTabUtils.blockedLinks, "isBlocked").callsFake(site => { + return NewTabUtils.blockedLinks.links[0].url === site.url; + }); + + let urlsReturned = (await TopSites.getLinksWithDefaults()).map( + link => link.url + ); + Assert.ok(!urlsReturned.includes("https://amazon.ca")); + await cleanup(); + } + + { + info("TopSites should update frecent search topsite icon"); + let cleanup = stubTopSites(sandbox); + prepTopSites(); + sandbox.stub(TopSites._tippyTopProvider, "processSite").callsFake(site => { + site.tippyTopIcon = "icon.png"; + site.backgroundColor = "#fff"; + return site; + }); + gGetTopSitesStub.resolves([{ url: "https://google.com" }]); + + let urlsReturned = await TopSites.getLinksWithDefaults(); + + let defaultSearchTopsite = urlsReturned.find( + s => s.url === "https://google.com" + ); + Assert.ok(defaultSearchTopsite.searchTopSite); + Assert.equal(defaultSearchTopsite.tippyTopIcon, "icon.png"); + Assert.equal(defaultSearchTopsite.backgroundColor, "#fff"); + gGetTopSitesStub.resolves(FAKE_LINKS); + TopSites._tippyTopProvider.processSite.restore(); + await cleanup(); + } + + { + info("TopSites should update default search topsite icon"); + let cleanup = stubTopSites(sandbox); + prepTopSites(); + sandbox.stub(TopSites._tippyTopProvider, "processSite").callsFake(site => { + site.tippyTopIcon = "icon.png"; + site.backgroundColor = "#fff"; + return site; + }); + gGetTopSitesStub.resolves([{ url: "https://foo.com" }]); + TopSites.onAction({ + type: at.PREFS_INITIAL_VALUES, + data: { "default.sites": "google.com,amazon.com" }, + }); + + let urlsReturned = await TopSites.getLinksWithDefaults(); + + let defaultSearchTopsite = urlsReturned.find( + s => s.url === "https://amazon.com" + ); + Assert.ok(defaultSearchTopsite.searchTopSite); + Assert.equal(defaultSearchTopsite.tippyTopIcon, "icon.png"); + Assert.equal(defaultSearchTopsite.backgroundColor, "#fff"); + gGetTopSitesStub.resolves(FAKE_LINKS); + TopSites._tippyTopProvider.processSite.restore(); + TopSites.store.dispatch.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites should dispatch UPDATE_SEARCH_SHORTCUTS on " + + "updateCustomSearchShortcuts" + ); + let cleanup = stubTopSites(sandbox); + prepTopSites(); + TopSites.store.state.Prefs.values[ + "improvesearch.noDefaultSearchTile" + ] = true; + await TopSites.updateCustomSearchShortcuts(); + Assert.ok( + TopSites.store.dispatch.calledOnce, + "TopSites.store.dispatch called once" + ); + Assert.ok( + TopSites.store.dispatch.calledWith({ + data: { + searchShortcuts: [ + { + keyword: "@google", + shortURL: "google", + url: "https://google.com", + backgroundColor: undefined, + smallFavicon: + "chrome://activity-stream/content/data/content/tippytop/favicons/google-com.ico", + tippyTopIcon: + "chrome://activity-stream/content/data/content/tippytop/images/google-com@2x.png", + }, + { + keyword: "@amazon", + shortURL: "amazon", + url: "https://amazon.com", + backgroundColor: undefined, + smallFavicon: + "chrome://activity-stream/content/data/content/tippytop/favicons/amazon.ico", + tippyTopIcon: + "chrome://activity-stream/content/data/content/tippytop/images/amazon@2x.png", + }, + ], + }, + meta: { + from: "ActivityStream:Main", + to: "ActivityStream:Content", + isStartup: false, + }, + type: "UPDATE_SEARCH_SHORTCUTS", + }) + ); + await cleanup(); + } + + sandbox.restore(); +}); + +add_task(async function test_updatePinnedSearchShortcuts() { + let sandbox = sinon.createSandbox(); + sandbox.stub(NewTabUtils.pinnedLinks, "pin"); + sandbox.stub(NewTabUtils.pinnedLinks, "unpin"); + + { + info( + "TopSites.updatePinnedSearchShortcuts should unpin a " + + "shortcut in deletedShortcuts" + ); + let cleanup = stubTopSites(sandbox); + + let deletedShortcuts = [ + { + url: "https://google.com", + searchVendor: "google", + label: "google", + searchTopSite: true, + }, + ]; + let addedShortcuts = []; + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [ + null, + null, + { + url: "https://amazon.com", + searchVendor: "amazon", + label: "amazon", + searchTopSite: true, + }, + ]); + TopSites.updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }); + Assert.ok( + NewTabUtils.pinnedLinks.pin.notCalled, + "NewTabUtils.pinnedLinks.pin not called" + ); + Assert.ok( + NewTabUtils.pinnedLinks.unpin.calledOnce, + "NewTabUtils.pinnedLinks.unpin called once" + ); + Assert.ok( + NewTabUtils.pinnedLinks.unpin.calledWith({ + url: "https://google.com", + }) + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + NewTabUtils.pinnedLinks.unpin.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.updatePinnedSearchShortcuts should pin a shortcut " + + "in addedShortcuts" + ); + let cleanup = stubTopSites(sandbox); + + let addedShortcuts = [ + { + url: "https://google.com", + searchVendor: "google", + label: "google", + searchTopSite: true, + }, + ]; + let deletedShortcuts = []; + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [ + null, + null, + { + url: "https://amazon.com", + searchVendor: "amazon", + label: "amazon", + searchTopSite: true, + }, + ]); + TopSites.updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }); + Assert.ok( + NewTabUtils.pinnedLinks.unpin.notCalled, + "NewTabUtils.pinnedLinks.unpin not called" + ); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledOnce, + "NewTabUtils.pinnedLinks.pin called once" + ); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledWith( + { + label: "google", + searchTopSite: true, + searchVendor: "google", + url: "https://google.com", + }, + 0 + ) + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + NewTabUtils.pinnedLinks.unpin.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.updatePinnedSearchShortcuts should pin and unpin " + + "in the same action" + ); + let cleanup = stubTopSites(sandbox); + + let addedShortcuts = [ + { + url: "https://google.com", + searchVendor: "google", + label: "google", + searchTopSite: true, + }, + { + url: "https://ebay.com", + searchVendor: "ebay", + label: "ebay", + searchTopSite: true, + }, + ]; + let deletedShortcuts = [ + { + url: "https://amazon.com", + searchVendor: "amazon", + label: "amazon", + searchTopSite: true, + }, + ]; + + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => [ + { url: "https://foo.com" }, + { + url: "https://amazon.com", + searchVendor: "amazon", + label: "amazon", + searchTopSite: true, + }, + ]); + TopSites.updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }); + + Assert.ok( + NewTabUtils.pinnedLinks.unpin.calledOnce, + "NewTabUtils.pinnedLinks.unpin called once" + ); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledTwice, + "NewTabUtils.pinnedLinks.pin called twice" + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + NewTabUtils.pinnedLinks.unpin.resetHistory(); + await cleanup(); + } + + { + info( + "TopSites.updatePinnedSearchShortcuts should pin a shortcut in " + + "addedShortcuts even if pinnedLinks is full" + ); + let cleanup = stubTopSites(sandbox); + + let addedShortcuts = [ + { + url: "https://google.com", + searchVendor: "google", + label: "google", + searchTopSite: true, + }, + ]; + let deletedShortcuts = []; + sandbox.stub(NewTabUtils.pinnedLinks, "links").get(() => FAKE_LINKS); + TopSites.updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }); + + Assert.ok( + NewTabUtils.pinnedLinks.unpin.notCalled, + "NewTabUtils.pinnedLinks.unpin not called" + ); + Assert.ok( + NewTabUtils.pinnedLinks.pin.calledWith( + { label: "google", searchTopSite: true, url: "https://google.com" }, + 0 + ), + "NewTabUtils.pinnedLinks.unpin not called" + ); + + NewTabUtils.pinnedLinks.pin.resetHistory(); + NewTabUtils.pinnedLinks.unpin.resetHistory(); + await cleanup(); + } + + sandbox.restore(); +}); + +// eslint-disable-next-line max-statements +add_task(async function test_ContileIntegration() { + let sandbox = sinon.createSandbox(); + Services.prefs.setStringPref( + TOP_SITES_BLOCKED_SPONSORS_PREF, + `["foo","bar"]` + ); + sandbox.stub(NimbusFeatures.newtab, "getVariable").returns(true); + + let fetchStub; + + let prepTopSites = () => { + TopSites.store.state.Prefs.values[SHOW_SPONSORED_PREF] = true; + fetchStub = sandbox.stub(TopSites, "fetch"); + function cleanupPrep() { + TopSites._contile._sites = []; + fetchStub.restore(); + } + return cleanupPrep; + }; + + { + info("TopSites._fetchSites should fetch sites from Contile"); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://www.test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + Assert.equal(TopSites._contile.sites.length, 2); + await cleanup(); + cleanupPrep(); + } + + { + info("TopSites._fetchSites should call allocatePositions"); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + sandbox.stub(TopSites, "allocatePositions").resolves(); + await TopSites._contile.refresh(); + + Assert.ok( + TopSites.allocatePositions.calledOnce, + "TopSites.allocatePositions called once" + ); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should fetch SOV (Share-of-Voice) " + + "settings from Contile" + ); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + let sov = { + name: "SOV-20230518215316", + allocations: [ + { + position: 1, + allocation: [ + { + partner: "foo", + percentage: 100, + }, + { + partner: "bar", + percentage: 0, + }, + ], + }, + { + position: 2, + allocation: [ + { + partner: "foo", + percentage: 80, + }, + { + partner: "bar", + percentage: 20, + }, + ], + }, + ], + }; + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + sov: btoa(JSON.stringify(sov)), + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://www.test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + Assert.deepEqual(TopSites._contile.sov, sov); + Assert.equal(TopSites._contile.sites.length, 2); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should not fetch from Contile if " + + "it's not enabled" + ); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + NimbusFeatures.newtab.getVariable.reset(); + NimbusFeatures.newtab.getVariable.returns(false); + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetchStub.notCalled, "TopSites.fetch was not called"); + Assert.ok(!fetched); + Assert.equal(TopSites._contile.sites.length, 0); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should still return two tiles when Contile " + + "provides more than 2 tiles and filtering results in more than 2 tiles" + ); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + NimbusFeatures.newtab.getVariable.reset(); + NimbusFeatures.newtab.getVariable.onCall(0).returns(true); + NimbusFeatures.newtab.getVariable.onCall(1).returns(true); + + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://foo.com", + image_url: "images/foo-com.png", + click_url: "https://www.foo-click.com", + impression_url: "https://www.foo-impression.com", + name: "foo", + }, + { + url: "https://bar.com", + image_url: "images/bar-com.png", + click_url: "https://www.bar-click.com", + impression_url: "https://www.bar-impression.com", + name: "bar", + }, + { + url: "https://test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + { + url: "https://test2.com", + image_url: "images/test2-com.png", + click_url: "https://www.test2-click.com", + impression_url: "https://www.test2-impression.com", + name: "test2", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + // Both "foo" and "bar" should be filtered + Assert.equal(TopSites._contile.sites.length, 2); + Assert.equal(TopSites._contile.sites[0].url, "https://www.test.com"); + Assert.equal(TopSites._contile.sites[1].url, "https://test1.com"); + + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should still return two tiles with " + + "replacement if the Nimbus variable was unset" + ); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + NimbusFeatures.newtab.getVariable.reset(); + NimbusFeatures.newtab.getVariable.onCall(0).returns(true); + NimbusFeatures.newtab.getVariable.onCall(1).returns(undefined); + + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://foo.com", + image_url: "images/foo-com.png", + click_url: "https://www.foo-click.com", + impression_url: "https://www.foo-impression.com", + name: "foo", + }, + { + url: "https://test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + Assert.equal(TopSites._contile.sites.length, 2); + Assert.equal(TopSites._contile.sites[0].url, "https://www.test.com"); + Assert.equal(TopSites._contile.sites[1].url, "https://test1.com"); + + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info("TopSites._fetchSites should filter the blocked sponsors"); + NimbusFeatures.newtab.getVariable.returns(true); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://foo.com", + image_url: "images/foo-com.png", + click_url: "https://www.foo-click.com", + impression_url: "https://www.foo-impression.com", + name: "foo", + }, + { + url: "https://bar.com", + image_url: "images/bar-com.png", + click_url: "https://www.bar-click.com", + impression_url: "https://www.bar-impression.com", + name: "bar", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + // Both "foo" and "bar" should be filtered + Assert.equal(TopSites._contile.sites.length, 1); + Assert.equal(TopSites._contile.sites[0].url, "https://www.test.com"); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should return false when Contile returns " + + "with error status and no values are stored in cache prefs" + ); + NimbusFeatures.newtab.getVariable.returns(true); + Services.prefs.setStringPref(CONTILE_CACHE_PREF, "[]"); + Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, 0); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + fetchStub.resolves({ + ok: false, + status: 500, + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(!fetched); + Assert.ok(!TopSites._contile.sites.length); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should return false when Contile " + + "returns with error status and cached tiles are expried" + ); + NimbusFeatures.newtab.getVariable.returns(true); + Services.prefs.setStringPref(CONTILE_CACHE_PREF, "[]"); + const THIRTY_MINUTES_AGO_IN_SECONDS = + Math.round(Date.now() / 1000) - 60 * 30; + Services.prefs.setIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + THIRTY_MINUTES_AGO_IN_SECONDS + ); + Services.prefs.setIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF, 60 * 15); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + fetchStub.resolves({ + ok: false, + status: 500, + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(!fetched); + Assert.ok(!TopSites._contile.sites.length); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should handle invalid payload " + + "properly from Contile" + ); + NimbusFeatures.newtab.getVariable.returns(true); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + fetchStub.resolves({ + ok: true, + status: 200, + json: () => + Promise.resolve({ + unknown: [], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(!fetched); + Assert.ok(!TopSites._contile.sites.length); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should handle empty payload properly " + + "from Contile" + ); + NimbusFeatures.newtab.getVariable.returns(true); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + Assert.ok(!TopSites._contile.sites.length); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info("TopSites._fetchSites should handle no content properly from Contile"); + NimbusFeatures.newtab.getVariable.returns(true); + + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + fetchStub.resolves({ ok: true, status: 204 }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(!fetched); + Assert.ok(!TopSites._contile.sites.length); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should set Caching Prefs after " + + "a successful request" + ); + NimbusFeatures.newtab.getVariable.returns(true); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + let tiles = [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://www.test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ]; + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles, + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + Assert.ok(fetched); + Assert.equal( + Services.prefs.getStringPref(CONTILE_CACHE_PREF), + JSON.stringify(tiles) + ); + Assert.equal( + Services.prefs.getIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF), + 11322 + ); + await cleanup(); + cleanupPrep(); + } + + { + info( + "TopSites._fetchSites should return cached valid tiles " + + "when Contile returns error status" + ); + NimbusFeatures.newtab.getVariable.returns(true); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + let tiles = [ + { + url: "https://www.test-cached.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://www.test1-cached.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ]; + + Services.prefs.setStringPref(CONTILE_CACHE_PREF, JSON.stringify(tiles)); + Services.prefs.setIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF, 60 * 15); + Services.prefs.setIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + Math.round(Date.now() / 1000) + ); + + fetchStub.resolves({ + status: 304, + }); + + let fetched = await TopSites._contile._fetchSites(); + Assert.ok(fetched); + Assert.equal(TopSites._contile.sites.length, 2); + Assert.equal(TopSites._contile.sites[0].url, "https://www.test-cached.com"); + Assert.equal( + TopSites._contile.sites[1].url, + "https://www.test1-cached.com" + ); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should not be successful when contile " + + "returns an error and no valid tiles are cached" + ); + NimbusFeatures.newtab.getVariable.returns(true); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + Services.prefs.setStringPref(CONTILE_CACHE_PREF, "[]"); + Services.prefs.setIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF, 0); + Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, 0); + + fetchStub.resolves({ + status: 500, + }); + + let fetched = await TopSites._contile._fetchSites(); + Assert.ok(!fetched); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should return cached valid tiles " + + "filtering blocked tiles when Contile returns error status" + ); + NimbusFeatures.newtab.getVariable.returns(true); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + let tiles = [ + { + url: "https://foo.com", + image_url: "images/foo-com.png", + click_url: "https://www.foo-click.com", + impression_url: "https://www.foo-impression.com", + name: "foo", + }, + { + url: "https://www.test1-cached.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + ]; + Services.prefs.setStringPref(CONTILE_CACHE_PREF, JSON.stringify(tiles)); + Services.prefs.setIntPref(CONTILE_CACHE_VALID_FOR_SECONDS_PREF, 60 * 15); + Services.prefs.setIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + Math.round(Date.now() / 1000) + ); + + fetchStub.resolves({ + status: 304, + }); + + let fetched = await TopSites._contile._fetchSites(); + Assert.ok(fetched); + Assert.equal(TopSites._contile.sites.length, 1); + Assert.equal( + TopSites._contile.sites[0].url, + "https://www.test1-cached.com" + ); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + { + info( + "TopSites._fetchSites should still return 3 tiles when nimbus " + + "variable overrides max num of sponsored contile tiles" + ); + NimbusFeatures.newtab.getVariable.returns(true); + let cleanup = stubTopSites(sandbox); + let cleanupPrep = prepTopSites(); + + sandbox.stub(NimbusFeatures.pocketNewtab, "getVariable").returns(3); + fetchStub.resolves({ + ok: true, + status: 200, + headers: new Map([ + ["cache-control", "private, max-age=859, stale-if-error=10463"], + ]), + json: () => + Promise.resolve({ + tiles: [ + { + url: "https://www.test.com", + image_url: "images/test-com.png", + click_url: "https://www.test-click.com", + impression_url: "https://www.test-impression.com", + name: "test", + }, + { + url: "https://test1.com", + image_url: "images/test1-com.png", + click_url: "https://www.test1-click.com", + impression_url: "https://www.test1-impression.com", + name: "test1", + }, + { + url: "https://test2.com", + image_url: "images/test2-com.png", + click_url: "https://www.test2-click.com", + impression_url: "https://www.test2-impression.com", + name: "test2", + }, + ], + }), + }); + + let fetched = await TopSites._contile._fetchSites(); + + Assert.ok(fetched); + Assert.equal(TopSites._contile.sites.length, 3); + Assert.equal(TopSites._contile.sites[0].url, "https://www.test.com"); + Assert.equal(TopSites._contile.sites[1].url, "https://test1.com"); + Assert.equal(TopSites._contile.sites[2].url, "https://test2.com"); + await cleanup(); + cleanupPrep(); + fetchStub.restore(); + } + + Services.prefs.clearUserPref(TOP_SITES_BLOCKED_SPONSORS_PREF); + sandbox.restore(); +}); diff --git a/browser/components/topsites/test/unit/xpcshell.toml b/browser/components/topsites/test/unit/xpcshell.toml new file mode 100644 index 0000000000..98b0fc5360 --- /dev/null +++ b/browser/components/topsites/test/unit/xpcshell.toml @@ -0,0 +1,4 @@ +[DEFAULT] +firefox-appdir = "browser" + +["test_top_sites.js"] |