diff options
Diffstat (limited to '')
23 files changed, 1307 insertions, 63 deletions
diff --git a/browser/components/search/test/browser/telemetry/browser.toml b/browser/components/search/test/browser/telemetry/browser.toml index 660fc4eae2..5e42a9187d 100644 --- a/browser/components/search/test/browser/telemetry/browser.toml +++ b/browser/components/search/test/browser/telemetry/browser.toml @@ -50,6 +50,15 @@ support-files = ["searchTelemetryDomainCategorizationReporting.html"] ["browser_search_telemetry_domain_categorization_extraction.js"] support-files = ["searchTelemetryDomainExtraction.html"] +["browser_search_telemetry_domain_categorization_no_sponsored_values.js"] +support-files = ["searchTelemetryDomainCategorizationReportingWithoutAds.html"] + +["browser_search_telemetry_domain_categorization_ping_submission.js"] +support-files = [ + "searchTelemetryDomainCategorizationReporting.html", + "searchTelemetryDomainExtraction.html", +] + ["browser_search_telemetry_domain_categorization_region.js"] support-files = ["searchTelemetryDomainCategorizationReporting.html"] @@ -103,13 +112,6 @@ support-files = [ "searchTelemetryAd_searchbox_with_content.html^headers^", ] -["browser_search_telemetry_engagement_non_ad.js"] -support-files = [ - "searchTelemetryAd_searchbox_with_content.html", - "searchTelemetryAd_searchbox_with_content.html^headers^", - "serp.css", -] - ["browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js"] support-files = [ "searchTelemetryAd_searchbox_with_redirecting_links.html", @@ -118,6 +120,13 @@ support-files = [ "serp.css", ] +["browser_search_telemetry_engagement_non_ad.js"] +support-files = [ + "searchTelemetryAd_searchbox_with_content.html", + "searchTelemetryAd_searchbox_with_content.html^headers^", + "serp.css", +] + ["browser_search_telemetry_engagement_query_params.js"] support-files = [ "searchTelemetryAd_components_query_parameters.html", diff --git a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js index e73a9601d4..8e9db64fae 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js +++ b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js @@ -74,11 +74,14 @@ add_setup(async function () { let oldCanRecord = Services.telemetry.canRecordExtended; Services.telemetry.canRecordExtended = true; - await insertRecordIntoCollectionAndSync(); // If the categorization preference is enabled, we should also wait for the // sync event to update the domain to categories map. if (lazy.serpEventsCategorizationEnabled) { - await waitForDomainToCategoriesUpdate(); + let promise = waitForDomainToCategoriesUpdate(); + await insertRecordIntoCollectionAndSync(); + await promise; + } else { + await insertRecordIntoCollectionAndSync(); } registerCleanupFunction(async () => { @@ -99,6 +102,11 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() { // the default branch, and not overwrite the user branch. prefBranch.setBoolPref(TELEMETRY_PREF, false); + // If it was true, we should wait until the map is fully un-inited. + if (originalPrefValue) { + await waitForDomainToCategoriesUninit(); + } + Assert.equal( lazy.serpEventsCategorizationEnabled, false, @@ -152,6 +160,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -160,6 +169,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() { info("End experiment."); await doExperimentCleanup(); + await waitForDomainToCategoriesUninit(); Assert.equal( lazy.serpEventsCategorizationEnabled, @@ -179,6 +189,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() { await new Promise(resolve => setTimeout(resolve, 1500)); BrowserTestUtils.removeTab(tab); + // We should not record telemetry if the experiment is un-enrolled. assertCategorizationValues([]); // Clean up. diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js index 246caf6f47..daccbf0c93 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js @@ -71,6 +71,16 @@ add_setup(async function () { await promise; registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } SearchSERPTelemetry.overrideSearchTelemetryForTests(); resetTelemetry(); }); @@ -103,6 +113,7 @@ add_task(async function test_load_serp_and_categorize() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -143,6 +154,7 @@ add_task(async function test_load_serp_and_categorize_and_click_organic() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -181,6 +193,7 @@ add_task(async function test_load_serp_and_categorize_and_click_sponsored() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "1", num_ads_visible: "2", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js index b8dd85da97..9bd215f697 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js @@ -82,11 +82,20 @@ add_setup(async function () { await db.clear(); - // Set the state of the pref to false so that tests toggle the preference, - // triggering the map to be updated. - await SpecialPowers.pushPrefEnv({ - set: [["browser.search.serpEventTelemetryCategorization.enabled", false]], - }); + // If the pref is by default on, disable it as the following tests toggle + // the preference to check what happens when the preference is off and the + // preference is turned on. + if ( + Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + let promise = waitForDomainToCategoriesUninit(); + await SpecialPowers.pushPrefEnv({ + set: [["browser.search.serpEventTelemetryCategorization.enabled", false]], + }); + await promise; + } let defaultDownloadSettings = { ...TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS, @@ -104,6 +113,16 @@ add_setup(async function () { TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.maxAdjust = 0; registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be initialized. + await SpecialPowers.popPrefEnv(); + if ( + Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesInit(); + } SearchSERPTelemetry.overrideSearchTelemetryForTests(); resetTelemetry(); TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS = { @@ -159,6 +178,7 @@ add_task(async function test_download_after_failure() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_visible: "2", num_ads_clicked: "0", }, @@ -166,6 +186,7 @@ add_task(async function test_download_after_failure() { // Clean up. await SpecialPowers.popPrefEnv(); + await waitForDomainToCategoriesUninit(); await resetCategorizationCollection(record); }); @@ -214,6 +235,7 @@ add_task(async function test_download_after_multiple_failures() { // Clean up. await SpecialPowers.popPrefEnv(); + await waitForDomainToCategoriesUninit(); await resetCategorizationCollection(record); }); @@ -245,6 +267,7 @@ add_task(async function test_cancel_download_timer() { }); await SpecialPowers.popPrefEnv(); await observeCancel; + await waitForDomainToCategoriesUninit(); // To ensure we don't attempt another download, wait a bit over how long the // the download error should take. @@ -263,7 +286,6 @@ add_task(async function test_cancel_download_timer() { Assert.ok(SearchSERPDomainToCategoriesMap.empty, "Map is empty"); // Clean up. - await SpecialPowers.popPrefEnv(); await resetCategorizationCollection(record); }); @@ -310,6 +332,7 @@ add_task(async function test_download_adjust() { // Clean up. await SpecialPowers.popPrefEnv(); + await waitForDomainToCategoriesUninit(); await resetCategorizationCollection(record); TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.base = TIMEOUT_IN_MS; TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.minAdjust = 0; diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js index e653be6c48..2d13b147a2 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js @@ -362,19 +362,57 @@ const TESTS = [ ], expectedDomains: ["organic.com"], }, + { + title: "Bing organic result with a path in the URL.", + extractorInfos: [ + { + selectors: "#test26 #b_results .b_algo .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["organic.com"], + }, + { + title: "Bing organic result with a path and query param in the URL.", + extractorInfos: [ + { + selectors: "#test27 #b_results .b_algo .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["organic.com"], + }, + { + title: + "Bing organic result with a path in the URL, but protocol appears in separate HTML element.", + extractorInfos: [ + { + selectors: "#test28 #b_results .b_algo .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["wikipedia.org"], + }, ]; add_setup(async function () { await SpecialPowers.pushPrefEnv({ - set: [ - ["browser.search.serpEventTelemetry.enabled", true], - ["browser.search.serpEventTelemetryCategorization.enabled", true], - ], + set: [["browser.search.serpEventTelemetryCategorization.enabled", true]], }); await SearchSERPTelemetry.init(); registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } resetTelemetry(); }); }); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js new file mode 100644 index 0000000000..2375cad82a --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js @@ -0,0 +1,141 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/* + * Checks reporting of pages without ads is accurate. + */ + +ChromeUtils.defineESModuleGetters(this, { + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + nonAdsLinkRegexps: [], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + shoppingTab: { + selector: "#shopping", + }, + // The search telemetry entry responsible for targeting the specific results. + domainExtraction: { + ads: [], + nonAds: [ + { + selectors: "#results .organic a", + method: "href", + }, + ], + }, + components: [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, + default: true, + }, + ], + }, +]; + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + + let promise = waitForDomainToCategoriesUpdate(); + await insertRecordIntoCollectionAndSync(); + await SpecialPowers.pushPrefEnv({ + set: [["browser.search.serpEventTelemetryCategorization.enabled", true]], + }); + await promise; + + registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + resetTelemetry(); + }); +}); + +add_task( + async function test_load_serp_without_sponsored_links_and_categorize() { + resetTelemetry(); + + let url = getSERPUrl( + "searchTelemetryDomainCategorizationReportingWithoutAds.html" + ); + info("Load a SERP with organic and ad components that are non-sponsored."); + let promise = waitForPageWithCategorizedDomains(); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + info("Assert there is a non-sponsored component on the page."); + assertSERPTelemetry([ + { + impression: { + shopping_tab_displayed: "true", + provider: "example", + source: "unknown", + tagged: "true", + is_private: "false", + is_shopping_page: "false", + partner_code: "ff", + }, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + info("Click on the non-sponsored component."); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#shopping", + {}, + tab.linkedBrowser + ); + + await BrowserTestUtils.removeTab(tab); + info("Assert no ads were visible or clicked on."); + assertCategorizationValues([ + { + organic_category: "3", + organic_num_domains: "1", + organic_num_inconclusive: "0", + organic_num_unknown: "0", + sponsored_category: "0", + sponsored_num_domains: "0", + sponsored_num_inconclusive: "0", + sponsored_num_unknown: "0", + mappings_version: "1", + app_version: APP_MAJOR_VERSION, + channel: CHANNEL, + region: REGION, + partner_code: "ff", + provider: "example", + tagged: "true", + is_shopping_page: "false", + num_ads_clicked: "0", + num_ads_visible: "0", + }, + ]); + } +); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js new file mode 100644 index 0000000000..0196483b8c --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js @@ -0,0 +1,302 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/* + * This test ensures we are correctly submitting the custom ping for SERP + * categorization. (Please see the search component's Marionette tests for + * a test of the ping's submission upon startup.) + */ + +ChromeUtils.defineESModuleGetters(this, { + CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchSERPTelemetry: "resource:///modules/SearchSERPTelemetry.sys.mjs", + SERPCategorizationRecorder: "resource:///modules/SearchSERPTelemetry.sys.mjs", + TELEMETRY_CATEGORIZATION_KEY: + "resource:///modules/SearchSERPTelemetry.sys.mjs", +}); + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + nonAdsLinkRegexps: [/^https:\/\/example.com/], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + // The search telemetry entry responsible for targeting the specific results. + domainExtraction: { + ads: [ + { + selectors: "[data-ad-domain]", + method: "data-attribute", + options: { + dataAttributeKey: "adDomain", + }, + }, + { + selectors: ".ad", + method: "href", + options: { + queryParamKey: "ad_domain", + }, + }, + ], + nonAds: [ + { + selectors: "#results .organic a", + method: "href", + }, + ], + }, + components: [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, + default: true, + }, + ], + }, +]; + +const client = RemoteSettings(TELEMETRY_CATEGORIZATION_KEY); +const db = client.db; + +function sleep(ms) { + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + return new Promise(resolve => setTimeout(resolve, ms)); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + await db.clear(); + + let promise = waitForDomainToCategoriesUpdate(); + await insertRecordIntoCollectionAndSync(); + await SpecialPowers.pushPrefEnv({ + set: [["browser.search.serpEventTelemetryCategorization.enabled", true]], + }); + await promise; + + registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } + + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +add_task(async function test_threshold_reached() { + resetTelemetry(); + + let oldThreshold = CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD; + // For testing, it's fine to categorize fewer SERPs before sending the ping. + CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD = 2; + SERPCategorizationRecorder.uninit(); + SERPCategorizationRecorder.init(); + + Assert.equal( + null, + Glean.serp.categorization.testGetValue(), + "Should not have recorded any metrics yet." + ); + + let submitted = false; + GleanPings.serpCategorization.testBeforeNextSubmit(reason => { + submitted = true; + Assert.equal( + "threshold_reached", + reason, + "Ping submission reason should be 'threshold_reached'." + ); + }); + + // Categorize first SERP, which results in one organic and one sponsored + // reporting. + let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html"); + info("Load a sample SERP with organic and sponsored results."); + let promise = waitForPageWithCategorizedDomains(); + let tab1 = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + Assert.equal( + false, + submitted, + "Ping should not be submitted before threshold is reached." + ); + + // Categorize second SERP, which results in one organic and one sponsored + // reporting. + url = getSERPUrl("searchTelemetryDomainExtraction.html"); + info("Load a sample SERP with organic and sponsored results."); + promise = waitForPageWithCategorizedDomains(); + let tab2 = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + BrowserTestUtils.removeTab(tab1); + BrowserTestUtils.removeTab(tab2); + + Assert.equal( + true, + submitted, + "Ping should be submitted once threshold is reached." + ); + + CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD = oldThreshold; +}); + +add_task(async function test_quick_activity_to_inactivity_alternation() { + resetTelemetry(); + + Assert.equal( + null, + Glean.serp.categorization.testGetValue(), + "Should not have recorded any metrics yet." + ); + + let submitted = false; + GleanPings.serpCategorization.testBeforeNextSubmit(() => { + submitted = true; + }); + + let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html"); + info("Load a sample SERP with organic and sponsored results."); + let promise = waitForPageWithCategorizedDomains(); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + let activityDetectedPromise = TestUtils.topicObserved( + "user-interaction-active" + ); + // Simulate ~2.5 seconds of activity. + for (let i = 0; i < 25; i++) { + EventUtils.synthesizeKey("KEY_Enter"); + await sleep(100); + } + await activityDetectedPromise; + + let inactivityDetectedPromise = TestUtils.topicObserved( + "user-interaction-inactive" + ); + await inactivityDetectedPromise; + + Assert.equal( + false, + submitted, + "Ping should not be submitted after a quick alternation from activity to inactivity." + ); + + BrowserTestUtils.removeTab(tab); +}); + +add_task(async function test_submit_after_activity_then_inactivity() { + resetTelemetry(); + let oldActivityLimit = Services.prefs.getIntPref( + "telemetry.fog.test.activity_limit" + ); + Services.prefs.setIntPref("telemetry.fog.test.activity_limit", 2); + + Assert.equal( + null, + Glean.serp.categorization.testGetValue(), + "Should not have recorded any metrics yet." + ); + + let submitted = false; + GleanPings.serpCategorization.testBeforeNextSubmit(reason => { + submitted = true; + Assert.equal( + "inactivity", + reason, + "Ping submission reason should be 'inactivity'." + ); + }); + + let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html"); + info("Load a sample SERP with organic and sponsored results."); + let promise = waitForPageWithCategorizedDomains(); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + let activityDetectedPromise = TestUtils.topicObserved( + "user-interaction-active" + ); + // Simulate ~2.5 seconds of activity. + for (let i = 0; i < 25; i++) { + EventUtils.synthesizeKey("KEY_Enter"); + await sleep(100); + } + await activityDetectedPromise; + + let inactivityDetectedPromise = TestUtils.topicObserved( + "user-interaction-inactive" + ); + await inactivityDetectedPromise; + + Assert.equal( + true, + submitted, + "Ping should be submitted after 2+ seconds of activity, followed by inactivity." + ); + + BrowserTestUtils.removeTab(tab); + Services.prefs.setIntPref( + "telemetry.fog.test.activity_limit", + oldActivityLimit + ); +}); + +add_task(async function test_no_observers_added_if_pref_is_off() { + resetTelemetry(); + + let prefOnActiveObserverCount = Array.from( + Services.obs.enumerateObservers("user-interaction-active") + ).length; + let prefOnInactiveObserverCount = Array.from( + Services.obs.enumerateObservers("user-interaction-inactive") + ).length; + + await SpecialPowers.pushPrefEnv({ + set: [["browser.search.serpEventTelemetryCategorization.enabled", false]], + }); + await waitForDomainToCategoriesUninit(); + + let prefOffActiveObserverCount = Array.from( + Services.obs.enumerateObservers("user-interaction-active") + ).length; + let prefOffInactiveObserverCount = Array.from( + Services.obs.enumerateObservers("user-interaction-inactive") + ).length; + + Assert.equal( + prefOnActiveObserverCount - prefOffActiveObserverCount, + 1, + "There should be one fewer active observer when the pref is off." + ); + Assert.equal( + prefOnInactiveObserverCount - prefOffInactiveObserverCount, + 1, + "There should be one fewer inactive observer when the pref is off." + ); + + await SpecialPowers.popPrefEnv(); + await waitForDomainToCategoriesInit(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js index 4c47b0b14a..7dbf605396 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js @@ -78,6 +78,17 @@ add_setup(async function () { Assert.equal(Region.home, "DE", "Region"); registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } + Region._setHomeRegion(originalHomeRegion); Region._setCurrentRegion(originalCurrentRegion); @@ -113,6 +124,7 @@ add_task(async function test_categorize_page_with_different_region() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js index 973f17b760..3c439844d7 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js @@ -19,6 +19,7 @@ const TEST_PROVIDER_INFO = [ queryParamNames: ["s"], codeParamName: "abc", taggedCodes: ["ff"], + organicCodes: [], adServerAttributes: ["mozAttr"], nonAdsLinkRegexps: [], extraAdServersRegexps: [ @@ -56,6 +57,9 @@ const TEST_PROVIDER_INFO = [ default: true, }, ], + shoppingTab: { + regexp: "&page=shop", + }, }, ]; @@ -69,6 +73,10 @@ add_setup(async function () { SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + let { record, attachment } = await insertRecordIntoCollection(); categorizationRecord = record; categorizationAttachment = attachment; @@ -82,7 +90,18 @@ add_setup(async function () { await promise; registerCleanupFunction(async () => { + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; resetTelemetry(); await db.clear(); }); @@ -115,6 +134,7 @@ add_task(async function test_categorization_reporting() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -147,6 +167,7 @@ add_task(async function test_no_reporting_if_download_failure() { await promise; await BrowserTestUtils.removeTab(tab); + // We should not record telemetry if attachments weren't downloaded. assertCategorizationValues([]); // Re-insert the attachment for other tests. @@ -177,6 +198,7 @@ add_task(async function test_no_reporting_if_no_records() { await promise; await BrowserTestUtils.removeTab(tab); + // We should not record telemetry if there are no records. assertCategorizationValues([]); }); @@ -218,8 +240,46 @@ add_task(async function test_reporting_limited_to_10_domains_of_each_kind() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "12", }, ]); }); + +add_task(async function test_categorization_reporting_for_shopping_page() { + resetTelemetry(); + + let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html"); + let shoppingUrl = new URL(url); + shoppingUrl.searchParams.set("page", "shop"); + shoppingUrl = shoppingUrl.toString(); + info("Load a sample shopping page SERP with organic and sponsored results."); + let promise = waitForPageWithCategorizedDomains(); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, shoppingUrl); + await promise; + + await BrowserTestUtils.removeTab(tab); + assertCategorizationValues([ + { + organic_category: "3", + organic_num_domains: "1", + organic_num_inconclusive: "0", + organic_num_unknown: "0", + sponsored_category: "4", + sponsored_num_domains: "2", + sponsored_num_inconclusive: "0", + sponsored_num_unknown: "0", + mappings_version: "1", + app_version: APP_MAJOR_VERSION, + channel: CHANNEL, + region: REGION, + partner_code: "ff", + provider: "example", + tagged: "true", + is_shopping_page: "true", + num_ads_clicked: "0", + num_ads_visible: "2", + }, + ]); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js index 9d3ac2c931..0e2d1c07fd 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js @@ -87,9 +87,20 @@ add_setup(async function () { await promise; registerCleanupFunction(async () => { - // The scheduler uses the mock idle service. - SearchSERPCategorizationEventScheduler.uninit(); - SearchSERPCategorizationEventScheduler.init(); + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } else { + // The scheduler uses the mock idle service. + SearchSERPCategorizationEventScheduler.uninit(); + SearchSERPCategorizationEventScheduler.init(); + } SearchSERPTelemetry.overrideSearchTelemetryForTests(); resetTelemetry(); }); @@ -126,6 +137,7 @@ add_task(async function test_categorize_serp_and_wait() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -170,6 +182,7 @@ add_task(async function test_categorize_serp_open_multiple_tabs() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }); @@ -223,6 +236,7 @@ add_task(async function test_categorize_serp_close_tab_and_wait() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -276,6 +290,7 @@ add_task(async function test_categorize_serp_open_ad_and_wait() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "1", num_ads_visible: "2", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js index c73e224eae..43c520a8d0 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js @@ -92,9 +92,20 @@ add_setup(async function () { await promise; registerCleanupFunction(async () => { - // The scheduler uses the mock idle service. - SearchSERPCategorizationEventScheduler.uninit(); - SearchSERPCategorizationEventScheduler.init(); + // Manually unload the pref so that we can check if we should wait for the + // the categories map to be un-initialized. + await SpecialPowers.popPrefEnv(); + if ( + !Services.prefs.getBoolPref( + "browser.search.serpEventTelemetryCategorization.enabled" + ) + ) { + await waitForDomainToCategoriesUninit(); + } else { + // The scheduler uses the mock idle service. + SearchSERPCategorizationEventScheduler.uninit(); + SearchSERPCategorizationEventScheduler.init(); + } CATEGORIZATION_SETTINGS.WAKE_TIMEOUT_MS = oldWakeTimeout; SearchSERPTelemetry.overrideSearchTelemetryForTests(); resetTelemetry(); @@ -138,6 +149,7 @@ add_task(async function test_categorize_serp_and_sleep() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, @@ -195,6 +207,7 @@ add_task(async function test_categorize_serp_and_sleep_not_long_enough() { partner_code: "ff", provider: "example", tagged: "true", + is_shopping_page: "false", num_ads_clicked: "0", num_ads_visible: "2", }, diff --git a/browser/components/search/test/browser/telemetry/head.js b/browser/components/search/test/browser/telemetry/head.js index b798099bdd..ecc6e38fa9 100644 --- a/browser/components/search/test/browser/telemetry/head.js +++ b/browser/components/search/test/browser/telemetry/head.js @@ -4,11 +4,14 @@ ChromeUtils.defineESModuleGetters(this, { ADLINK_CHECK_TIMEOUT_MS: "resource:///actors/SearchSERPTelemetryChild.sys.mjs", + CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs", CustomizableUITestUtils: "resource://testing-common/CustomizableUITestUtils.sys.mjs", Region: "resource://gre/modules/Region.sys.mjs", RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", SEARCH_TELEMETRY_SHARED: "resource:///modules/SearchSERPTelemetry.sys.mjs", + SearchSERPDomainToCategoriesMap: + "resource:///modules/SearchSERPTelemetry.sys.mjs", SearchSERPTelemetry: "resource:///modules/SearchSERPTelemetry.sys.mjs", SearchSERPTelemetryUtils: "resource:///modules/SearchSERPTelemetry.sys.mjs", SearchTestUtils: "resource://testing-common/SearchTestUtils.sys.mjs", @@ -193,11 +196,10 @@ async function assertSearchSourcesTelemetry( } function resetTelemetry() { - // TODO Bug 1868476: Replace when we're using Glean telemetry. - fakeTelemetryStorage = []; searchCounts.clear(); Services.telemetry.clearScalars(); Services.fog.testResetFOG(); + SERPCategorizationRecorder.testReset(); } /** @@ -377,23 +379,6 @@ function assertSERPTelemetry(expectedEvents) { ); } -// TODO Bug 1868476: Replace when we're using Glean telemetry. -let categorizationSandbox; -let fakeTelemetryStorage = []; -add_setup(function () { - categorizationSandbox = sinon.createSandbox(); - categorizationSandbox - .stub(SERPCategorizationRecorder, "recordCategorizationTelemetry") - .callsFake(input => { - fakeTelemetryStorage.push(input); - }); - - registerCleanupFunction(() => { - categorizationSandbox.restore(); - fakeTelemetryStorage = []; - }); -}); - async function openSerpInNewTab(url, expectedAds = true) { let promise; if (expectedAds) { @@ -435,12 +420,11 @@ async function synthesizePageAction({ } function assertCategorizationValues(expectedResults) { - // TODO Bug 1868476: Replace with calls to Glean telemetry. - let actualResults = [...fakeTelemetryStorage]; + let actualResults = Glean.serp.categorization.testGetValue() ?? []; Assert.equal( - expectedResults.length, actualResults.length, + expectedResults.length, "Should have the correct number of categorization impressions." ); @@ -458,7 +442,7 @@ function assertCategorizationValues(expectedResults) { } } for (let actual of actualResults) { - for (let key in actual) { + for (let key in actual.extra) { keys.add(key); } } @@ -467,14 +451,21 @@ function assertCategorizationValues(expectedResults) { for (let index = 0; index < expectedResults.length; ++index) { info(`Checking categorization at index: ${index}`); let expected = expectedResults[index]; - let actual = actualResults[index]; + let actual = actualResults[index].extra; + + Assert.ok( + Number(actual?.organic_num_domains) <= + CATEGORIZATION_SETTINGS.MAX_DOMAINS_TO_CATEGORIZE, + "Number of organic domains categorized should not exceed threshold." + ); + + Assert.ok( + Number(actual?.sponsored_num_domains) <= + CATEGORIZATION_SETTINGS.MAX_DOMAINS_TO_CATEGORIZE, + "Number of sponsored domains categorized should not exceed threshold." + ); + for (let key of keys) { - // TODO Bug 1868476: This conversion to strings is to mimic Glean - // converting all values into strings. Once we receive real values from - // Glean, it can be removed. - if (actual[key] != null && typeof actual[key] !== "string") { - actual[key] = actual[key].toString(); - } Assert.equal( actual[key], expected[key], @@ -508,6 +499,14 @@ function waitForDomainToCategoriesUpdate() { return TestUtils.topicObserved("domain-to-categories-map-update-complete"); } +function waitForDomainToCategoriesInit() { + return TestUtils.topicObserved("domain-to-categories-map-init"); +} + +function waitForDomainToCategoriesUninit() { + return TestUtils.topicObserved("domain-to-categories-map-uninit"); +} + registerCleanupFunction(async () => { await PlacesUtils.history.clear(); }); diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html b/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html new file mode 100644 index 0000000000..13d023e45d --- /dev/null +++ b/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html @@ -0,0 +1,18 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Document</title> +</head> +<body> + <div> + <a id="shopping" href="https://www.example.org/shopping">Shopping</a> + </div> + <div id="results"> + <div class="organic"> + <a href="https://www.foobar.org">Link</a> + </div> + </div> +</body> +</html> diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html index 28c31af959..fe52bb8b48 100644 --- a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html +++ b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html @@ -256,6 +256,37 @@ </div> </div> </div> + + <div id="test26"> + <div id="b_results"> + <div class="b_algo"> + <div class="b_attribution"> + <cite>https://organic.com/cats</cite> + </div> + </div> + </div> + </div> + + <div id="test27"> + <div id="b_results"> + <div class="b_algo"> + <div class="b_attribution"> + <cite>https://organic.com/testing?q=cats</cite> + </div> + </div> + </div> + </div> + + <div id="test28"> + <div id="b_results"> + <div class="b_algo"> + <div class="b_attribution"> + <span>HTTPS</span> + <cite>en.wikipedia.org/wiki/Cat</cite> + </div> + </div> + </div> + </div> </div> </body> </html> diff --git a/browser/components/search/test/marionette/manifest.toml b/browser/components/search/test/marionette/manifest.toml index 152442bc5b..9cc88e9f84 100644 --- a/browser/components/search/test/marionette/manifest.toml +++ b/browser/components/search/test/marionette/manifest.toml @@ -1,4 +1,6 @@ [DEFAULT] run-if = ["buildapp == 'browser'"] +["include:telemetry/manifest.toml"] + ["test_engines_on_restart.py"] diff --git a/browser/components/search/test/marionette/telemetry/manifest.toml b/browser/components/search/test/marionette/telemetry/manifest.toml new file mode 100644 index 0000000000..1fe35945c9 --- /dev/null +++ b/browser/components/search/test/marionette/telemetry/manifest.toml @@ -0,0 +1,4 @@ +[DEFAULT] +run-if = ["buildapp == 'browser'"] + +["test_ping_submitted.py"] diff --git a/browser/components/search/test/marionette/telemetry/test_ping_submitted.py b/browser/components/search/test/marionette/telemetry/test_ping_submitted.py new file mode 100644 index 0000000000..cefe2d72d1 --- /dev/null +++ b/browser/components/search/test/marionette/telemetry/test_ping_submitted.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from marionette_driver import Wait +from marionette_harness.marionette_test import MarionetteTestCase + + +class TestPingSubmitted(MarionetteTestCase): + def setUp(self): + super(TestPingSubmitted, self).setUp() + + self.marionette.set_context(self.marionette.CONTEXT_CHROME) + + self.marionette.enforce_gecko_prefs( + { + "datareporting.healthreport.uploadEnabled": True, + "telemetry.fog.test.localhost_port": 3000, + "browser.search.log": True, + } + ) + # The categorization ping is submitted on startup. If anything delays + # its initialization, turning the preference on and immediately + # attaching a categorization event could result in the ping being + # submitted after the test event is reported but before the browser + # restarts. + script = """ + let [outerResolve] = arguments; + (async () => { + if (!Services.prefs.getBoolPref("browser.search.serpEventTelemetryCategorization.enabled")) { + let inited = new Promise(innerResolve => { + Services.obs.addObserver(function callback() { + Services.obs.removeObserver(callback, "categorization-recorder-init"); + innerResolve(); + }, "categorization-recorder-init"); + }); + Services.prefs.setBoolPref("browser.search.serpEventTelemetryCategorization.enabled", true); + await inited; + } + })().then(outerResolve); + """ + self.marionette.execute_async_script(script) + + def test_ping_submit_on_start(self): + # Record an event for the ping to eventually submit. + self.marionette.execute_script( + """ + Glean.serp.categorization.record({ + organic_category: "3", + organic_num_domains: "1", + organic_num_inconclusive: "0", + organic_num_unknown: "0", + sponsored_category: "4", + sponsored_num_domains: "2", + sponsored_num_inconclusive: "0", + sponsored_num_unknown: "0", + mappings_version: "1", + app_version: "124", + channel: "nightly", + region: "US", + partner_code: "ff", + provider: "example", + tagged: "true", + num_ads_clicked: "0", + num_ads_visible: "2", + }); + """ + ) + + Wait(self.marionette, timeout=60).until( + lambda _: self.marionette.execute_script( + """ + return (Glean.serp.categorization.testGetValue()?.length ?? 0) == 1; + """ + ), + message="Should have recorded a SERP categorization event before restart.", + ) + + self.marionette.restart(clean=False, in_app=True) + + Wait(self.marionette, timeout=60).until( + lambda _: self.marionette.execute_script( + """ + return (Glean.serp.categorization.testGetValue()?.length ?? 0) == 0; + """ + ), + message="SERP categorization should have been sent some time after restart.", + ) diff --git a/browser/components/search/test/unit/corruptDB.sqlite b/browser/components/search/test/unit/corruptDB.sqlite Binary files differnew file mode 100644 index 0000000000..b234246cac --- /dev/null +++ b/browser/components/search/test/unit/corruptDB.sqlite diff --git a/browser/components/search/test/unit/test_domain_to_categories_store.js b/browser/components/search/test/unit/test_domain_to_categories_store.js new file mode 100644 index 0000000000..e3af0c8de5 --- /dev/null +++ b/browser/components/search/test/unit/test_domain_to_categories_store.js @@ -0,0 +1,361 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Ensure that the domain to categories store public methods work as expected + * and it handles all error cases as expected. + */ + +ChromeUtils.defineESModuleGetters(this, { + CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs", + DomainToCategoriesStore: "resource:///modules/SearchSERPTelemetry.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", + Sqlite: "resource://gre/modules/Sqlite.sys.mjs", +}); + +let store = new DomainToCategoriesStore(); +let defaultStorePath; +let fileContents = [convertToBuffer({ foo: [0, 1] })]; + +async function createCorruptedStore() { + info("Create a corrupted store."); + let storePath = PathUtils.join( + PathUtils.profileDir, + CATEGORIZATION_SETTINGS.STORE_FILE + ); + let src = PathUtils.join(do_get_cwd().path, "corruptDB.sqlite"); + await IOUtils.copy(src, storePath); + Assert.ok(await IOUtils.exists(storePath), "Store exists."); + return storePath; +} + +function convertToBuffer(obj) { + return new TextEncoder().encode(JSON.stringify(obj)).buffer; +} + +/** + * Deletes data from the store and removes any files that were generated due + * to them. + */ +async function cleanup() { + info("Clean up store."); + + // In these tests, we sometimes use read-only files to test permission error + // handling. On Windows, we have to change it to writable to allow for their + // deletion so that subsequent tests aren't affected. + if ( + (await IOUtils.exists(defaultStorePath)) && + Services.appinfo.OS == "WINNT" + ) { + await IOUtils.setPermissions(defaultStorePath, 0o600); + } + + await store.testDelete(); + Assert.equal(store.empty, true, "Store should be empty."); + Assert.equal(await IOUtils.exists(defaultStorePath), false, "Store exists."); + Assert.equal( + await store.getVersion(), + 0, + "Version number should be 0 when store is empty." + ); + + await store.uninit(); +} + +async function createReadOnlyStore() { + info("Create a store that can't be read."); + let storePath = PathUtils.join( + PathUtils.profileDir, + CATEGORIZATION_SETTINGS.STORE_FILE + ); + + let conn = await Sqlite.openConnection({ path: storePath }); + await conn.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)"); + await conn.close(); + + await changeStoreToReadOnly(); +} + +async function changeStoreToReadOnly() { + info("Change store to read only."); + let storePath = PathUtils.join( + PathUtils.profileDir, + CATEGORIZATION_SETTINGS.STORE_FILE + ); + let stat = await IOUtils.stat(storePath); + await IOUtils.setPermissions(storePath, 0o444); + stat = await IOUtils.stat(storePath); + Assert.equal(stat.permissions, 0o444, "Permissions should be read only."); + Assert.ok(await IOUtils.exists(storePath), "Store exists."); +} + +add_setup(async function () { + // We need a profile directory to create the store and open a connection. + do_get_profile(); + defaultStorePath = PathUtils.join( + PathUtils.profileDir, + CATEGORIZATION_SETTINGS.STORE_FILE + ); + registerCleanupFunction(async () => { + await cleanup(); + }); +}); + +// Ensure the test only function deletes the store. +add_task(async function delete_store() { + let storePath = await createCorruptedStore(); + await store.testDelete(); + Assert.ok(!(await IOUtils.exists(storePath)), "Store doesn't exist."); +}); + +/** + * These tests check common no fail scenarios. + */ + +add_task(async function init_insert_uninit() { + await store.init(); + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal( + await store.getVersion(), + 0, + "Version number should not be set." + ); + Assert.equal(store.empty, true, "Store should be empty."); + + info("Try inserting after init."); + await store.insertFileContents(fileContents, 1); + + result = await store.getCategories("foo"); + Assert.deepEqual(result, [0, 1], "foo should have a matching result."); + Assert.equal(await store.getVersion(), 1, "Version number should be set."); + Assert.equal(store.empty, false, "Store should not be empty."); + + info("Un-init store."); + await store.uninit(); + + result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should be removed from store."); + Assert.equal(store.empty, true, "Store should be empty."); + Assert.equal(await store.getVersion(), 0, "Version should be reset."); + + await cleanup(); +}); + +add_task(async function insert_and_re_init() { + await store.init(); + await store.insertFileContents(fileContents, 20240202); + + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [0, 1], "foo should have a matching result."); + Assert.equal( + await store.getVersion(), + 20240202, + "Version number should be set." + ); + Assert.equal(store.empty, false, "Is store empty."); + + info("Simulate a restart."); + await store.uninit(); + await store.init(); + + result = await store.getCategories("foo"); + Assert.deepEqual( + result, + [0, 1], + "After restart, foo should still be in the store." + ); + Assert.equal( + await store.getVersion(), + 20240202, + "Version number should still be in the store." + ); + Assert.equal(store.empty, false, "Is store empty."); + + await cleanup(); +}); + +// Simulate consecutive updates. +add_task(async function insert_multiple_times() { + await store.init(); + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal( + await store.getVersion(), + 0, + "Version number should not be set." + ); + Assert.equal(store.empty, true, "Is store empty."); + + for (let i = 0; i < 3; ++i) { + info("Try inserting after init."); + await store.insertFileContents(fileContents, 1); + + result = await store.getCategories("foo"); + Assert.deepEqual(result, [0, 1], "foo should have a matching result."); + Assert.equal(store.empty, false, "Is store empty."); + Assert.equal(await store.getVersion(), 1, "Version number is set."); + + await store.dropData(); + result = await store.getCategories("foo"); + Assert.deepEqual( + result, + [], + "After dropping data, foo should no longer have a matching result." + ); + Assert.equal(await store.getVersion(), 0, "Version should be reset."); + Assert.equal(store.empty, true, "Is store empty."); + } + + await cleanup(); +}); + +/** + * The following tests check failures on store initialization. + */ + +add_task(async function init_with_corrupted_store() { + await createCorruptedStore(); + + info("Initialize the store."); + await store.init(); + + info("Try inserting after the corrupted store was replaced."); + await store.insertFileContents(fileContents, 1); + + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [0, 1], "foo should have a matching result."); + Assert.equal(await store.getVersion(), 1, "Version number is set."); + Assert.equal(store.empty, false, "Is store empty."); + + await cleanup(); +}); + +add_task(async function init_with_unfixable_store() { + let sandbox = sinon.createSandbox(); + sandbox.stub(Sqlite, "openConnection").throws(); + + info("Initialize the store."); + await store.init(); + + info("Try inserting content even if the connection is impossible to fix."); + await store.dropData(); + await store.insertFileContents(fileContents, 20240202); + + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal(await store.getVersion(), 0, "Version should be reset."); + Assert.equal(store.empty, true, "Store should be empty."); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function init_read_only_store() { + await createReadOnlyStore(); + await store.init(); + + info("Insert contents into the store."); + await store.insertFileContents(fileContents, 20240202); + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal( + await store.getVersion(), + 0, + "Version number should not be set." + ); + Assert.equal(store.empty, true, "Store should be empty."); + + await cleanup(); +}); + +add_task(async function init_close_to_shutdown() { + let sandbox = sinon.createSandbox(); + sandbox.stub(Sqlite.shutdown, "addBlocker").throws(new Error()); + await store.init(); + + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal( + await store.getVersion(), + 0, + "Version number should not be set." + ); + Assert.equal(store.empty, true, "Store should be empty."); + + sandbox.restore(); + await cleanup(); +}); + +/** + * The following tests check error handling when inserting data into the store. + */ + +add_task(async function insert_broken_file() { + await store.init(); + + Assert.equal( + await store.getVersion(), + 0, + "Version number should not be set." + ); + + info("Try inserting one valid file and an invalid file."); + let contents = [...fileContents, new ArrayBuffer(0).buffer]; + await store.insertFileContents(contents, 20240202); + + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal(await store.getVersion(), 0, "Version should remain unset."); + Assert.equal(store.empty, true, "Store should remain empty."); + + await cleanup(); +}); + +add_task(async function insert_into_read_only_store() { + await createReadOnlyStore(); + await store.init(); + + await store.dropData(); + await store.insertFileContents(fileContents, 20240202); + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [], "foo should not have a result."); + Assert.equal(await store.getVersion(), 0, "Version should remain unset."); + Assert.equal(store.empty, true, "Store should remain empty."); + + await cleanup(); +}); + +// If the store becomes read only with content already inside of it, +// the next time we try opening it, we'll encounter an error trying to write to +// it. Since we are no longer able to manipulate it, the results should always +// be empty. +add_task(async function restart_with_read_only_store() { + await store.init(); + await store.insertFileContents(fileContents, 20240202); + + info("Check store has content."); + let result = await store.getCategories("foo"); + Assert.deepEqual(result, [0, 1], "foo should have a matching result."); + Assert.equal( + await store.getVersion(), + 20240202, + "Version number should be set." + ); + Assert.equal(store.empty, false, "Store should not be empty."); + + await changeStoreToReadOnly(); + await store.uninit(); + await store.init(); + + result = await store.getCategories("foo"); + Assert.deepEqual( + result, + [], + "foo should no longer have a matching value from the store." + ); + Assert.equal(await store.getVersion(), 0, "Version number should be unset."); + Assert.equal(store.empty, true, "Store should be empty."); + + await cleanup(); +}); diff --git a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js index 40d38efbba..2351347d77 100644 --- a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js +++ b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js @@ -9,6 +9,7 @@ ChromeUtils.defineESModuleGetters(this, { RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchSERPCategorization: "resource:///modules/SearchSERPTelemetry.sys.mjs", SearchSERPDomainToCategoriesMap: "resource:///modules/SearchSERPTelemetry.sys.mjs", TELEMETRY_CATEGORIZATION_KEY: @@ -158,7 +159,7 @@ add_task(async function test_initial_import() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); add_task(async function test_update_records() { @@ -219,7 +220,7 @@ add_task(async function test_update_records() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); add_task(async function test_delayed_initial_import() { @@ -273,7 +274,7 @@ add_task(async function test_delayed_initial_import() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); add_task(async function test_remove_record() { @@ -332,7 +333,7 @@ add_task(async function test_remove_record() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); add_task(async function test_different_versions_coexisting() { @@ -380,7 +381,7 @@ add_task(async function test_different_versions_coexisting() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); add_task(async function test_download_error() { @@ -449,5 +450,67 @@ add_task(async function test_download_error() { // Clean up. await db.clear(); - SearchSERPDomainToCategoriesMap.uninit(); + await SearchSERPDomainToCategoriesMap.uninit(true); +}); + +add_task(async function test_mock_restart() { + info("Create record containing domain_category_mappings_2a.json attachment."); + let record2a = await mockRecordWithCachedAttachment(RECORDS.record2a); + await db.create(record2a); + + info("Create record containing domain_category_mappings_2b.json attachment."); + let record2b = await mockRecordWithCachedAttachment(RECORDS.record2b); + await db.create(record2b); + + info("Add data to Remote Settings DB."); + await db.importChanges({}, Date.now()); + + info("Initialize search categorization mappings."); + let promise = waitForDomainToCategoriesUpdate(); + await SearchSERPCategorization.init(); + await promise; + + Assert.deepEqual( + await SearchSERPDomainToCategoriesMap.get("example.com"), + [ + { + category: 1, + score: 80, + }, + ], + "Should have a record." + ); + + Assert.equal( + SearchSERPDomainToCategoriesMap.version, + 2, + "Version should be the latest." + ); + + info("Mock a restart by un-initializing the map."); + await SearchSERPCategorization.uninit(); + promise = waitForDomainToCategoriesUpdate(); + await SearchSERPCategorization.init(); + await promise; + + Assert.deepEqual( + await SearchSERPDomainToCategoriesMap.get("example.com"), + [ + { + category: 1, + score: 80, + }, + ], + "Should have a record." + ); + + Assert.equal( + SearchSERPDomainToCategoriesMap.version, + 2, + "Version should be the latest." + ); + + // Clean up. + await db.clear(); + await SearchSERPDomainToCategoriesMap.uninit(true); }); diff --git a/browser/components/search/test/unit/test_search_telemetry_config_validation.js b/browser/components/search/test/unit/test_search_telemetry_config_validation.js index 8897b1e7c7..d14f7a3918 100644 --- a/browser/components/search/test/unit/test_search_telemetry_config_validation.js +++ b/browser/components/search/test/unit/test_search_telemetry_config_validation.js @@ -57,7 +57,7 @@ function disallowAdditionalProperties(section) { add_task(async function test_search_telemetry_validates_to_schema() { let schema = await IOUtils.readJSON( - PathUtils.join(do_get_cwd().path, "search-telemetry-schema.json") + PathUtils.join(do_get_cwd().path, "search-telemetry-v2-schema.json") ); disallowAdditionalProperties(schema); diff --git a/browser/components/search/test/unit/test_ui_schemas_valid.js b/browser/components/search/test/unit/test_ui_schemas_valid.js new file mode 100644 index 0000000000..3396f38238 --- /dev/null +++ b/browser/components/search/test/unit/test_ui_schemas_valid.js @@ -0,0 +1,31 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +let schemas = [ + ["search-telemetry-v2-schema.json", "search-telemetry-v2-ui-schema.json"], +]; + +async function checkUISchemaValid(configSchema, uiSchema) { + for (let key of Object.keys(configSchema.properties)) { + Assert.ok( + uiSchema["ui:order"].includes(key), + `Should have ${key} listed at the top-level of the ui schema` + ); + } +} + +add_task(async function test_ui_schemas_valid() { + for (let [schema, uiSchema] of schemas) { + info(`Validating ${uiSchema} has every top-level from ${schema}`); + let schemaData = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, schema) + ); + let uiSchemaData = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, uiSchema) + ); + + await checkUISchemaValid(schemaData, uiSchemaData); + } +}); diff --git a/browser/components/search/test/unit/xpcshell.toml b/browser/components/search/test/unit/xpcshell.toml index 423d218d19..24e1d78eb5 100644 --- a/browser/components/search/test/unit/xpcshell.toml +++ b/browser/components/search/test/unit/xpcshell.toml @@ -6,6 +6,9 @@ prefs = ["browser.search.log=true"] skip-if = ["os == 'android'"] # bug 1730213 firefox-appdir = "browser" +["test_domain_to_categories_store.js"] +support-files = ["corruptDB.sqlite"] + ["test_search_telemetry_categorization_logic.js"] ["test_search_telemetry_categorization_sync.js"] @@ -14,7 +17,13 @@ prefs = ["browser.search.serpEventTelemetryCategorization.enabled=true"] ["test_search_telemetry_compare_urls.js"] ["test_search_telemetry_config_validation.js"] -support-files = ["../../schema/search-telemetry-schema.json"] +support-files = ["../../schema/search-telemetry-v2-schema.json"] + +["test_ui_schemas_valid.js"] +support-files = [ + "../../schema/search-telemetry-v2-schema.json", + "../../schema/search-telemetry-v2-ui-schema.json", +] ["test_urlTelemetry.js"] |