/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- * vim: sw=4 ts=4 sts=4 et * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ // This file tests several things. // // 1. We verify that we can forcefully opt-in to (particular branches of) // experiments, that the resulting Firefox Messaging Experiment applies, and // that the Firefox Messaging System respects lifetime frequency caps. // 2. We verify that Nimbus randomization works with specific Normandy // randomization IDs. // 3. We verify that relevant opt-out prefs disable the Nimbus and Firefox // Messaging System experience. const { ASRouterTargeting } = ChromeUtils.importESModule( "resource:///modules/asrouter/ASRouterTargeting.sys.mjs" ); const { ExperimentFakes } = ChromeUtils.importESModule( "resource://testing-common/NimbusTestUtils.sys.mjs" ); // These randomization IDs were extracted by hand from Firefox instances. // Randomization is sufficiently stable to hard-code these IDs rather than // generating new ones at test time. const BRANCH_MAP = { "treatment-a": { randomizationId: "d0e95fc3-fb15-4bc4-8151-a89582a56e29", title: "Treatment A", text: "Body A", }, "treatment-b": { randomizationId: "90a60347-66cc-4716-9fef-cf49dd992d51", title: "Treatment B", text: "Body B", }, }; setupProfileService(); let taskProfile; let manager; // Arrange a dummy Remote Settings server so that no non-local network // connections are opened. // And arrange dummy task profile. add_setup(async () => { info("Setting up profile service"); let profileService = Cc["@mozilla.org/toolkit/profile-service;1"].getService( Ci.nsIToolkitProfileService ); let taskProfD = do_get_profile(); taskProfD.append("test_backgroundtask_experiments_task"); taskProfile = profileService.createUniqueProfile( taskProfD, "test_backgroundtask_experiments_task" ); registerCleanupFunction(() => { taskProfile.remove(true); }); // Arrange fake experiment enrollment details. manager = ExperimentFakes.manager(); await manager.onStartup(); await manager.store.addEnrollment(ExperimentFakes.experiment("foo")); manager.unenroll("foo", "some-reason"); await manager.store.addEnrollment( ExperimentFakes.experiment("bar", { active: false }) ); await manager.store.addEnrollment( ExperimentFakes.experiment("baz", { active: true }) ); manager.store.addEnrollment(ExperimentFakes.rollout("rol1")); manager.unenroll("rol1", "some-reason"); manager.store.addEnrollment(ExperimentFakes.rollout("rol2")); }); function resetProfile(profile) { profile.rootDir.remove(true); profile.rootDir.create(Ci.nsIFile.DIRECTORY_TYPE, 0o700); info(`Reset profile '${profile.rootDir.path}'`); } // Run the "message" background task with some default configuration. Return // "info" items output from the task as an array and as a map. async function doMessage({ extraArgs = [], extraEnv = {} } = {}) { let sentinel = Services.uuid.generateUUID().toString(); sentinel = sentinel.substring(1, sentinel.length - 1); let infoArray = []; let exitCode = await do_backgroundtask("message", { extraArgs: [ "--sentinel", sentinel, // Use a test-specific non-ephemeral profile, not the system-wide shared // task-specific profile. "--profile", taskProfile.rootDir.path, // Don't actually show a toast notification. "--disable-alerts-service", // Don't contact Remote Settings server. Can be overridden by subsequent // `--experiments ...`. "--no-experiments", ...extraArgs, ], extraEnv: { MOZ_LOG: "Dump:5,BackgroundTasks:5", ...extraEnv, }, onStdoutLine: line => { if (line.includes(sentinel)) { let info = JSON.parse(line.split(sentinel)[1]); infoArray.push(info); } }, }); Assert.equal( 0, exitCode, "The message background task exited with exit code 0" ); // Turn [{x:...}, {y:...}] into {x:..., y:...}. let infoMap = Object.assign({}, ...infoArray); return { infoArray, infoMap }; } // Opt-in to an experiment. Verify that the experiment state is non-ephemeral, // i.e., persisted. Verify that messages are shown until we hit the lifetime // frequency caps. // // It's awkward to inspect the `ASRouter.jsm` internal state directly in this // manner, but this is the pattern for testing such things at the time of // writing. add_task(async function test_backgroundtask_caps() { let experimentFile = do_get_file("experiment.json"); let experimentFileURI = Services.io.newFileURI(experimentFile); let { infoMap } = await doMessage({ extraArgs: [ // Opt-in to an experiment from a file. "--url", `${experimentFileURI.spec}?optin_branch=treatment-a`, ], }); // Verify that the correct experiment and branch generated an impression. let impressions = infoMap.ASRouterState.messageImpressions; Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); Assert.equal(impressions["test-experiment:treatment-a"].length, 1); // Verify that the correct toast notification was shown. let alert = infoMap.showAlert.args[0]; Assert.equal(alert.title, "Treatment A"); Assert.equal(alert.text, "Body A"); Assert.equal(alert.name, "optin-test-experiment:treatment-a"); // Now, do it again. No need to opt-in to the experiment this time. ({ infoMap } = await doMessage({})); // Verify that only the correct experiment and branch generated an impression. impressions = infoMap.ASRouterState.messageImpressions; Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); Assert.equal(impressions["test-experiment:treatment-a"].length, 2); // Verify that the correct toast notification was shown. alert = infoMap.showAlert.args[0]; Assert.equal(alert.title, "Treatment A"); Assert.equal(alert.text, "Body A"); Assert.equal(alert.name, "optin-test-experiment:treatment-a"); // A third time. We'll hit the lifetime frequency cap (which is 2). ({ infoMap } = await doMessage({})); // Verify that the correct experiment and branch impressions are untouched. impressions = infoMap.ASRouterState.messageImpressions; Assert.deepEqual(Object.keys(impressions), ["test-experiment:treatment-a"]); Assert.equal(impressions["test-experiment:treatment-a"].length, 2); // Verify that no toast notication was shown. Assert.ok(!("showAlert" in infoMap), "No alert shown"); }); // Test that background tasks are enrolled into branches based on the Normandy // randomization ID as expected. Run the message task with a hard-coded list of // a single experiment and known randomization IDs, and verify that the enrolled // branches are as expected. add_task(async function test_backgroundtask_randomization() { let experimentFile = do_get_file("experiment.json"); for (let [branchSlug, branchDetails] of Object.entries(BRANCH_MAP)) { // Start fresh each time. resetProfile(taskProfile); // Invoke twice; verify the branch is consistent each time. for (let count = 1; count <= 2; count++) { let { infoMap } = await doMessage({ extraArgs: [ // Read experiments from a file. "--experiments", experimentFile.path, // Fixed randomization ID yields a deterministic enrollment branch assignment. "--randomizationId", branchDetails.randomizationId, ], }); // Verify that only the correct experiment and branch generated an impression. let impressions = infoMap.ASRouterState.messageImpressions; Assert.deepEqual(Object.keys(impressions), [ `test-experiment:${branchSlug}`, ]); Assert.equal(impressions[`test-experiment:${branchSlug}`].length, count); // Verify that the correct toast notification was shown. let alert = infoMap.showAlert.args[0]; Assert.equal(alert.title, branchDetails.title, "Title is correct"); Assert.equal(alert.text, branchDetails.text, "Text is correct"); Assert.equal( alert.name, `test-experiment:${branchSlug}`, "Name (tag) is correct" ); } } }); // Test that background tasks respect the datareporting and studies opt-out // preferences. add_task(async function test_backgroundtask_optout_preferences() { let experimentFile = do_get_file("experiment.json"); let OPTION_MAP = { "--no-datareporting": { "datareporting.healthreport.uploadEnabled": false, "app.shield.optoutstudies.enabled": true, }, "--no-optoutstudies": { "datareporting.healthreport.uploadEnabled": true, "app.shield.optoutstudies.enabled": false, }, }; for (let [option, expectedPrefs] of Object.entries(OPTION_MAP)) { // Start fresh each time. resetProfile(taskProfile); let { infoMap } = await doMessage({ extraArgs: [ option, // Read experiments from a file. Opting in to an experiment with // `--url` does not consult relevant preferences. "--experiments", experimentFile.path, ], }); Assert.deepEqual(infoMap.taskProfilePrefs, expectedPrefs); // Verify that no experiment generated an impression. let impressions = infoMap.ASRouterState.messageImpressions; Assert.deepEqual( impressions, [], `No impressions generated with ${option}` ); // Verify that no toast notication was shown. Assert.ok(!("showAlert" in infoMap), `No alert shown with ${option}`); } }); const TARGETING_LIST = [ // Target based on background task details. ["isBackgroundTaskMode", 1], ["backgroundTaskName == 'message'", 1], ["backgroundTaskName == 'unrecognized'", 0], // Filter based on `defaultProfile` targeting snapshot. ["(currentDate|date - defaultProfile.currentDate|date) > 0", 1], ["(currentDate|date - defaultProfile.currentDate|date) > 999999", 0], // Filter based on `defaultProfile` experiment enrollment details. ["'baz' in defaultProfile.activeExperiments", 1], ["'bar' in defaultProfile.previousExperiments", 1], ["'rol2' in defaultProfile.activeRollouts", 1], ["'rol1' in defaultProfile.previousRollouts", 1], ["defaultProfile.enrollmentsMap['baz'] == 'treatment'", 1], ["defaultProfile.enrollmentsMap['bar'] == 'treatment'", 1], ["'unknown' in defaultProfile.enrollmentsMap", 0], ]; // Test that background tasks targeting works for Nimbus experiments. add_task(async function test_backgroundtask_Nimbus_targeting() { let experimentFile = do_get_file("experiment.json"); let experimentData = await IOUtils.readJSON(experimentFile.path); // We can't take a full environment snapshot under `xpcshell`. Select a few // items that do work. let target = { currentDate: ASRouterTargeting.Environment.currentDate, firefoxVersion: ASRouterTargeting.Environment.firefoxVersion, }; let targetSnapshot = await ASRouterTargeting.getEnvironmentSnapshot({ targets: [manager.createTargetingContext(), target], }); for (let [targeting, expectedLength] of TARGETING_LIST) { // Start fresh each time. resetProfile(taskProfile); let snapshotFile = taskProfile.rootDir.clone(); snapshotFile.append("targeting.snapshot.json"); await IOUtils.writeJSON(snapshotFile.path, targetSnapshot); // Write updated experiment data. experimentData.data.targeting = targeting; let targetingExperimentFile = taskProfile.rootDir.clone(); targetingExperimentFile.append("targeting.experiment.json"); await IOUtils.writeJSON(targetingExperimentFile.path, experimentData); let { infoMap } = await doMessage({ extraArgs: [ "--experiments", targetingExperimentFile.path, "--targeting-snapshot", snapshotFile.path, ], }); // Verify that the given targeting generated the expected number of impressions. let impressions = infoMap.ASRouterState.messageImpressions; Assert.equal( Object.keys(impressions).length, expectedLength, `${expectedLength} impressions generated with targeting '${targeting}'` ); } }); // Test that background tasks targeting works for Firefox Messaging System branches. add_task(async function test_backgroundtask_Messaging_targeting() { // Don't target the Nimbus experiment at all. Use a consistent // randomization ID to always enroll in the first branch. Target // the first branch of the Firefox Messaging Experiment to the given // targeting. Therefore, we either get the first branch if the // targeting matches, or nothing at all. let treatmentARandomizationId = BRANCH_MAP["treatment-a"].randomizationId; let experimentFile = do_get_file("experiment.json"); let experimentData = await IOUtils.readJSON(experimentFile.path); // We can't take a full environment snapshot under `xpcshell`. Select a few // items that do work. let target = { currentDate: ASRouterTargeting.Environment.currentDate, firefoxVersion: ASRouterTargeting.Environment.firefoxVersion, }; let targetSnapshot = await ASRouterTargeting.getEnvironmentSnapshot({ targets: [manager.createTargetingContext(), target], }); for (let [targeting, expectedLength] of TARGETING_LIST) { // Start fresh each time. resetProfile(taskProfile); let snapshotFile = taskProfile.rootDir.clone(); snapshotFile.append("targeting.snapshot.json"); await IOUtils.writeJSON(snapshotFile.path, targetSnapshot); // Write updated experiment data. experimentData.data.targeting = "true"; experimentData.data.branches[0].features[0].value.targeting = targeting; let targetingExperimentFile = taskProfile.rootDir.clone(); targetingExperimentFile.append("targeting.experiment.json"); await IOUtils.writeJSON(targetingExperimentFile.path, experimentData); let { infoMap } = await doMessage({ extraArgs: [ "--experiments", targetingExperimentFile.path, "--targeting-snapshot", snapshotFile.path, "--randomizationId", treatmentARandomizationId, ], }); // Verify that the given targeting generated the expected number of impressions. let impressions = infoMap.ASRouterState.messageImpressions; Assert.equal( Object.keys(impressions).length, expectedLength, `${expectedLength} impressions generated with targeting '${targeting}'` ); if (expectedLength > 0) { // Verify that the correct toast notification was shown. let alert = infoMap.showAlert.args[0]; Assert.equal( alert.title, BRANCH_MAP["treatment-a"].title, "Title is correct" ); Assert.equal( alert.text, BRANCH_MAP["treatment-a"].text, "Text is correct" ); Assert.equal( alert.name, `test-experiment:treatment-a`, "Name (tag) is correct" ); } } }); // Verify that `RemoteSettingsClient.sync` is invoked before any // `RemoteSettingsClient.get` invocations. This ensures the Remote Settings // recipe collection is not allowed to go stale. add_task( async function test_backgroundtask_RemoteSettingsClient_invokes_sync() { let { infoArray, infoMap } = await doMessage({}); Assert.ok( "RemoteSettingsClient.get" in infoMap, "RemoteSettingsClient.get was invoked" ); for (let info of infoArray) { if ("RemoteSettingsClient.get" in info) { const { options: calledOptions } = info["RemoteSettingsClient.get"]; Assert.ok( calledOptions.forceSync, "RemoteSettingsClient.get was first called with `forceSync`" ); return; } } } );