diff options
Diffstat (limited to '')
35 files changed, 11850 insertions, 0 deletions
diff --git a/toolkit/components/nimbus/test/NimbusTestUtils.jsm b/toolkit/components/nimbus/test/NimbusTestUtils.jsm new file mode 100644 index 0000000000..afa487a9a4 --- /dev/null +++ b/toolkit/components/nimbus/test/NimbusTestUtils.jsm @@ -0,0 +1,469 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const { ExperimentStore } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentStore.jsm" +); + +const { FileTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/FileTestUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + _ExperimentManager: "resource://nimbus/lib/ExperimentManager.jsm", + ExperimentManager: "resource://nimbus/lib/ExperimentManager.jsm", + NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm", + NormandyUtils: "resource://normandy/lib/NormandyUtils.jsm", + _RemoteSettingsExperimentLoader: + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm", + sinon: "resource://testing-common/Sinon.jsm", + FeatureManifest: "resource://nimbus/FeatureManifest.js", +}); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; + +const PATH = FileTestUtils.getTempFile("shared-data-map").path; + +async function fetchSchema(url) { + const response = await fetch(url); + const schema = await response.json(); + if (!schema) { + throw new Error(`Failed to load ${url}`); + } + return schema; +} + +const EXPORTED_SYMBOLS = ["ExperimentTestUtils", "ExperimentFakes"]; + +const ExperimentTestUtils = { + _validateSchema(schema, value, errorMsg) { + const result = lazy.JsonSchema.validate(value, schema, { + shortCircuit: false, + }); + if (result.errors.length) { + throw new Error( + `${errorMsg}: ${JSON.stringify(result.errors, undefined, 2)}` + ); + } + return value; + }, + + _validateFeatureValueEnum({ branch }) { + let { features } = branch; + for (let feature of features) { + // If we're not using a real feature skip this check + if (!lazy.FeatureManifest[feature.featureId]) { + return true; + } + let { variables } = lazy.FeatureManifest[feature.featureId]; + for (let varName of Object.keys(variables)) { + let varValue = feature.value[varName]; + if ( + varValue && + variables[varName].enum && + !variables[varName].enum.includes(varValue) + ) { + throw new Error( + `${varName} should have one of the following values: ${JSON.stringify( + variables[varName].enum + )} but has value '${varValue}'` + ); + } + } + } + return true; + }, + + /** + * Checks if an experiment is valid acording to existing schema + */ + async validateExperiment(experiment) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusExperiment.schema.json" + ); + + // Ensure that the `featureIds` field is properly set + const { branches } = experiment; + branches.forEach(branch => { + branch.features.map(({ featureId }) => { + if (!experiment.featureIds.includes(featureId)) { + throw new Error( + `Branch(${branch.slug}) contains feature(${featureId}) but that's not declared in recipe(${experiment.slug}).featureIds` + ); + } + }); + }); + + return this._validateSchema( + schema, + experiment, + `Experiment ${experiment.slug} not valid` + ); + }, + async validateEnrollment(enrollment) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); + + // We still have single feature experiment recipes for backwards + // compatibility testing but we don't do schema validation + if (!enrollment.branch.features && enrollment.branch.feature) { + return true; + } + + return ( + this._validateFeatureValueEnum(enrollment) && + this._validateSchema( + schema, + enrollment, + `Enrollment ${enrollment.slug} is not valid` + ) + ); + }, + async validateRollouts(rollout) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); + + return this._validateSchema( + schema, + rollout, + `Rollout configuration ${rollout.slug} is not valid` + ); + }, + /** + * Add features for tests. + * + * These features will only be visible to the JS Nimbus client. The native + * Nimbus client will have no access. + * + * @params features A list of |_NimbusFeature|s. + * + * @returns A cleanup function to remove the features once the test has completed. + */ + addTestFeatures(...features) { + for (const feature of features) { + if (Object.hasOwn(lazy.NimbusFeatures, feature.featureId)) { + throw new Error( + `Cannot add feature ${feature.featureId} -- a feature with this ID already exists!` + ); + } + lazy.NimbusFeatures[feature.featureId] = feature; + } + return () => { + for (const { featureId } of features) { + delete lazy.NimbusFeatures[featureId]; + } + }; + }, +}; + +const ExperimentFakes = { + manager(store) { + let sandbox = lazy.sinon.createSandbox(); + let manager = new lazy._ExperimentManager({ store: store || this.store() }); + // We want calls to `store.addEnrollment` to implicitly validate the + // enrollment before saving to store + let origAddExperiment = manager.store.addEnrollment.bind(manager.store); + sandbox.stub(manager.store, "addEnrollment").callsFake(async enrollment => { + await ExperimentTestUtils.validateEnrollment(enrollment); + return origAddExperiment(enrollment); + }); + + return manager; + }, + store() { + return new ExperimentStore("FakeStore", { + path: PATH, + isParent: true, + }); + }, + waitForExperimentUpdate(ExperimentAPI, options) { + if (!options) { + throw new Error("Must specify an expected recipe update"); + } + + return new Promise(resolve => ExperimentAPI.on("update", options, resolve)); + }, + async enrollWithRollout( + featureConfig, + { manager = lazy.ExperimentManager, source } = {} + ) { + await manager.store.init(); + const rollout = this.rollout(`${featureConfig.featureId}-rollout`, { + branch: { + slug: `${featureConfig.featureId}-rollout-branch`, + features: [featureConfig], + }, + }); + if (source) { + rollout.source = source; + } + await ExperimentTestUtils.validateRollouts(rollout); + // After storing the remote configuration to store and updating the feature + // we want to flush so that NimbusFeature usage in content process also + // receives the update + await manager.store.addEnrollment(rollout); + manager.store._syncToChildren({ flush: true }); + + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, enrollment) => { + if (enrollment.slug === rollout.slug && !enrollment.active) { + manager.store._deleteForTests(rollout.slug); + resolve(); + } + }) + ); + + return () => { + let promise = unenrollCompleted(rollout.slug); + manager.unenroll(rollout.slug, "cleanup"); + return promise; + }; + }, + async enrollWithFeatureConfig( + featureConfig, + { manager = lazy.ExperimentManager, isRollout = false } = {} + ) { + await manager.store.ready(); + // Use id passed in featureConfig value to compute experimentId + // This help filter telemetry events (such as expose) in race conditions when telemetry + // from multiple experiments with same featureId co-exist in snapshot + let experimentId = `${featureConfig.featureId}${ + featureConfig?.value?.id ? "-" + featureConfig?.value?.id : "" + }-experiment-${Math.random()}`; + + let recipe = this.recipe(experimentId, { + bucketConfig: { + namespace: "mstest-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + branches: [ + { + slug: "control", + ratio: 1, + features: [featureConfig], + }, + ], + isRollout, + }); + let { + enrollmentPromise, + doExperimentCleanup, + } = this.enrollmentHelper(recipe, { manager }); + + await enrollmentPromise; + + return doExperimentCleanup; + }, + enrollmentHelper(recipe, { manager = lazy.ExperimentManager } = {}) { + if (!recipe?.slug) { + throw new Error("Enrollment helper expects a recipe"); + } + + let enrollmentPromise = new Promise(resolve => + manager.store.on(`update:${recipe.slug}`, (event, experiment) => { + if (experiment.active) { + manager.store._syncToChildren({ flush: true }); + resolve(experiment); + } + }) + ); + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(recipe.slug); + resolve(); + } + }) + ); + let doExperimentCleanup = async () => { + const experiment = manager.store.get(recipe.slug); + let promise = unenrollCompleted(experiment.slug); + manager.unenroll(experiment.slug, "cleanup"); + await promise; + }; + + if (!manager.store._isReady) { + throw new Error("Manager store not ready, call `manager.onStartup`"); + } + manager.enroll(recipe, "enrollmentHelper"); + + return { enrollmentPromise, doExperimentCleanup }; + }, + async cleanupAll(slugs, { manager = lazy.ExperimentManager } = {}) { + function unenrollCompleted(slug) { + return new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(slug); + resolve(); + } + }) + ); + } + + for (const slug of slugs) { + let promise = unenrollCompleted(slug); + manager.unenroll(slug, "cleanup"); + await promise; + } + + if (manager.store.getAllActive().length) { + throw new Error("Cleanup failed"); + } + }, + // Experiment store caches in prefs Enrollments for fast sync access + cleanupStorePrefCache() { + try { + Services.prefs.deleteBranch(SYNC_DATA_PREF_BRANCH); + Services.prefs.deleteBranch(SYNC_DEFAULTS_PREF_BRANCH); + } catch (e) { + // Expected if nothing is cached + } + }, + childStore() { + return new ExperimentStore("FakeStore", { isParent: false }); + }, + rsLoader() { + const loader = new lazy._RemoteSettingsExperimentLoader(); + // Replace RS client with a fake + Object.defineProperty(loader, "remoteSettingsClient", { + value: { get: () => Promise.resolve([]) }, + }); + // Replace xman with a fake + loader.manager = this.manager(); + + return loader; + }, + experiment(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "NimbusTestUtils", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: props?.branch?.features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, + rollout(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + isRollout: true, + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "rollout", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: (props?.branch?.features || props?.features)?.map( + f => f.featureId + ) || ["testFeature"], + ...props, + }; + }, + recipe(slug = lazy.NormandyUtils.generateUuid(), props = {}) { + return { + // This field is required for populating remote settings + id: lazy.NormandyUtils.generateUuid(), + schemaVersion: "1.7.0", + appName: "firefox_desktop", + appId: "firefox-desktop", + channel: "nightly", + slug, + isEnrollmentPaused: false, + probeSets: [], + startDate: null, + endDate: null, + proposedEnrollment: 7, + referenceBranch: "control", + application: "firefox-desktop", + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + ], + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 100, + total: 1000, + }, + userFacingName: "Nimbus recipe", + userFacingDescription: "NimbusTestUtils recipe", + featureIds: props?.branches?.[0].features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, +}; diff --git a/toolkit/components/nimbus/test/browser/browser.ini b/toolkit/components/nimbus/test/browser/browser.ini new file mode 100644 index 0000000000..a2845089ea --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser.ini @@ -0,0 +1,25 @@ +[DEFAULT] +support-files = + head.js +prefs = + # This turns off the update interval for fetching recipes from Remote Settings + app.normandy.run_interval_seconds=0 +skip-if = + toolkit == "android" + appname == "thunderbird" + os == "win" && os_version == "6.1" # Skip on Azure - frequent failure + +[browser_experiment_single_feature_enrollment.js] +[browser_prefs.js] +[browser_remotesettingsexperimentloader_remote_defaults.js] +[browser_remotesettingsexperimentloader_force_enrollment.js] +[browser_experimentstore_load.js] +[browser_experimentstore_load_single_feature.js] +skip-if = + os == "win" && os_version == "6.1" # Skip on Azure - frequent failure +[browser_remotesettings_experiment_enroll.js] +[browser_experiment_evaluate_jexl.js] +[browser_remotesettingsexperimentloader_init.js] +[browser_nimbus_telemetry.js] +tags = remote-settings + diff --git a/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js b/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js new file mode 100644 index 0000000000..c3667e04e7 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js @@ -0,0 +1,113 @@ +"use strict"; + +const { RemoteSettingsExperimentLoader } = ChromeUtils.import( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +add_setup(async function setup() { + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["app.shield.optoutstudies.enabled", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + }); +}); + +const FAKE_CONTEXT = { + experiment: ExperimentFakes.recipe("fake-test-experiment"), + source: "browser_experiment_evaluate_jexl", +}; + +add_task(async function test_throws_if_no_experiment_in_context() { + await Assert.rejects( + RemoteSettingsExperimentLoader.evaluateJexl("true", { + customThing: 1, + source: "test_throws_if_no_experiment_in_context", + }), + /Expected an .experiment/, + "should throw if experiment is not passed to the custom context" + ); +}); + +add_task(async function test_evaluate_jexl() { + Assert.deepEqual( + await RemoteSettingsExperimentLoader.evaluateJexl( + `["hello"]`, + FAKE_CONTEXT + ), + ["hello"], + "should return the evaluated result of a jexl expression" + ); +}); + +add_task(async function test_evaluate_custom_context() { + const result = await RemoteSettingsExperimentLoader.evaluateJexl( + "experiment.slug", + FAKE_CONTEXT + ); + Assert.equal( + result, + "fake-test-experiment", + "should have the custom .experiment context" + ); +}); + +add_task(async function test_evaluate_active_experiments_isFirstStartup() { + const result = await RemoteSettingsExperimentLoader.evaluateJexl( + "isFirstStartup", + FAKE_CONTEXT + ); + Assert.equal( + typeof result, + "boolean", + "should have a .isFirstStartup property from ExperimentManager " + ); +}); + +add_task(async function test_evaluate_active_experiments_activeExperiments() { + // Add an experiment to active experiments + const slug = "foo" + Math.random(); + // Init the store before we use it + await ExperimentManager.onStartup(); + + let recipe = ExperimentFakes.recipe(slug); + recipe.branches[0].slug = "mochitest-active-foo"; + delete recipe.branches[1]; + + let { + enrollmentPromise, + doExperimentCleanup, + } = ExperimentFakes.enrollmentHelper(recipe); + + await enrollmentPromise; + + Assert.equal( + await RemoteSettingsExperimentLoader.evaluateJexl( + `"${slug}" in activeExperiments`, + FAKE_CONTEXT + ), + true, + "should find an active experiment" + ); + + Assert.equal( + await RemoteSettingsExperimentLoader.evaluateJexl( + `"does-not-exist-fake" in activeExperiments`, + FAKE_CONTEXT + ), + false, + "should not find an experiment that doesn't exist" + ); + + await doExperimentCleanup(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js b/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js new file mode 100644 index 0000000000..17e821019f --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js @@ -0,0 +1,130 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentAPI, NimbusFeatures } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); + +const SINGLE_FEATURE_RECIPE = { + appId: "firefox-desktop", + appName: "firefox_desktop", + arguments: {}, + branches: [ + { + feature: { + featureId: "urlbar", + isEarlyStartup: true, + value: { + enabled: true, + quickSuggestEnabled: false, + quickSuggestNonSponsoredIndex: -1, + quickSuggestShouldShowOnboardingDialog: true, + quickSuggestShowOnboardingDialogAfterNRestarts: 2, + quickSuggestSponsoredIndex: -1, + }, + }, + ratio: 1, + slug: "control", + }, + { + feature: { + featureId: "urlbar", + isEarlyStartup: true, + value: { + enabled: true, + quickSuggestEnabled: true, + quickSuggestNonSponsoredIndex: -1, + quickSuggestShouldShowOnboardingDialog: false, + quickSuggestShowOnboardingDialogAfterNRestarts: 2, + quickSuggestSponsoredIndex: -1, + }, + }, + ratio: 1, + slug: "treatment", + }, + ], + bucketConfig: { + count: 10000, + namespace: "urlbar-9", + randomizationUnit: "normandy_id", + start: 0, + total: 10000, + }, + channel: "release", + endDate: null, + featureIds: ["urlbar"], + id: "firefox-suggest-history-vs-offline", + isEnrollmentPaused: false, + outcomes: [], + probeSets: [], + proposedDuration: 28, + proposedEnrollment: 7, + referenceBranch: "control", + schemaVersion: "1.5.0", + slug: "firefox-suggest-history-vs-offline", + startDate: "2021-07-21", + targeting: "true", + userFacingDescription: "Smarter suggestions in the AwesomeBar", + userFacingName: "Firefox Suggest - History vs Offline", +}; + +const SYNC_DATA_PREF_BRANCH = "nimbus.syncdatastore."; + +add_task(async function test_TODO() { + let { + enrollmentPromise, + doExperimentCleanup, + } = ExperimentFakes.enrollmentHelper(SINGLE_FEATURE_RECIPE); + let sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await enrollmentPromise; + + Assert.ok( + ExperimentAPI.getExperiment({ featureId: "urlbar" }), + "Should enroll in single feature experiment" + ); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}urlbar`), + "Should store early startup feature for sync access" + ); + Assert.equal( + Services.prefs.getIntPref( + `${SYNC_DATA_PREF_BRANCH}urlbar.quickSuggestSponsoredIndex` + ), + -1, + "Should store early startup variable for sync access" + ); + + Assert.equal( + NimbusFeatures.urlbar.getVariable( + "quickSuggestShowOnboardingDialogAfterNRestarts" + ), + 2, + "Should return value" + ); + + NimbusFeatures.urlbar.recordExposureEvent(); + + Assert.ok(stub.calledOnce, "Should be called once by urlbar"); + Assert.equal( + stub.firstCall.args[0].experimentSlug, + "firefox-suggest-history-vs-offline", + "Should have expected slug" + ); + Assert.equal( + stub.firstCall.args[0].featureId, + "urlbar", + "Should have expected featureId" + ); + + await doExperimentCleanup(); + sandbox.restore(); + NimbusFeatures.urlbar._didSendExposureEvent = false; +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js b/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js new file mode 100644 index 0000000000..92993a9771 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentStore } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentStore.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentFeatures } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +ChromeUtils.defineESModuleGetters(this, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", +}); + +function getPath() { + const profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile).path; + // NOTE: If this test is failing because you have updated this path in `ExperimentStore`, + // users will lose their old experiment data. You should do something to migrate that data. + return PathUtils.join(profileDir, "ExperimentStoreData.json"); +} + +// Ensure that data persisted to disk is succesfully loaded by the store. +// We write data to the expected location in the user profile and +// instantiate an ExperimentStore that should then see the value. +add_task(async function test_loadFromFile() { + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data.test = { + slug: "test", + active: true, + lastSeen: Date.now(), + }; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + await store.init(); + await store.ready(); + + Assert.equal( + previousSession.path, + store._store.path, + "Should have the same path" + ); + + Assert.ok( + store.get("test"), + "This should pass if the correct store path loaded successfully" + ); +}); + +add_task(async function test_load_from_disk_event() { + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + lastSeen: Date.now(), + }); + const stub = sinon.stub(); + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data.foo = experiment; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + store._onFeatureUpdate("green", stub); + + await store.init(); + await store.ready(); + + Assert.equal( + previousSession.path, + store._store.path, + "Should have the same path as previousSession." + ); + + await TestUtils.waitForCondition(() => stub.called, "Stub was called"); + + Assert.ok(stub.firstCall.args[1], "feature-experiment-loaded"); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js b/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js new file mode 100644 index 0000000000..7a7dc4fe47 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentStore } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentStore.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { NimbusFeatures, ExperimentAPI } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +ChromeUtils.defineESModuleGetters(this, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", +}); + +const SINGLE_FEATURE_RECIPE = { + ...ExperimentFakes.experiment(), + branch: { + feature: { + featureId: "urlbar", + value: { + valueThatWillDefinitelyShowUp: 42, + quickSuggestNonSponsoredIndex: 2021, + }, + }, + ratio: 1, + slug: "control", + }, + featureIds: ["urlbar"], + slug: "browser_experimentstore_load_single_feature", + userFacingDescription: "Smarter suggestions in the AwesomeBar", + userFacingName: "Firefox Suggest - History vs Offline", +}; + +function getPath() { + const profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile).path; + // NOTE: If this test is failing because you have updated this path in `ExperimentStore`, + // users will lose their old experiment data. You should do something to migrate that data. + return PathUtils.join(profileDir, "ExperimentStoreData.json"); +} + +add_task(async function test_load_from_disk_event() { + Services.prefs.setStringPref("messaging-system.log", "all"); + const stub = sinon.stub(); + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data[SINGLE_FEATURE_RECIPE.slug] = SINGLE_FEATURE_RECIPE; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + let apiStoreStub = sinon.stub(ExperimentAPI, "_store").get(() => store); + + store._onFeatureUpdate("urlbar", stub); + + await store.init(); + await store.ready(); + + await TestUtils.waitForCondition(() => stub.called, "Stub was called"); + Assert.ok( + store.get(SINGLE_FEATURE_RECIPE.slug)?.slug, + "Experiment is loaded from disk" + ); + Assert.ok(stub.firstCall.args[1], "feature-experiment-loaded"); + Assert.equal( + NimbusFeatures.urlbar.getAllVariables().valueThatWillDefinitelyShowUp, + SINGLE_FEATURE_RECIPE.branch.feature.value.valueThatWillDefinitelyShowUp, + "Should match getAllVariables" + ); + Assert.equal( + NimbusFeatures.urlbar.getVariable("quickSuggestNonSponsoredIndex"), + SINGLE_FEATURE_RECIPE.branch.feature.value.quickSuggestNonSponsoredIndex, + "Should match getVariable" + ); + + registerCleanupFunction(async () => { + // Remove the experiment from disk + const fileStore = new JSONFile({ path: getPath() }); + await fileStore.load(); + fileStore.data = {}; + fileStore.saveSoon(); + await fileStore.finalize(); + apiStoreStub.restore(); + }); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js b/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js new file mode 100644 index 0000000000..9197a78dad --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js @@ -0,0 +1,159 @@ +"use strict"; + +const { + ExperimentAPI, + _ExperimentFeature: ExperimentFeature, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const TELEMETRY_CATEGORY = "normandy"; +const TELEMETRY_OBJECT = "nimbus_experiment"; +// Included with active experiment information +const EXPERIMENT_TYPE = "nimbus"; +const EVENT_FILTER = { category: TELEMETRY_CATEGORY }; + +add_setup(async function() { + let sandbox = sinon.createSandbox(); + // stub the `observe` method to make sure the Experiment Manager + // pref listener doesn't trigger and cause side effects + sandbox.stub(ExperimentManager, "observe"); + await SpecialPowers.pushPrefEnv({ + set: [["app.shield.optoutstudies.enabled", true]], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + sandbox.restore(); + }); +}); + +add_task(async function test_experiment_enroll_unenroll_Telemetry() { + Services.telemetry.clearEvents(); + const cleanup = await ExperimentFakes.enrollWithFeatureConfig({ + featureId: "test-feature", + value: { enabled: false }, + }); + let experiment = ExperimentAPI.getExperiment({ + featureId: "test-feature", + }); + + Assert.ok(experiment.branch, "Should be enrolled in the experiment"); + TelemetryTestUtils.assertEvents( + [ + { + method: "enroll", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + experimentType: EXPERIMENT_TYPE, + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "unenroll", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + reason: "cleanup", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + }, + ], + EVENT_FILTER + ); +}); + +add_task(async function test_experiment_expose_Telemetry() { + const featureManifest = { + description: "Test feature", + exposureDescription: "Used in tests", + }; + const cleanup = await ExperimentFakes.enrollWithFeatureConfig({ + featureId: "test-feature", + value: { enabled: false }, + }); + + let experiment = ExperimentAPI.getExperiment({ + featureId: "test-feature", + }); + + const { featureId } = experiment.branch.features[0]; + const feature = new ExperimentFeature(featureId, featureManifest); + + Services.telemetry.clearEvents(); + feature.recordExposureEvent(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "expose", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + branchSlug: experiment.branch.slug, + featureId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); +}); + +add_task(async function test_rollout_expose_Telemetry() { + const featureManifest = { + description: "Test feature", + exposureDescription: "Used in tests", + }; + const cleanup = await ExperimentFakes.enrollWithRollout({ + featureId: "test-feature", + value: { enabled: false }, + }); + + let rollout = ExperimentAPI.getRolloutMetaData({ + featureId: "test-feature", + }); + + Assert.ok(rollout.slug, "Found enrolled experiment"); + + const feature = new ExperimentFeature("test-feature", featureManifest); + + Services.telemetry.clearEvents(); + feature.recordExposureEvent(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "expose", + object: TELEMETRY_OBJECT, + value: rollout.slug, + extra: { + branchSlug: rollout.branch.slug, + featureId: feature.featureId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_prefs.js b/toolkit/components/nimbus/test/browser/browser_prefs.js new file mode 100644 index 0000000000..7e63618a24 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_prefs.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentAPI, NimbusFeatures } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); + +const EXPERIMENT_VALUE = "experiment-value"; +const ROLLOUT_VALUE = "rollout-value"; +const ROLLOUT = "rollout"; +const EXPERIMENT = "experiment"; + +const VALUES = { + [ROLLOUT]: ROLLOUT_VALUE, + [EXPERIMENT]: EXPERIMENT_VALUE, +}; + +add_task(async function test_prefs_priority() { + const pref = "nimbus.testing.testSetString"; + const featureId = "testFeature"; + + async function doTest({ settingEnrollments, expectedValue }) { + info( + `Enrolling in a rollout and experiment where the ${settingEnrollments.join( + " and " + )} set the same pref variable.` + ); + const enrollmentCleanup = []; + + for (const enrollmentKind of [ROLLOUT, EXPERIMENT]) { + const config = { + featureId, + value: {}, + }; + + if (settingEnrollments.includes(enrollmentKind)) { + config.value.testSetString = VALUES[enrollmentKind]; + } + + enrollmentCleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(config, { + isRollout: enrollmentKind === ROLLOUT, + }) + ); + } + + is( + NimbusFeatures[featureId].getVariable("testSetString"), + expectedValue, + "Expected the variable to match the expected value" + ); + + is( + Services.prefs.getStringPref(pref), + expectedValue, + "Expected the pref to match the expected value" + ); + + for (const cleanup of enrollmentCleanup) { + await cleanup(); + } + + Services.prefs.deleteBranch(pref); + } + + for (const settingEnrollments of [ + [ROLLOUT], + [EXPERIMENT], + [ROLLOUT, EXPERIMENT], + ]) { + const expectedValue = settingEnrollments.includes(EXPERIMENT) + ? EXPERIMENT_VALUE + : ROLLOUT_VALUE; + + await doTest({ settingEnrollments, expectedValue }); + } +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js b/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js new file mode 100644 index 0000000000..10143a74cb --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js @@ -0,0 +1,115 @@ +"use strict"; + +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.import( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm" +); +const { ExperimentAPI } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +let rsClient; + +add_setup(async function() { + rsClient = RemoteSettings("nimbus-desktop-experiments"); + await rsClient.db.importChanges({}, Date.now(), [], { clear: true }); + + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["datareporting.healthreport.uploadEnabled", true], + ["app.shield.optoutstudies.enabled", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + await rsClient.db.clear(); + }); +}); + +add_task(async function test_experimentEnrollment() { + // Need to randomize the slug so subsequent test runs don't skip enrollment + // due to a conflicting slug + const recipe = ExperimentFakes.recipe("foo" + Math.random(), { + bucketConfig: { + start: 0, + // Make sure the experiment enrolls + count: 10000, + total: 10000, + namespace: "mochitest", + randomizationUnit: "normandy_id", + }, + }); + await rsClient.db.importChanges({}, Date.now(), [recipe], { + clear: true, + }); + + let waitForExperimentEnrollment = ExperimentFakes.waitForExperimentUpdate( + ExperimentAPI, + { slug: recipe.slug } + ); + RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + + await waitForExperimentEnrollment; + + let experiment = ExperimentAPI.getExperiment({ + slug: recipe.slug, + }); + + Assert.ok(experiment.active, "Should be enrolled in the experiment"); + + let waitForExperimentUnenrollment = ExperimentFakes.waitForExperimentUpdate( + ExperimentAPI, + { slug: recipe.slug } + ); + ExperimentManager.unenroll(recipe.slug, "mochitest-cleanup"); + + await waitForExperimentUnenrollment; + + experiment = ExperimentAPI.getExperiment({ + slug: recipe.slug, + }); + + Assert.ok(!experiment.active, "Experiment is no longer active"); + ExperimentAPI._store._deleteForTests(recipe.slug); +}); + +add_task(async function test_experimentEnrollment_startup() { + // Studies pref can turn the feature off but if the feature pref is off + // then it stays off. + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.rsexperimentloader.enabled", false], + ["app.shield.optoutstudies.enabled", false], + ], + }); + + Assert.ok(!RemoteSettingsExperimentLoader.enabled, "Should be disabled"); + + await SpecialPowers.pushPrefEnv({ + set: [["app.shield.optoutstudies.enabled", true]], + }); + + Assert.ok( + !RemoteSettingsExperimentLoader.enabled, + "Should still be disabled (feature pref is off)" + ); + + await SpecialPowers.pushPrefEnv({ + set: [["messaging-system.rsexperimentloader.enabled", true]], + }); + + Assert.ok( + RemoteSettingsExperimentLoader.enabled, + "Should finally be enabled" + ); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js new file mode 100644 index 0000000000..584b595dac --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js @@ -0,0 +1,147 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.import( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); + +async function setup(recipes) { + const client = RemoteSettings("nimbus-desktop-experiments"); + await client.db.importChanges({}, Date.now(), recipes, { + clear: true, + }); + + await BrowserTestUtils.waitForCondition( + async () => (await client.get()).length, + "RS is ready" + ); + + registerCleanupFunction(async () => { + await client.db.clear(); + }); + + return client; +} + +add_setup(async function() { + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["datareporting.healthreport.uploadEnabled", true], + ["app.shield.optoutstudies.enabled", true], + ["nimbus.debug", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + }); +}); + +add_task(async function test_fetch_recipe_and_branch_no_debug() { + const sandbox = sinon.createSandbox(); + Services.prefs.setBoolPref("nimbus.debug", false); + let stub = sandbox.stub(ExperimentManager, "forceEnroll").returns(true); + let recipes = [ExperimentFakes.recipe("slug123")]; + + await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "control", + }), + /Could not opt in/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + Services.prefs.setBoolPref("nimbus.debug", true); + + const result = await RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "control", + }); + + Assert.ok(result, "Pref was turned on"); + Assert.ok(stub.called, "forceEnroll is called"); + + sandbox.restore(); +}); + +add_task(async function test_fetch_recipe_and_branch_badslug() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll").returns(true); + let recipes = [ExperimentFakes.recipe("slug123")]; + + await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "other_slug", + branch: "control", + }), + /Could not find experiment slug other_slug/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + sandbox.restore(); +}); + +add_task(async function test_fetch_recipe_and_branch_badbranch() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll").returns(true); + let recipes = [ExperimentFakes.recipe("slug123")]; + + await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "other_branch", + }), + /Could not find branch slug other_branch in slug123/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + sandbox.restore(); +}); + +add_task(async function test_fetch_recipe_and_branch() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll").returns(true); + let recipes = [ExperimentFakes.recipe("slug_fetch_recipe")]; + + await setup(recipes); + let result = await RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug_fetch_recipe", + branch: "control", + }); + + Assert.ok(result, "Recipe found"); + Assert.ok(stub.called, "Called forceEnroll"); + Assert.deepEqual(stub.firstCall.args[0], recipes[0], "Called with recipe"); + Assert.deepEqual( + stub.firstCall.args[1], + recipes[0].branches[0], + "Called with branch" + ); + + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js new file mode 100644 index 0000000000..b2479ae48b --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js @@ -0,0 +1,80 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); +const { ExperimentAPI } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); + +function getRecipe(slug) { + return ExperimentFakes.recipe(slug, { + bucketConfig: { + start: 0, + // Make sure the experiment enrolls + count: 10000, + total: 10000, + namespace: "mochitest", + randomizationUnit: "normandy_id", + }, + targeting: "!(experiment.slug in activeExperiments)", + }); +} + +add_task(async function test_double_feature_enrollment() { + let sandbox = sinon.createSandbox(); + let sendFailureTelemetryStub = sandbox.stub( + ExperimentManager, + "sendFailureTelemetry" + ); + await ExperimentAPI.ready(); + + Assert.ok(ExperimentManager.store.getAllActive().length === 0, "Clean state"); + + let recipe1 = getRecipe("foo" + Math.random()); + let recipe2 = getRecipe("foo" + Math.random()); + + let enrollPromise1 = ExperimentFakes.waitForExperimentUpdate(ExperimentAPI, { + slug: recipe1.slug, + }); + + ExperimentManager.enroll(recipe1, "test_double_feature_enrollment"); + await enrollPromise1; + ExperimentManager.enroll(recipe2, "test_double_feature_enrollment"); + + Assert.equal( + ExperimentManager.store.getAllActive().length, + 1, + "1 active experiment" + ); + + await BrowserTestUtils.waitForCondition( + () => sendFailureTelemetryStub.callCount === 1, + "Expected to fail one of the recipes" + ); + + Assert.equal( + sendFailureTelemetryStub.firstCall.args[0], + "enrollFailed", + "Check expected event" + ); + Assert.ok( + sendFailureTelemetryStub.firstCall.args[1] === recipe1.slug || + sendFailureTelemetryStub.firstCall.args[1] === recipe2.slug, + "Failed one of the two recipes" + ); + Assert.equal( + sendFailureTelemetryStub.firstCall.args[2], + "feature-conflict", + "Check expected reason" + ); + + await ExperimentFakes.cleanupAll([recipe1.slug]); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js new file mode 100644 index 0000000000..211657660c --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js @@ -0,0 +1,573 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { + _ExperimentFeature: ExperimentFeature, + NimbusFeatures, + ExperimentAPI, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); +const { ExperimentTestUtils } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.import( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm" +); +const { TelemetryEnvironment } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryEnvironment.sys.mjs" +); + +const FOO_FAKE_FEATURE_MANIFEST = { + isEarlyStartup: true, + variables: { + remoteValue: { + type: "int", + }, + enabled: { + type: "boolean", + }, + }, +}; + +const BAR_FAKE_FEATURE_MANIFEST = { + isEarlyStartup: true, + variables: { + remoteValue: { + type: "int", + }, + enabled: { + type: "boolean", + }, + }, +}; + +const ENSURE_ENROLLMENT = { + targeting: "true", + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, +}; + +const REMOTE_CONFIGURATION_FOO = ExperimentFakes.recipe("foo-rollout", { + isRollout: true, + branches: [ + { + slug: "foo-rollout-branch", + ratio: 1, + features: [ + { + featureId: "foo", + isEarlyStartup: true, + value: { remoteValue: 42, enabled: true }, + }, + ], + }, + ], + ...ENSURE_ENROLLMENT, +}); +const REMOTE_CONFIGURATION_BAR = ExperimentFakes.recipe("bar-rollout", { + isRollout: true, + branches: [ + { + slug: "bar-rollout-branch", + ratio: 1, + features: [ + { + featureId: "bar", + isEarlyStartup: true, + value: { remoteValue: 3, enabled: true }, + }, + ], + }, + ], + ...ENSURE_ENROLLMENT, +}); + +const SYNC_DEFAULTS_PREF_BRANCH = "nimbus.syncdefaultsstore."; + +async function setup(configuration) { + const client = RemoteSettings("nimbus-desktop-experiments"); + await client.db.importChanges( + {}, + Date.now(), + configuration || [REMOTE_CONFIGURATION_FOO, REMOTE_CONFIGURATION_BAR], + { + clear: true, + } + ); + + // Simulate a state where no experiment exists. + const cleanup = () => + client.db.importChanges({}, Date.now(), [], { clear: true }); + return { client, cleanup }; +} + +add_task(async function test_remote_fetch_and_ready() { + const fooInstance = new ExperimentFeature("foo", FOO_FAKE_FEATURE_MANIFEST); + const barInstance = new ExperimentFeature("bar", BAR_FAKE_FEATURE_MANIFEST); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + fooInstance, + barInstance + ); + + const sandbox = sinon.createSandbox(); + const setExperimentActiveStub = sandbox.stub( + TelemetryEnvironment, + "setExperimentActive" + ); + const setExperimentInactiveStub = sandbox.stub( + TelemetryEnvironment, + "setExperimentInactive" + ); + + Assert.equal( + fooInstance.getVariable("remoteValue"), + undefined, + "This prop does not exist before we sync" + ); + + // Create to promises that get resolved when the features update + // with the remote setting rollouts + let fooUpdate = new Promise(resolve => fooInstance.onUpdate(resolve)); + let barUpdate = new Promise(resolve => barInstance.onUpdate(resolve)); + + await ExperimentAPI.ready(); + + let { cleanup } = await setup(); + + // Fake being initialized so we can update recipes + // we don't need to start any timers + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes( + "browser_rsel_remote_defaults" + ); + + // We need to await here because remote configurations are processed + // async to evaluate targeting + await Promise.all([fooUpdate, barUpdate]); + + Assert.equal( + fooInstance.getVariable("remoteValue"), + REMOTE_CONFIGURATION_FOO.branches[0].features[0].value.remoteValue, + "`foo` feature is set by remote defaults" + ); + Assert.equal( + barInstance.getVariable("remoteValue"), + REMOTE_CONFIGURATION_BAR.branches[0].features[0].value.remoteValue, + "`bar` feature is set by remote defaults" + ); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`), + "Pref cache is set" + ); + + // Check if we sent active experiment data for defaults + Assert.equal( + setExperimentActiveStub.callCount, + 2, + "setExperimentActive called once per feature" + ); + + Assert.ok( + setExperimentActiveStub.calledWith( + REMOTE_CONFIGURATION_FOO.slug, + REMOTE_CONFIGURATION_FOO.branches[0].slug, + { + type: "nimbus-rollout", + enrollmentId: sinon.match.string, + } + ), + "should call setExperimentActive with `foo` feature" + ); + Assert.ok( + setExperimentActiveStub.calledWith( + REMOTE_CONFIGURATION_BAR.slug, + REMOTE_CONFIGURATION_BAR.branches[0].slug, + { + type: "nimbus-rollout", + enrollmentId: sinon.match.string, + } + ), + "should call setExperimentActive with `bar` feature" + ); + + // Test Glean experiment API interaction + Assert.equal( + Services.fog.testGetExperimentData(REMOTE_CONFIGURATION_FOO.slug).branch, + REMOTE_CONFIGURATION_FOO.branches[0].slug, + "Glean.setExperimentActive called with `foo` feature" + ); + Assert.equal( + Services.fog.testGetExperimentData(REMOTE_CONFIGURATION_BAR.slug).branch, + REMOTE_CONFIGURATION_BAR.branches[0].slug, + "Glean.setExperimentActive called with `bar` feature" + ); + + Assert.equal(fooInstance.getVariable("remoteValue"), 42, "Has rollout value"); + Assert.equal(barInstance.getVariable("remoteValue"), 3, "Has rollout value"); + + // Clear RS db and load again. No configurations so should clear the cache. + await cleanup(); + await RemoteSettingsExperimentLoader.updateRecipes( + "browser_rsel_remote_defaults" + ); + + Assert.ok( + !fooInstance.getVariable("remoteValue"), + "foo-rollout should be removed" + ); + Assert.ok( + !barInstance.getVariable("remoteValue"), + "bar-rollout should be removed" + ); + + // Check if we sent active experiment data for defaults + Assert.equal( + setExperimentInactiveStub.callCount, + 2, + "setExperimentInactive called once per feature" + ); + + Assert.ok( + setExperimentInactiveStub.calledWith(REMOTE_CONFIGURATION_FOO.slug), + "should call setExperimentInactive with `foo` feature" + ); + Assert.ok( + setExperimentInactiveStub.calledWith(REMOTE_CONFIGURATION_BAR.slug), + "should call setExperimentInactive with `bar` feature" + ); + + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Should clear the pref" + ); + Assert.ok(!barInstance.getVariable("remoteValue"), "Should be missing"); + + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + ExperimentAPI._store._deleteForTests(REMOTE_CONFIGURATION_FOO.slug); + ExperimentAPI._store._deleteForTests(REMOTE_CONFIGURATION_BAR.slug); + sandbox.restore(); + + cleanupTestFeatures(); + await cleanup(); +}); + +add_task(async function test_remote_fetch_on_updateRecipes() { + let sandbox = sinon.createSandbox(); + let updateRecipesStub = sandbox.stub( + RemoteSettingsExperimentLoader, + "updateRecipes" + ); + // Work around the pref change callback that would trigger `setTimer` + sandbox.replaceGetter( + RemoteSettingsExperimentLoader, + "intervalInSeconds", + () => 1 + ); + + // This will un-register the timer + RemoteSettingsExperimentLoader._initialized = true; + RemoteSettingsExperimentLoader.uninit(); + Services.prefs.clearUserPref( + "app.update.lastUpdateTime.rs-experiment-loader-timer" + ); + + RemoteSettingsExperimentLoader.setTimer(); + + await BrowserTestUtils.waitForCondition( + () => updateRecipesStub.called, + "Wait for timer to call" + ); + + Assert.ok(updateRecipesStub.calledOnce, "Timer calls function"); + Assert.equal(updateRecipesStub.firstCall.args[0], "timer", "Called by timer"); + sandbox.restore(); + // This will un-register the timer + RemoteSettingsExperimentLoader._initialized = true; + RemoteSettingsExperimentLoader.uninit(); + Services.prefs.clearUserPref( + "app.update.lastUpdateTime.rs-experiment-loader-timer" + ); +}); + +add_task(async function test_finalizeRemoteConfigs_cleanup() { + const featureFoo = new ExperimentFeature("foo", { + description: "mochitests", + variables: { + foo: { type: "boolean" }, + }, + }); + const featureBar = new ExperimentFeature("bar", { + description: "mochitests", + variables: { + bar: { type: "boolean" }, + }, + }); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + featureFoo, + featureBar + ); + + let fooCleanup = await ExperimentFakes.enrollWithRollout( + { + featureId: "foo", + isEarlyStartup: true, + value: { foo: true }, + }, + { + source: "rs-loader", + } + ); + await ExperimentFakes.enrollWithRollout( + { + featureId: "bar", + isEarlyStartup: true, + value: { bar: true }, + }, + { + source: "rs-loader", + } + ); + let stubFoo = sinon.stub(); + let stubBar = sinon.stub(); + featureFoo.onUpdate(stubFoo); + featureBar.onUpdate(stubBar); + let cleanupPromise = new Promise(resolve => featureBar.onUpdate(resolve)); + + Services.prefs.setStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}foo`, + JSON.stringify({ foo: true, branch: { feature: { featureId: "foo" } } }) + ); + Services.prefs.setStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}bar`, + JSON.stringify({ bar: true, branch: { feature: { featureId: "bar" } } }) + ); + + const remoteConfiguration = { + ...REMOTE_CONFIGURATION_FOO, + branches: [ + { + ...REMOTE_CONFIGURATION_FOO.branches[0], + features: [ + { + ...REMOTE_CONFIGURATION_FOO.branches[0].features[0], + value: { + foo: true, + }, + }, + ], + }, + ], + }; + + const { cleanup } = await setup([remoteConfiguration]); + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes(); + await cleanupPromise; + + Assert.ok( + stubFoo.notCalled, + "Not called, not enrolling in rollout feature already exists" + ); + Assert.ok(stubBar.called, "Called because no recipe is seen, cleanup"); + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}foo`), + "Pref is not cleared" + ); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Pref was cleared" + ); + + await fooCleanup(); + // This will also remove the inactive recipe from the store + // the previous update (from recipe not seen code path) + // only sets the recipe as inactive + ExperimentAPI._store._deleteForTests("bar-rollout"); + ExperimentAPI._store._deleteForTests("foo-rollout"); + + cleanupTestFeatures(); + cleanup(); +}); + +// If the remote config data returned from the store is not modified +// this test should not throw +add_task(async function remote_defaults_no_mutation() { + let sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getRolloutForFeature").returns( + Cu.cloneInto( + { + featureIds: ["foo"], + branch: { + features: [{ featureId: "foo", value: { remoteStub: true } }], + }, + }, + {}, + { deepFreeze: true } + ) + ); + + let fooInstance = new ExperimentFeature("foo", FOO_FAKE_FEATURE_MANIFEST); + let config = fooInstance.getAllVariables(); + + Assert.ok(config.remoteStub, "Got back the expected value"); + + sandbox.restore(); +}); + +add_task(async function remote_defaults_active_remote_defaults() { + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + let barFeature = new ExperimentFeature("bar", { + description: "mochitest", + variables: { enabled: { type: "boolean" } }, + }); + let fooFeature = new ExperimentFeature("foo", { + description: "mochitest", + variables: { enabled: { type: "boolean" } }, + }); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + barFeature, + fooFeature + ); + + let rollout1 = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "bar-rollout-branch", + ratio: 1, + features: [ + { + featureId: "bar", + value: { enabled: true }, + }, + ], + }, + ], + isRollout: true, + ...ENSURE_ENROLLMENT, + targeting: "true", + }); + + let rollout2 = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "foo-rollout-branch", + ratio: 1, + features: [ + { + featureId: "foo", + value: { enabled: true }, + }, + ], + }, + ], + isRollout: true, + ...ENSURE_ENROLLMENT, + targeting: "'bar' in activeRollouts", + }); + + // Order is important, rollout2 won't match at first + const { cleanup } = await setup([rollout2, rollout1]); + let updatePromise = new Promise(resolve => barFeature.onUpdate(resolve)); + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + + await updatePromise; + + Assert.ok(barFeature.getVariable("enabled"), "Enabled on first sync"); + Assert.ok(!fooFeature.getVariable("enabled"), "Targeting doesn't match"); + + let featureUpdate = new Promise(resolve => fooFeature.onUpdate(resolve)); + await RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + await featureUpdate; + + Assert.ok(fooFeature.getVariable("enabled"), "Targeting should match"); + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + + cleanup(); + cleanupTestFeatures(); +}); + +add_task(async function remote_defaults_variables_storage() { + let barFeature = new ExperimentFeature("bar", { + description: "mochitest", + variables: { + enabled: { + type: "boolean", + }, + storage: { + type: "int", + }, + object: { + type: "json", + }, + string: { + type: "string", + }, + bool: { + type: "boolean", + }, + }, + }); + let rolloutValue = { + storage: 42, + object: { foo: "foo" }, + string: "string", + bool: true, + enabled: true, + }; + + let doCleanup = await ExperimentFakes.enrollWithRollout({ + featureId: "bar", + isEarlyStartup: true, + value: rolloutValue, + }); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Experiment stored in prefs" + ); + Assert.ok( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, 0), + "Stores variable in separate pref" + ); + Assert.equal( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, 0), + 42, + "Stores variable in correct type" + ); + Assert.deepEqual( + barFeature.getAllVariables(), + rolloutValue, + "Test types are returned correctly" + ); + + await doCleanup(); + + Assert.equal( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, -1), + -1, + "Variable pref is cleared" + ); + Assert.ok(!barFeature.getVariable("string"), "Variable is no longer defined"); + ExperimentAPI._store._deleteForTests("bar"); + ExperimentAPI._store._deleteForTests("bar-rollout"); +}); diff --git a/toolkit/components/nimbus/test/browser/head.js b/toolkit/components/nimbus/test/browser/head.js new file mode 100644 index 0000000000..c63d7ab1dd --- /dev/null +++ b/toolkit/components/nimbus/test/browser/head.js @@ -0,0 +1,38 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Globals +const { sinon } = ChromeUtils.import("resource://testing-common/Sinon.jsm"); +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +XPCOMUtils.defineLazyModuleGetters(this, { + ExperimentManager: "resource://nimbus/lib/ExperimentManager.jsm", + ExperimentTestUtils: "resource://testing-common/NimbusTestUtils.jsm", + ExperimentFakes: "resource://testing-common/NimbusTestUtils.jsm", +}); + +add_setup(function() { + let sandbox = sinon.createSandbox(); + + /* We stub the functions that operate with enrollments and remote rollouts + * so that any access to store something is implicitly validated against + * the schema and no records have missing (or extra) properties while in tests + */ + + let origAddExperiment = ExperimentManager.store.addEnrollment.bind( + ExperimentManager.store + ); + sandbox + .stub(ExperimentManager.store, "addEnrollment") + .callsFake(async enrollment => { + await ExperimentTestUtils.validateEnrollment(enrollment); + return origAddExperiment(enrollment); + }); + + registerCleanupFunction(() => { + sandbox.restore(); + }); +}); diff --git a/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp b/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp new file mode 100644 index 0000000000..36df15e963 --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp @@ -0,0 +1,187 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Preferences.h" +#include "mozilla/browser/NimbusFeatures.h" + +using namespace mozilla; + +static bool gPrefUpdate = false; + +TEST(NimbusFeaturesGet, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 42, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 42); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.foo.enabled", true, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, false)); + + ASSERT_EQ(Preferences::ClearUser("nimbus.syncdatastore.foo.value"), NS_OK); +} + +TEST(NimbusFeaturesGetRollout, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.rollout.value", 7, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("rollout"_ns, "value"_ns, 0), 7); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdefaultsstore.rollout.enabled", + true, PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("rollout"_ns, "enabled"_ns, false)); +} + +TEST(NimbusFeaturesExperimentPriorityOverRollouts, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.feature.value", 12, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.feature.value", 22, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("feature"_ns, "value"_ns, 0), 12); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.feature.enabled", true, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdefaultsstore.feature.enabled", + false, PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("feature"_ns, "enabled"_ns, false)); +} + +TEST(NimbusFeaturesDataSourcePrecedence, Errors) +{ + const auto FALLBACK_VALUE = 1; + const auto EXPERIMENT_VALUE = 2; + const auto ROLLOUT_VALUE = 3; + + ASSERT_EQ(Preferences::SetInt("nimbus.testing.testInt", FALLBACK_VALUE, + PrefValueKind::User), + NS_OK); + + // If there is no experiment or rollout, the fallback value should be + // returned. + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + FALLBACK_VALUE); + + // Enroll in an experiment. + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.testFeature.testInt", + EXPERIMENT_VALUE, PrefValueKind::User), + NS_OK); + + // Enroll in a rollout. + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.testFeature.testInt", + ROLLOUT_VALUE, PrefValueKind::User), + NS_OK); + + // Experiment value should take precedence. + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + EXPERIMENT_VALUE); + + // After experiments it should default to rollouts. + Preferences::ClearUser("nimbus.syncdatastore.testFeature.testInt"); + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + ROLLOUT_VALUE); + + // Cleanup + Preferences::ClearUser("nimbus.syncdefaultsstore.testFeature.testInt"); + Preferences::ClearUser("nimbus.testing.testInt"); +} + +static void FooValueUpdated(const char* aPref, void* aUserData) { + ASSERT_STREQ(aPref, "nimbus.syncdatastore.foo.value"); + ASSERT_EQ(aUserData, reinterpret_cast<void*>(13)); + + ASSERT_FALSE(gPrefUpdate); + gPrefUpdate = true; + + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 24); +} + +static void BarRolloutValueUpdated(const char* aPref, void* aUserData) { + ASSERT_STREQ(aPref, "nimbus.syncdefaultsstore.bar.value"); + + ASSERT_FALSE(gPrefUpdate); + gPrefUpdate = true; +} + +TEST(NimbusFeaturesGetFallback, Errors) +{ + // No experiment is set and we expect to return fallback pref values + + // As defined by fallbackPref browser.aboutwelcome.skipFocus + // in FeatureManifest.yaml + Preferences::SetBool("browser.aboutwelcome.skipFocus", true, + PrefValueKind::Default); + ASSERT_EQ(NimbusFeatures::GetBool("aboutwelcome"_ns, "skipFocus"_ns, false), + true); + Preferences::SetBool("browser.aboutwelcome.skipFocus", false, + PrefValueKind::User); + ASSERT_EQ(NimbusFeatures::GetBool("aboutwelcome"_ns, "skipFocus"_ns, true), + false); + Preferences::ClearUser("browser.aboutwelcome.skipFocus"); + + const auto FALLBACK_VALUE = 5; + const auto DEFAULT_VALUE = 42; + + Preferences::SetInt("nimbus.testing.testInt", FALLBACK_VALUE, + PrefValueKind::Default); + ASSERT_EQ( + NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, DEFAULT_VALUE), + FALLBACK_VALUE); + + Preferences::ClearUser("nimbus.testing.testInt"); +} + +TEST(NimbusFeaturesUpdate, Errors) +{ + // Verify updating foo.value calls FooValueUpdated. + ASSERT_EQ(NimbusFeatures::OnUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ( + NimbusFeatures::OnUpdate("bar"_ns, "value"_ns, BarRolloutValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 24, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(gPrefUpdate); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 24); + + // Verify updating foo.enabled doesn't call FooValueUpdated. + ASSERT_TRUE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, false)); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.foo.enabled", false, + PrefValueKind::User), + NS_OK); + ASSERT_FALSE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, true)); + gPrefUpdate = false; + + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.bar.value", 25, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(gPrefUpdate); + gPrefUpdate = false; + + // Verify OffUpdate requires a matching user data pointer to unregister. + ASSERT_EQ(NimbusFeatures::OffUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(14)), + NS_ERROR_FAILURE); + + // Verify updating foo.value no longer calls FooValueUpdated after it has + // been unregistered. + ASSERT_EQ(NimbusFeatures::OffUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 25, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 25); +} diff --git a/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp b/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp new file mode 100644 index 0000000000..7b0d1e0e7b --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Preferences.h" +#include "mozilla/browser/NimbusFeatures.h" +#include "js/Array.h" +#include "js/PropertyAndElement.h" +#include "js/TypeDecls.h" +#include "TelemetryFixture.h" +#include "TelemetryTestHelpers.h" + +using namespace mozilla; +using namespace TelemetryTestHelpers; + +class NimbusTelemetryFixture : public TelemetryTestFixture {}; + +TEST_F(NimbusTelemetryFixture, NimbusFeaturesTelemetry) { + constexpr auto prefName = "nimbus.syncdatastore.foo"_ns; + constexpr auto prefValue = + R"({"slug":"experiment-slug","branch":{"slug":"branch-slug"}})"; + AutoJSContextWithGlobal cx(mCleanGlobal); + Unused << mTelemetry->ClearEvents(); + + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns), NS_ERROR_UNEXPECTED) + << "Should fail because not enrolled in experiment"; + // Set the experiment info for `foo` + Preferences::SetCString(prefName.get(), prefValue); + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns), NS_OK) + << "Should work for the 2nd call"; + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns, true), NS_ERROR_ABORT) + << "Should abort because we've sent exposure and aOnce is true"; + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("bar"_ns), NS_ERROR_UNEXPECTED) + << "Should fail because we don't have an experiment for bar"; + + JS::Rooted<JS::Value> eventsSnapshot(cx.GetJSContext()); + GetEventSnapshot(cx.GetJSContext(), &eventsSnapshot); + ASSERT_TRUE(EventPresent(cx.GetJSContext(), eventsSnapshot, "normandy"_ns, + "expose"_ns, "nimbus_experiment"_ns)); +} diff --git a/toolkit/components/nimbus/test/gtest/moz.build b/toolkit/components/nimbus/test/gtest/moz.build new file mode 100644 index 0000000000..8eec5dd2c2 --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/moz.build @@ -0,0 +1,18 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "NimbusFeatures_GetTest.cpp", + "NimbusFeatures_RecordExposure.cpp", +] + +LOCAL_INCLUDES += [ + "/toolkit/components/telemetry/tests/gtest", +] + +FINAL_LIBRARY = "xul-gtest" + +REQUIRES_UNIFIED_BUILD = True diff --git a/toolkit/components/nimbus/test/unit/head.js b/toolkit/components/nimbus/test/unit/head.js new file mode 100644 index 0000000000..3f25420d19 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/head.js @@ -0,0 +1,11 @@ +"use strict"; +// Globals + +const { sinon } = ChromeUtils.import("resource://testing-common/Sinon.jsm"); +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +XPCOMUtils.defineLazyModuleGetters(this, { + ExperimentFakes: "resource://testing-common/NimbusTestUtils.jsm", + ExperimentTestUtils: "resource://testing-common/NimbusTestUtils.jsm", +}); diff --git a/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json b/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json new file mode 100644 index 0000000000..8966140c34 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json @@ -0,0 +1,190 @@ +{ + "id": "msw-late-setdefault", + "template": "multistage", + "screens": [ + { + "id": "AW_GET_STARTED", + "order": 0, + "content": { + "zap": true, + "title": { + "string_id": "onboarding-multistage-welcome-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-welcome-subtitle" + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-welcome-primary-button-label" + }, + "action": { + "navigate": true + } + }, + "secondary_button": { + "text": { + "string_id": "onboarding-multistage-welcome-secondary-button-text" + }, + "label": { + "string_id": "onboarding-multistage-welcome-secondary-button-label" + }, + "position": "top", + "action": { + "type": "SHOW_FIREFOX_ACCOUNTS", + "addFlowParams": true, + "data": { + "entrypoint": "activity-stream-firstrun" + } + } + } + } + }, + { + "id": "AW_IMPORT_SETTINGS", + "order": 1, + "content": { + "zap": true, + "disclaimer": { + "string_id": "onboarding-import-sites-disclaimer" + }, + "title": { + "string_id": "onboarding-multistage-import-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-import-subtitle" + }, + "tiles": { + "type": "topsites", + "showTitles": true + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-import-primary-button-label" + }, + "action": { + "type": "SHOW_MIGRATION_WIZARD", + "navigate": true + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-import-secondary-button-label" + }, + "action": { + "navigate": true + } + } + } + }, + { + "id": "AW_CHOOSE_THEME", + "order": 2, + "content": { + "zap": true, + "title": { + "string_id": "onboarding-multistage-theme-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-theme-subtitle" + }, + "tiles": { + "type": "theme", + "action": { + "theme": "<event>" + }, + "data": [ + { + "theme": "automatic", + "label": { + "string_id": "onboarding-multistage-theme-label-automatic" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-automatic-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-automatic-2" + } + }, + { + "theme": "light", + "label": { + "string_id": "onboarding-multistage-theme-label-light" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-light-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-light" + } + }, + { + "theme": "dark", + "label": { + "string_id": "onboarding-multistage-theme-label-dark" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-dark-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-dark" + } + }, + { + "theme": "alpenglow", + "label": { + "string_id": "onboarding-multistage-theme-label-alpenglow" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-alpenglow-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-alpenglow" + } + } + ] + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-theme-primary-button-label" + }, + "action": { + "navigate": true + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-theme-secondary-button-label" + }, + "action": { + "theme": "automatic", + "navigate": true + } + } + } + }, + { + "id": "AW_SET_DEFAULT", + "order": 3, + "content": { + "zap": true, + "title": "Make Firefox your default browser", + "subtitle": "Speed, safety, and privacy every time you browse.", + "primary_button": { + "label": "Make Default", + "action": { + "navigate": true, + "type": "SET_DEFAULT_BROWSER" + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-import-secondary-button-label" + }, + "action": { + "navigate": true + } + } + } + } + ] +} diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js new file mode 100644 index 0000000000..1166f99976 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js @@ -0,0 +1,578 @@ +"use strict"; + +const { ExperimentAPI } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); +const COLLECTION_ID_PREF = "messaging-system.rsexperimentloader.collection_id"; + +/** + * #getExperiment + */ +add_task(async function test_getExperiment_fromChild_slug() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => ExperimentFakes.childStore()); + + await manager.store.addEnrollment(expected); + + // Wait to sync to child + await TestUtils.waitForCondition( + () => ExperimentAPI.getExperiment({ slug: "foo" }), + "Wait for child to sync" + ); + + Assert.equal( + ExperimentAPI.getExperiment({ slug: "foo" }).slug, + expected.slug, + "should return an experiment by slug" + ); + + Assert.deepEqual( + ExperimentAPI.getExperiment({ slug: "foo" }).branch, + expected.branch, + "should return the right branch by slug" + ); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_fromParent_slug() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + Assert.equal( + ExperimentAPI.getExperiment({ slug: "foo" }).slug, + expected.slug, + "should return an experiment by slug" + ); + + sandbox.restore(); +}); + +add_task(async function test_getExperimentMetaData() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + let metadata = ExperimentAPI.getExperimentMetaData({ slug: expected.slug }); + + Assert.equal( + Object.keys(metadata.branch).length, + 1, + "Should only expose one property" + ); + Assert.equal( + metadata.branch.slug, + expected.branch.slug, + "Should have the slug prop" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this method"); + + sandbox.restore(); +}); + +add_task(async function test_getRolloutMetaData() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.rollout("foo"); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + let metadata = ExperimentAPI.getExperimentMetaData({ slug: expected.slug }); + + Assert.equal( + Object.keys(metadata.branch).length, + 1, + "Should only expose one property" + ); + Assert.equal( + metadata.branch.slug, + expected.branch.slug, + "Should have the slug prop" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this method"); + + sandbox.restore(); +}); + +add_task(function test_getExperimentMetaData_safe() { + const sandbox = sinon.createSandbox(); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + sandbox.stub(ExperimentAPI._store, "get").throws(); + sandbox.stub(ExperimentAPI._store, "getExperimentForFeature").throws(); + + try { + let metadata = ExperimentAPI.getExperimentMetaData({ slug: "foo" }); + Assert.equal(metadata, null, "Should not throw"); + } catch (e) { + Assert.ok(false, "Error should be caught in ExperimentAPI"); + } + + Assert.ok(ExperimentAPI._store.get.calledOnce, "Sanity check"); + + try { + let metadata = ExperimentAPI.getExperimentMetaData({ featureId: "foo" }); + Assert.equal(metadata, null, "Should not throw"); + } catch (e) { + Assert.ok(false, "Error should be caught in ExperimentAPI"); + } + + Assert.ok( + ExperimentAPI._store.getExperimentForFeature.calledOnce, + "Sanity check" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this feature"); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_feature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + features: [{ featureId: "cfr", value: null }], + feature: { + featureId: "unused-feature-id-for-legacy-support", + enabled: false, + value: {}, + }, + }, + }); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => ExperimentFakes.childStore()); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.store.addEnrollment(expected); + + // Wait to sync to child + await TestUtils.waitForCondition( + () => ExperimentAPI.getExperiment({ featureId: "cfr" }), + "Wait for child to sync" + ); + + Assert.equal( + ExperimentAPI.getExperiment({ featureId: "cfr" }).slug, + expected.slug, + "should return an experiment by featureId" + ); + + Assert.deepEqual( + ExperimentAPI.getExperiment({ featureId: "cfr" }).branch, + expected.branch, + "should return the right branch by featureId" + ); + + Assert.ok(exposureStub.notCalled, "Not called by default"); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_safe() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getExperimentForFeature").throws(); + + try { + Assert.equal( + ExperimentAPI.getExperiment({ featureId: "foo" }), + null, + "It should not fail even when it throws." + ); + } catch (e) { + Assert.ok(false, "Error should be caught by ExperimentAPI"); + } + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_featureAccess() { + const sandbox = sinon.createSandbox(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + value: { title: "hi" }, + features: [{ featureId: "cfr", value: { message: "content" } }], + }, + }); + const stub = sandbox + .stub(ExperimentAPI._store, "getExperimentForFeature") + .returns(expected); + + let { branch } = ExperimentAPI.getExperiment({ featureId: "cfr" }); + + Assert.equal(branch.slug, "treatment"); + let feature = branch.cfr; + Assert.ok(feature, "Should allow to access by featureId"); + Assert.equal(feature.value.message, "content"); + + stub.restore(); +}); + +add_task(async function test_getExperiment_featureAccess_backwardsCompat() { + const sandbox = sinon.createSandbox(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + feature: { featureId: "cfr", value: { message: "content" } }, + }, + }); + const stub = sandbox + .stub(ExperimentAPI._store, "getExperimentForFeature") + .returns(expected); + + let { branch } = ExperimentAPI.getExperiment({ featureId: "cfr" }); + + Assert.equal(branch.slug, "treatment"); + let feature = branch.cfr; + Assert.ok(feature, "Should allow to access by featureId"); + Assert.equal(feature.value.message, "content"); + + stub.restore(); +}); + +/** + * #getRecipe + */ +add_task(async function test_getRecipe() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + const collectionName = Services.prefs.getStringPref(COLLECTION_ID_PREF); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const recipe = await ExperimentAPI.getRecipe("foo"); + Assert.deepEqual( + recipe, + RECIPE, + "should return an experiment recipe if found" + ); + Assert.equal( + ExperimentAPI._remoteSettingsClient.collectionName, + collectionName, + "Loaded the expected collection" + ); + + sandbox.restore(); +}); + +add_task(async function test_getRecipe_Failure() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").throws(); + + const recipe = await ExperimentAPI.getRecipe("foo"); + Assert.equal(recipe, undefined, "should return undefined if RS throws"); + + sandbox.restore(); +}); + +/** + * #getAllBranches + */ +add_task(async function test_getAllBranches() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + + sandbox.restore(); +}); + +// API used by Messaging System +add_task(async function test_getAllBranches_featureIdAccessor() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + branches.forEach(branch => { + Assert.equal( + branch.testFeature.featureId, + "testFeature", + "Should use the experimentBranchAccessor proxy getter" + ); + }); + + sandbox.restore(); +}); + +// For schema version before 1.6.2 branch.feature was accessed +// instead of branch.features +add_task(async function test_getAllBranches_backwardsCompat() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + delete RECIPE.branches[0].features; + delete RECIPE.branches[1].features; + let feature = { + featureId: "backwardsCompat", + value: { + enabled: true, + }, + }; + RECIPE.branches[0].feature = feature; + RECIPE.branches[1].feature = feature; + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + branches.forEach(branch => { + Assert.equal( + branch.backwardsCompat.featureId, + "backwardsCompat", + "Should use the experimentBranchAccessor proxy getter" + ); + }); + + sandbox.restore(); +}); + +add_task(async function test_getAllBranches_Failure() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").throws(); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.equal(branches, undefined, "should return undefined if RS throws"); + + sandbox.restore(); +}); + +/** + * #on + * #off + */ +add_task(async function test_addEnrollment_eventEmit_add() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + ExperimentAPI.on("update", { slug: "foo" }, slugStub); + ExperimentAPI.on("update", { featureId: "purple" }, featureStub); + + await store.addEnrollment(experiment); + + Assert.equal( + slugStub.callCount, + 1, + "should call 'update' callback for slug when experiment is added" + ); + Assert.equal(slugStub.firstCall.args[1].slug, experiment.slug); + Assert.equal( + featureStub.callCount, + 1, + "should call 'update' callback for featureId when an experiment is added" + ); + Assert.equal(featureStub.firstCall.args[1].slug, experiment.slug); +}); + +add_task(async function test_updateExperiment_eventEmit_add_and_update() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + await store.addEnrollment(experiment); + + ExperimentAPI.on("update", { slug: "foo" }, slugStub); + ExperimentAPI.on("update", { featureId: "purple" }, featureStub); + + store.updateExperiment(experiment.slug, experiment); + + await TestUtils.waitForCondition( + () => slugStub.callCount == 2, + "Wait for `on` method to notify callback about the `add` event." + ); + // Called twice, once when attaching the event listener (because there is an + // existing experiment with that name) and 2nd time for the update event + Assert.equal(slugStub.firstCall.args[1].slug, experiment.slug); + Assert.equal(featureStub.callCount, 2, "Called twice for feature"); + Assert.equal(featureStub.firstCall.args[1].slug, experiment.slug); +}); + +add_task(async function test_updateExperiment_eventEmit_off() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + ExperimentAPI.on("update", { slug: "foo" }, slugStub); + ExperimentAPI.on("update", { featureId: "purple" }, featureStub); + + await store.addEnrollment(experiment); + + ExperimentAPI.off("update:foo", slugStub); + ExperimentAPI.off("update:purple", featureStub); + + store.updateExperiment(experiment.slug, experiment); + + Assert.equal(slugStub.callCount, 1, "Called only once before `off`"); + Assert.equal(featureStub.callCount, 1, "Called only once before `off`"); +}); + +add_task(async function test_getActiveBranch() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green", value: null }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + + Assert.deepEqual( + ExperimentAPI.getActiveBranch({ featureId: "green" }), + experiment.branch, + "Should return feature of active experiment" + ); + + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_safe() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getAllActive").throws(); + + try { + Assert.equal( + ExperimentAPI.getActiveBranch({ featureId: "green" }), + null, + "Should not throw" + ); + } catch (e) { + Assert.ok(false, "Should catch error in ExperimentAPI"); + } + + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_storeFailure() { + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + // Adding stub later because `addEnrollment` emits update events + const stub = sandbox.stub(store, "emit"); + // Call getActiveBranch to trigger an activation event + sandbox.stub(store, "getAllActive").throws(); + try { + ExperimentAPI.getActiveBranch({ featureId: "green" }); + } catch (e) { + /* This is expected */ + } + + Assert.equal(stub.callCount, 0, "Not called if store somehow fails"); + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_noActivationEvent() { + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + // Adding stub later because `addEnrollment` emits update events + const stub = sandbox.stub(store, "emit"); + // Call getActiveBranch to trigger an activation event + ExperimentAPI.getActiveBranch({ featureId: "green" }); + + Assert.equal(stub.callCount, 0, "Not called: sendExposureEvent is false"); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js new file mode 100644 index 0000000000..f0aa2be521 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js @@ -0,0 +1,318 @@ +"use strict"; + +const { + ExperimentAPI, + _ExperimentFeature: ExperimentFeature, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +function setDefaultBranch(pref, value) { + let branch = Services.prefs.getDefaultBranch(""); + branch.setStringPref(pref, value); +} + +const TEST_FALLBACK_PREF = "testprefbranch.config"; +const FAKE_FEATURE_MANIFEST = { + description: "Test feature", + exposureDescription: "Used in tests", + variables: { + enabled: { + type: "boolean", + fallbackPref: "testprefbranch.enabled", + }, + config: { + type: "json", + fallbackPref: TEST_FALLBACK_PREF, + }, + remoteValue: { + type: "boolean", + }, + test: { + type: "boolean", + }, + title: { + type: "string", + }, + }, +}; + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +add_task(async function test_ExperimentFeature_test_helper_ready() { + const { manager } = await setupForExperimentFeature(); + await manager.store.ready(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + + await ExperimentFakes.enrollWithRollout( + { + featureId: "foo", + value: { remoteValue: "mochitest", enabled: true }, + }, + { + manager, + } + ); + + Assert.equal( + featureInstance.getVariable("remoteValue"), + "mochitest", + "set by remote config" + ); +}); + +add_task(async function test_record_exposure_event() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + const getExperimentSpy = sandbox.spy(ExperimentAPI, "getExperimentMetaData"); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + featureInstance.recordExposureEvent(); + + Assert.ok( + exposureSpy.notCalled, + "should not emit an exposure event when no experiment is active" + ); + + // Check that there aren't any Glean exposure events yet + var exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + Assert.equal( + undefined, + exposureEvents, + "no Glean exposure events before exposure" + ); + + await manager.store.addEnrollment( + ExperimentFakes.experiment("blah", { + branch: { + slug: "treatment", + features: [ + { + featureId: "foo", + value: { enabled: false }, + }, + ], + }, + }) + ); + + featureInstance.recordExposureEvent(); + + Assert.ok( + exposureSpy.calledOnce, + "should emit an exposure event when there is an experiment" + ); + Assert.equal(getExperimentSpy.callCount, 2, "Should be called every time"); + + // Check that the Glean exposure event was recorded. + exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect only one event + Assert.equal(1, exposureEvents.length); + // And that one event matches the expected + Assert.equal( + "blah", + exposureEvents[0].extra.experiment, + "Glean.nimbusEvents.exposure recorded with correct experiment slug" + ); + Assert.equal( + "treatment", + exposureEvents[0].extra.branch, + "Glean.nimbusEvents.exposure recorded with correct branch slug" + ); + Assert.equal( + "foo", + exposureEvents[0].extra.feature_id, + "Glean.nimbusEvents.exposure recorded with correct feature id" + ); + + sandbox.restore(); +}); + +add_task(async function test_record_exposure_event_once() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.store.addEnrollment( + ExperimentFakes.experiment("blah", { + branch: { + slug: "treatment", + features: [ + { + featureId: "foo", + value: { enabled: false }, + }, + ], + }, + }) + ); + + featureInstance.recordExposureEvent({ once: true }); + featureInstance.recordExposureEvent({ once: true }); + featureInstance.recordExposureEvent({ once: true }); + + Assert.ok( + exposureSpy.calledOnce, + "Should emit a single exposure event when the once param is true." + ); + + // Check that the Glean exposure event was recorded. + let exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect only one event + Assert.equal(1, exposureEvents.length); + + sandbox.restore(); +}); + +add_task(async function test_allow_multiple_exposure_events() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + let doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "foo", + value: { enabled: false }, + }, + { manager } + ); + + featureInstance.recordExposureEvent(); + featureInstance.recordExposureEvent(); + featureInstance.recordExposureEvent(); + + Assert.ok(exposureSpy.called, "Should emit exposure event"); + Assert.equal( + exposureSpy.callCount, + 3, + "Should emit an exposure event for each function call" + ); + + // Check that the Glean exposure event was recorded. + let exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect 3 events + Assert.equal(3, exposureEvents.length); + + sandbox.restore(); + await doExperimentCleanup(); +}); + +add_task(async function test_onUpdate_before_store_ready() { + let sandbox = sinon.createSandbox(); + const feature = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const stub = sandbox.stub(); + const manager = ExperimentFakes.manager(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.stub(manager.store, "getAllActive").returns([ + ExperimentFakes.experiment("foo-experiment", { + branch: { + slug: "control", + features: [ + { + featureId: "foo", + value: null, + }, + ], + }, + }), + ]); + + // We register for updates before the store finished loading experiments + // from disk + feature.onUpdate(stub); + + await manager.onStartup(); + + Assert.ok( + stub.calledOnce, + "Called on startup after loading experiments from disk" + ); + Assert.equal( + stub.firstCall.args[1], + "feature-experiment-loaded", + "Called for the expected reason" + ); +}); + +add_task(async function test_ExperimentFeature_test_ready_late() { + const { manager, sandbox } = await setupForExperimentFeature(); + const stub = sandbox.stub(); + + const featureInstance = new ExperimentFeature( + "test-feature", + FAKE_FEATURE_MANIFEST + ); + + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "slug", + features: [ + { + featureId: featureInstance.featureId, + value: { + title: "hello", + enabled: true, + }, + }, + ], + }, + }); + + sandbox.stub(manager.store, "getAllRollouts").returns([rollout]); + + await manager.onStartup(); + + featureInstance.onUpdate(stub); + + await featureInstance.ready(); + + Assert.ok(stub.notCalled, "We register too late to catch any events"); + + setDefaultBranch(TEST_FALLBACK_PREF, JSON.stringify({ foo: true })); + + Assert.deepEqual( + featureInstance.getVariable("config"), + { foo: true }, + "Feature is ready even when initialized after store update" + ); + Assert.equal( + featureInstance.getVariable("title"), + "hello", + "Returns the NimbusTestUtils rollout default value" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js new file mode 100644 index 0000000000..b29d84c937 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js @@ -0,0 +1,251 @@ +"use strict"; + +const { + ExperimentAPI, + _ExperimentFeature: ExperimentFeature, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +const { cleanupStorePrefCache } = ExperimentFakes; + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +const FEATURE_ID = "aboutwelcome"; +const TEST_FALLBACK_PREF = "browser.aboutwelcome.screens"; +const FAKE_FEATURE_MANIFEST = { + variables: { + screens: { + type: "json", + fallbackPref: TEST_FALLBACK_PREF, + }, + source: { + type: "string", + }, + }, +}; + +add_task( + async function test_ExperimentFeature_getAllVariables_prefsOverDefaults() { + const { sandbox } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables().screens?.length, + undefined, + "pref is not set" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[]"); + + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the user pref value over the defaults" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + sandbox.restore(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_experimentOverPref() { + const { sandbox, manager } = await setupForExperimentFeature(); + const recipe = ExperimentFakes.experiment("awexperiment", { + branch: { + slug: "treatment", + features: [ + { + featureId: "aboutwelcome", + value: { screens: ["test-value"] }, + }, + ], + }, + }); + + await manager.store.addEnrollment(recipe); + + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.ok( + !!featureInstance.getAllVariables().screens, + "should return the AW experiment value" + ); + + Assert.equal( + featureInstance.getAllVariables().screens[0], + "test-value", + "should return the AW experiment value" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[]"); + Assert.equal( + featureInstance.getAllVariables().screens[0], + "test-value", + "should return the AW experiment value" + ); + + await ExperimentFakes.cleanupAll([recipe.slug], { manager }); + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the user pref value" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + sandbox.restore(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_experimentOverRemote() { + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + const recipe = ExperimentFakes.experiment("aw-experiment", { + branch: { + slug: "treatment", + features: [ + { + featureId: FEATURE_ID, + value: { screens: ["test-value"] }, + }, + ], + }, + }); + const rollout = ExperimentFakes.rollout("aw-rollout", { + branch: { + slug: "treatment", + features: [ + { featureId: FEATURE_ID, value: { screens: [], source: "rollout" } }, + ], + }, + }); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + const rolloutPromise = new Promise(resolve => + featureInstance.onUpdate((feature, reason) => { + if (reason === "rollout-updated") { + resolve(); + } + }) + ); + const experimentPromise = new Promise(resolve => + featureInstance.onUpdate((feature, reason) => { + if (reason === "experiment-updated") { + resolve(); + } + }) + ); + manager.store.addEnrollment(recipe); + manager.store.addEnrollment(rollout); + await rolloutPromise; + await experimentPromise; + + let allVariables = featureInstance.getAllVariables(); + + Assert.equal(allVariables.screens.length, 1, "Returns experiment value"); + Assert.ok(!allVariables.source, "Does not include rollout value"); + + await ExperimentFakes.cleanupAll([recipe.slug], { manager }); + cleanupStorePrefCache(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_rolloutOverPrefDefaults() { + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + const rollout = ExperimentFakes.rollout("foo-aw", { + branch: { + slug: "getAllVariables", + features: [{ featureId: FEATURE_ID, value: { screens: [] } }], + }, + }); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables().screens?.length, + undefined, + "Pref is not set" + ); + + const updatePromise = new Promise(resolve => + featureInstance.onUpdate(resolve) + ); + // Load remote defaults + manager.store.addEnrollment(rollout); + + // Wait for feature to load the rollout + await updatePromise; + + Assert.deepEqual( + featureInstance.getAllVariables().screens?.length, + 0, + "Should return the rollout value over the defaults" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[1,2,3]"); + + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the rollout value over the user pref" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + cleanupStorePrefCache(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_defaultValuesParam() { + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables({ defaultValues: { screens: null } }) + .screens, + null, + "should return defaultValues param over default pref settings" + ); + } +); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js new file mode 100644 index 0000000000..4fbbb3792f --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js @@ -0,0 +1,198 @@ +"use strict"; + +const { + ExperimentAPI, + _ExperimentFeature: ExperimentFeature, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +const FEATURE_ID = "testfeature1"; +// Note: this gets deleted at the end of tests +const TEST_PREF_BRANCH = "testfeature1."; +const TEST_VARIABLES = { + enabled: { + type: "boolean", + fallbackPref: `${TEST_PREF_BRANCH}enabled`, + }, + name: { + type: "string", + fallbackPref: `${TEST_PREF_BRANCH}name`, + }, + count: { + type: "int", + fallbackPref: `${TEST_PREF_BRANCH}count`, + }, + items: { + type: "json", + fallbackPref: `${TEST_PREF_BRANCH}items`, + }, +}; + +function createInstanceWithVariables(variables) { + return new ExperimentFeature(FEATURE_ID, { + variables, + }); +} + +add_task(async function test_ExperimentFeature_getFallbackPrefName() { + const instance = createInstanceWithVariables(TEST_VARIABLES); + + Assert.equal( + instance.getFallbackPrefName("enabled"), + "testfeature1.enabled", + "should return the fallback preference name" + ); +}); + +add_task(async function test_ExperimentFeature_getVariable_notRegistered() { + const instance = createInstanceWithVariables(TEST_VARIABLES); + + if (Cu.isInAutomation || AppConstants.NIGHTLY_BUILD) { + Assert.throws( + () => { + instance.getVariable("non_existant_variable"); + }, + /Nimbus: Warning - variable "non_existant_variable" is not defined in FeatureManifest\.js/, + "should throw in automation for variables not defined in the manifest" + ); + } else { + info("Won't throw when running in Beta and release candidates"); + } +}); + +add_task(async function test_ExperimentFeature_getVariable_noFallbackPref() { + const instance = createInstanceWithVariables({ + foo: { type: "json" }, + }); + + Assert.equal( + instance.getVariable("foo"), + undefined, + "should return undefined if no values are set and no fallback pref is defined" + ); +}); + +add_task(async function test_ExperimentFeature_getVariable_precedence() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const instance = createInstanceWithVariables(TEST_VARIABLES); + const prefName = TEST_VARIABLES.items.fallbackPref; + const rollout = ExperimentFakes.rollout(`${FEATURE_ID}-rollout`, { + branch: { + slug: "slug", + features: [ + { + featureId: FEATURE_ID, + value: { items: [4, 5, 6] }, + }, + ], + }, + }); + + Services.prefs.clearUserPref(prefName); + + Assert.equal( + instance.getVariable("items"), + undefined, + "should return undefined if the fallback pref is not set" + ); + + // Default pref values + Services.prefs.setStringPref(prefName, JSON.stringify([1, 2, 3])); + + Assert.deepEqual( + instance.getVariable("items"), + [1, 2, 3], + "should return the default pref value" + ); + + // Remote default values + await manager.store.addEnrollment(rollout); + + Assert.deepEqual( + instance.getVariable("items"), + [4, 5, 6], + "should return the remote default value over the default pref value" + ); + + // Experiment values + const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: FEATURE_ID, + value: { + items: [7, 8, 9], + }, + }, + { manager } + ); + + Assert.deepEqual( + instance.getVariable("items"), + [7, 8, 9], + "should return the experiment value over the remote value" + ); + + // Cleanup + Services.prefs.deleteBranch(TEST_PREF_BRANCH); + await doExperimentCleanup(); + sandbox.restore(); +}); + +add_task(async function test_ExperimentFeature_getVariable_partial_values() { + const { sandbox, manager } = await setupForExperimentFeature(); + const instance = createInstanceWithVariables(TEST_VARIABLES); + const rollout = ExperimentFakes.rollout(`${FEATURE_ID}-rollout`, { + branch: { + slug: "slug", + features: [ + { + featureId: FEATURE_ID, + value: { name: "abc" }, + }, + ], + }, + }); + + // Set up a pref value for .enabled, + // a remote value for .name, + // an experiment value for .items + Services.prefs.setBoolPref(TEST_VARIABLES.enabled.fallbackPref, true); + await manager.store.addEnrollment(rollout); + const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: FEATURE_ID, + value: {}, + }, + { manager } + ); + + Assert.equal( + instance.getVariable("enabled"), + true, + "should skip missing variables from remote defaults" + ); + + Assert.equal( + instance.getVariable("name"), + "abc", + "should skip missing variables from experiments" + ); + + // Cleanup + Services.prefs.getDefaultBranch("").deleteBranch(TEST_PREF_BRANCH); + Services.prefs.deleteBranch(TEST_PREF_BRANCH); + await doExperimentCleanup(); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js new file mode 100644 index 0000000000..7c4de4ec42 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js @@ -0,0 +1,294 @@ +"use strict"; + +const { + ExperimentAPI, + _ExperimentFeature: ExperimentFeature, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); + +const { JsonSchema } = ChromeUtils.importESModule( + "resource://gre/modules/JsonSchema.sys.mjs" +); + +Cu.importGlobalProperties(["fetch"]); + +XPCOMUtils.defineLazyGetter(this, "fetchSchema", () => { + return fetch("resource://nimbus/schemas/NimbusEnrollment.schema.json", { + credentials: "omit", + }).then(rsp => rsp.json()); +}); + +const NON_MATCHING_ROLLOUT = Object.freeze( + ExperimentFakes.rollout("non-matching-rollout", { + branch: { + slug: "slug", + features: [ + { + featureId: "aboutwelcome", + value: { skipFocus: false, enabled: false }, + }, + ], + }, + }) +); +const MATCHING_ROLLOUT = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + features: [ + { + featureId: "aboutwelcome", + value: { skipFocus: false, enabled: true }, + }, + ], + }, + }) +); + +const AW_FAKE_MANIFEST = { + description: "Different manifest with a special test variable", + isEarlyStartup: true, + variables: { + remoteValue: { + type: "boolean", + description: "Test value", + }, + mochitest: { + type: "boolean", + }, + enabled: { + type: "boolean", + }, + skipFocus: { + type: "boolean", + }, + }, +}; + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +add_task(async function validSchema() { + const validator = new JsonSchema.Validator(await fetchSchema, { + shortCircuit: false, + }); + + { + const result = validator.validate(NON_MATCHING_ROLLOUT); + Assert.ok(result.valid, JSON.stringify(result.errors, undefined, 2)); + } + { + const result = validator.validate(MATCHING_ROLLOUT); + Assert.ok(result.valid, JSON.stringify(result.errors, undefined, 2)); + } +}); + +add_task(async function readyCallAfterStore_with_remote_value() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + + Assert.ok(feature.getVariable("skipFocus"), "Feature is true by default"); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(!feature.getVariable("skipFocus"), "Loads value from store"); + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function has_sync_value_before_ready() { + let { manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome", AW_FAKE_MANIFEST); + + Assert.equal( + feature.getVariable("remoteValue"), + undefined, + "Feature is true by default" + ); + + Services.prefs.setStringPref( + "nimbus.syncdefaultsstore.aboutwelcome", + JSON.stringify({ + ...MATCHING_ROLLOUT, + branch: { feature: MATCHING_ROLLOUT.branch.features[0] }, + }) + ); + + Services.prefs.setBoolPref( + "nimbus.syncdefaultsstore.aboutwelcome.remoteValue", + true + ); + + Assert.equal(feature.getVariable("remoteValue"), true, "Sync load from pref"); + + manager.store._deleteForTests("aboutwelcome"); +}); + +add_task(async function update_remote_defaults_onUpdate() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + let stub = sandbox.stub(); + + feature.onUpdate(stub); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(stub.called, "update event called"); + Assert.equal(stub.callCount, 1, "Called once for remote configs"); + Assert.equal(stub.firstCall.args[1], "rollout-updated", "Correct reason"); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function test_features_over_feature() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + const rollout_features_and_feature = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + feature: { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + features: [ + { + featureId: "aboutwelcome", + value: { skipFocus: false, enabled: true }, + }, + ], + }, + }) + ); + const rollout_just_feature = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + feature: { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + }, + }) + ); + + await manager.store.addEnrollment(rollout_features_and_feature); + Assert.ok( + feature.getVariable("enabled"), + "Should read from the features property over feature" + ); + + manager.store._deleteForTests("aboutwelcome"); + manager.store._deleteForTests("matching-rollout"); + + await manager.store.addEnrollment(rollout_just_feature); + Assert.ok( + !feature.getVariable("enabled"), + "Should read from the feature property when features doesn't exist" + ); + + manager.store._deleteForTests("aboutwelcome"); + manager.store._deleteForTests("matching-rollout"); + sandbox.restore(); +}); + +add_task(async function update_remote_defaults_readyPromise() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + let stub = sandbox.stub(); + + feature.onUpdate(stub); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(stub.calledOnce, "Update called after enrollment processed."); + Assert.ok( + stub.calledWith("featureUpdate:aboutwelcome", "rollout-updated"), + "Update called after enrollment processed." + ); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function update_remote_defaults_enabled() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + + Assert.equal( + feature.getVariable("enabled"), + true, + "Feature is enabled by manifest.variables.enabled" + ); + + await manager.store.addEnrollment(NON_MATCHING_ROLLOUT); + + Assert.ok( + !feature.getVariable("enabled"), + "Feature is disabled by remote configuration" + ); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +// If the branch data returned from the store is not modified +// this test should not throw +add_task(async function test_getVariable_no_mutation() { + let { sandbox, manager } = await setupForExperimentFeature(); + sandbox.stub(manager.store, "getExperimentForFeature").returns( + Cu.cloneInto( + { + branch: { + features: [{ featureId: "aboutwelcome", value: { mochitest: true } }], + }, + }, + {}, + { deepFreeze: true } + ) + ); + let feature = new ExperimentFeature("aboutwelcome", AW_FAKE_MANIFEST); + + Assert.ok(feature.getVariable("mochitest"), "Got back the expected feature"); + + sandbox.restore(); +}); + +add_task(async function remote_isEarlyStartup_config() { + let { manager } = await setupForExperimentFeature(); + let rollout = ExperimentFakes.rollout("password-autocomplete", { + branch: { + slug: "remote-config-isEarlyStartup", + features: [ + { + featureId: "password-autocomplete", + enabled: true, + value: { remote: true }, + isEarlyStartup: true, + }, + ], + }, + }); + + await manager.onStartup(); + await manager.store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.prefHasUserValue( + "nimbus.syncdefaultsstore.password-autocomplete" + ), + "Configuration is marked early startup" + ); + + Services.prefs.clearUserPref( + "nimbus.syncdefaultsstore.password-autocomplete" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js new file mode 100644 index 0000000000..b7ac49e905 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js @@ -0,0 +1,41 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +const { FirstStartup } = ChromeUtils.importESModule( + "resource://gre/modules/FirstStartup.sys.mjs" +); + +add_task(async function test_createTargetingContext() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const rollout = ExperimentFakes.rollout("bar"); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([recipe]); + sandbox.stub(manager.store, "getAllRollouts").returns([rollout]); + + let context = manager.createTargetingContext(); + const activeSlugs = await context.activeExperiments; + const activeRollouts = await context.activeRollouts; + + Assert.ok(!context.isFirstStartup, "should not set the first startup flag"); + Assert.deepEqual( + activeSlugs, + ["foo"], + "should return slugs for all the active experiment" + ); + Assert.deepEqual( + activeRollouts, + ["bar"], + "should return slugs for all rollouts stored" + ); + + // Pretend to be in the first startup + FirstStartup._state = FirstStartup.IN_PROGRESS; + context = manager.createTargetingContext(); + + Assert.ok(context.isFirstStartup, "should set the first startup flag"); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js new file mode 100644 index 0000000000..b5310f7835 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js @@ -0,0 +1,984 @@ +"use strict"; + +const { NormandyTestUtils } = ChromeUtils.import( + "resource://testing-common/NormandyTestUtils.jsm" +); +const { Sampling } = ChromeUtils.import( + "resource://gre/modules/components-utils/Sampling.jsm" +); +const { ClientEnvironment } = ChromeUtils.import( + "resource://normandy/lib/ClientEnvironment.jsm" +); +const { cleanupStorePrefCache } = ExperimentFakes; + +const { ExperimentStore } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentStore.jsm" +); +const { TelemetryEnvironment } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryEnvironment.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.import( + "resource://normandy/lib/TelemetryEvents.jsm" +); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; + +const globalSandbox = sinon.createSandbox(); +globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); +globalSandbox.spy(TelemetryEvents, "sendEvent"); +registerCleanupFunction(() => { + globalSandbox.restore(); +}); + +async function assertEmptyStore(store) { + Assert.deepEqual( + store + .getAll() + .filter(e => e.active) + .map(e => e.slug), + [], + "Store should have no active enrollments" + ); + + Assert.deepEqual( + store + .getAll() + .filter(e => e.inactive) + .map(e => e.slug), + [], + "Store should have no inactive enrollments" + ); + + store._store.saveSoon(); + await store._store.finalize(); + await IOUtils.remove(store._store.path); +} + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +/** + * The normal case: Enrollment of a new experiment + */ +add_task(async function test_add_to_store() { + const manager = ExperimentFakes.manager(); + const recipe = ExperimentFakes.recipe("foo"); + const enrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + + await manager.onStartup(); + + await manager.enroll(recipe, "test_add_to_store"); + await enrollPromise; + const experiment = manager.store.get("foo"); + + Assert.ok(experiment, "should add an experiment with slug foo"); + Assert.ok( + recipe.branches.includes(experiment.branch), + "should choose a branch from the recipe.branches" + ); + Assert.equal(experiment.active, true, "should set .active = true"); + Assert.ok( + NormandyTestUtils.isUuid(experiment.enrollmentId), + "should add a valid enrollmentId" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_add_rollout_to_store() { + const manager = ExperimentFakes.manager(); + const recipe = { + ...ExperimentFakes.recipe("rollout-slug"), + branches: [ExperimentFakes.rollout("rollout").branch], + isRollout: true, + active: true, + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + }; + const enrollPromise = new Promise(resolve => + manager.store.on("update:rollout-slug", resolve) + ); + + await manager.onStartup(); + + await manager.enroll(recipe, "test_add_rollout_to_store"); + await enrollPromise; + const experiment = manager.store.get("rollout-slug"); + + Assert.ok(experiment, `Should add an experiment with slug ${recipe.slug}`); + Assert.ok( + recipe.branches.includes(experiment.branch), + "should choose a branch from the recipe.branches" + ); + Assert.equal(experiment.isRollout, true, "should have .isRollout"); + + manager.unenroll("rollout-slug", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task( + async function test_setExperimentActive_sendEnrollmentTelemetry_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const enrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + sandbox.spy(manager, "setExperimentActive"); + sandbox.spy(manager, "sendEnrollmentTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Ensure there is no experiment active with the id in FOG + Assert.equal( + undefined, + Services.fog.testGetExperimentData("foo"), + "no active experiment exists before enrollment" + ); + + // Check that there aren't any Glean enrollment events yet + var enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + Assert.equal( + undefined, + enrollmentEvents, + "no Glean enrollment events before enrollment" + ); + + await manager.enroll( + ExperimentFakes.recipe("foo"), + "test_setExperimentActive_sendEnrollmentTelemetry_called" + ); + await enrollPromise; + const experiment = manager.store.get("foo"); + + Assert.equal( + manager.setExperimentActive.calledWith(experiment), + true, + "should call setExperimentActive after an enrollment" + ); + + Assert.equal( + manager.sendEnrollmentTelemetry.calledWith(experiment), + true, + "should call sendEnrollmentTelemetry after an enrollment" + ); + + // Test Glean experiment API interaction + Assert.notEqual( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "Glean.setExperimentActive called with `foo` feature" + ); + + // Check that the Glean enrollment event was recorded. + enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + // We expect only one event + Assert.equal(1, enrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + enrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.enrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + enrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.enrollment recorded with correct branch slug" + ); + Assert.equal( + experiment.experimentType, + enrollmentEvents[0].extra.experiment_type, + "Glean.nimbusEvents.enrollment recorded with correct experiment type" + ); + Assert.equal( + experiment.enrollmentId, + enrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.enrollment recorded with correct enrollment id" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); + } +); + +add_task(async function test_setRolloutActive_sendEnrollmentTelemetry_called() { + globalSandbox.reset(); + globalSandbox.spy(TelemetryEnvironment, "setExperimentActive"); + globalSandbox.spy(TelemetryEvents.sendEvent); + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const rolloutRecipe = { + ...ExperimentFakes.recipe("rollout"), + branches: [ExperimentFakes.rollout("rollout").branch], + isRollout: true, + }; + const enrollPromise = new Promise(resolve => + manager.store.on("update:rollout", resolve) + ); + sandbox.spy(manager, "setExperimentActive"); + sandbox.spy(manager, "sendEnrollmentTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData("rollout"), + "no rollout active before enrollment" + ); + + // Check that there aren't any Glean enrollment events yet + var enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + Assert.equal( + undefined, + enrollmentEvents, + "no Glean enrollment events before enrollment" + ); + + let result = await manager.enroll( + rolloutRecipe, + "test_setRolloutActive_sendEnrollmentTelemetry_called" + ); + + await enrollPromise; + + const enrollment = manager.store.get("rollout"); + + Assert.ok(!!result && !!enrollment, "Enrollment was successful"); + + Assert.equal( + TelemetryEnvironment.setExperimentActive.called, + true, + "should call setExperimentActive" + ); + Assert.ok( + manager.setExperimentActive.calledWith(enrollment), + "Should call setExperimentActive with the rollout" + ); + Assert.equal( + manager.setExperimentActive.firstCall.args[0].experimentType, + "rollout", + "Should have the correct experimentType" + ); + Assert.equal( + manager.sendEnrollmentTelemetry.calledWith(enrollment), + true, + "should call sendEnrollmentTelemetry after an enrollment" + ); + Assert.ok( + TelemetryEvents.sendEvent.calledOnce, + "Should send out enrollment telemetry" + ); + Assert.ok( + TelemetryEvents.sendEvent.calledWith( + "enroll", + sinon.match.string, + enrollment.slug, + { + experimentType: "rollout", + branch: enrollment.branch.slug, + enrollmentId: enrollment.enrollmentId, + } + ), + "Should send telemetry with expected values" + ); + + // Test Glean experiment API interaction + Assert.equal( + enrollment.branch.slug, + Services.fog.testGetExperimentData(enrollment.slug).branch, + "Glean.setExperimentActive called with expected values" + ); + + // Check that the Glean enrollment event was recorded. + enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + // We expect only one event + Assert.equal(1, enrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + enrollment.slug, + enrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.enrollment recorded with correct experiment slug" + ); + Assert.equal( + enrollment.branch.slug, + enrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.enrollment recorded with correct branch slug" + ); + Assert.equal( + enrollment.experimentType, + enrollmentEvents[0].extra.experiment_type, + "Glean.nimbusEvents.enrollment recorded with correct experiment type" + ); + Assert.equal( + enrollment.enrollmentId, + enrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.enrollment recorded with correct enrollment id" + ); + + manager.unenroll("rollout", "test-cleanup"); + + await assertEmptyStore(manager.store); + + globalSandbox.restore(); +}); + +// /** +// * Failure cases: +// * - slug conflict +// * - group conflict +// */ + +add_task(async function test_failure_name_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // simulate adding a previouly enrolled experiment + await manager.store.addEnrollment(ExperimentFakes.experiment("foo")); + + await Assert.rejects( + manager.enroll(ExperimentFakes.recipe("foo"), "test_failure_name_conflict"), + /An experiment with the slug "foo" already exists/, + "should throw if a conflicting experiment exists" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + "foo", + "name-conflict" + ), + true, + "should send failure telemetry if a conflicting experiment exists" + ); + + // Check that the Glean enrollment event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + "foo", + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "name-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_failure_group_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // Two conflicting branches that both have the group "pink" + // These should not be allowed to exist simultaneously. + const existingBranch = { + slug: "treatment", + features: [{ featureId: "pink", value: {} }], + }; + const newBranch = { + slug: "treatment", + features: [{ featureId: "pink", value: {} }], + }; + + // simulate adding an experiment with a conflicting group "pink" + await manager.store.addEnrollment( + ExperimentFakes.experiment("foo", { + branch: existingBranch, + }) + ); + + // ensure .enroll chooses the special branch with the conflict + sandbox.stub(manager, "chooseBranch").returns(newBranch); + Assert.equal( + await manager.enroll( + ExperimentFakes.recipe("bar", { branches: [newBranch] }), + "test_failure_group_conflict" + ), + null, + "should not enroll if there is a feature conflict" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + "bar", + "feature-conflict" + ), + true, + "should send failure telemetry if a feature conflict exists" + ); + + // Check that the Glean enroll_failed event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that event matches the expected experiment and reason + Assert.equal( + "bar", + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "feature-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_rollout_failure_group_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const rollout = ExperimentFakes.rollout("rollout-enrollment"); + const recipe = { + ...ExperimentFakes.recipe("rollout-recipe"), + branches: [rollout.branch], + isRollout: true, + }; + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // simulate adding an experiment with a conflicting group "pink" + await manager.store.addEnrollment(rollout); + + Assert.equal( + await manager.enroll(recipe, "test_rollout_failure_group_conflict"), + null, + "should not enroll if there is a feature conflict" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + recipe.slug, + "feature-conflict" + ), + true, + "should send failure telemetry if a feature conflict exists" + ); + + // Check that the Glean enroll_failed event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that event matches the expected experiment and reason + Assert.equal( + recipe.slug, + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "feature-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("rollout-enrollment", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_rollout_experiment_no_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const experiment = ExperimentFakes.recipe("experiment"); + const rollout = ExperimentFakes.recipe("rollout", { isRollout: true }); + + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + await ExperimentFakes.enrollmentHelper(experiment, { + manager, + }).enrollmentPromise; + await ExperimentFakes.enrollmentHelper(rollout, { + manager, + }).enrollmentPromise; + + Assert.ok( + manager.store.get(experiment.slug).active, + "Enrolled in the experiment for the feature" + ); + + Assert.ok( + manager.store.get(rollout.slug).active, + "Enrolled in the rollout for the feature" + ); + + Assert.ok( + manager.sendFailureTelemetry.notCalled, + "Should send failure telemetry if a feature conflict exists" + ); + + // Check that there aren't any Glean enroll_failed events + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + await ExperimentFakes.cleanupAll([experiment.slug, rollout.slug], { + manager, + }); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_sampling_check() { + const manager = ExperimentFakes.manager(); + let recipe = ExperimentFakes.recipe("foo", { bucketConfig: null }); + const sandbox = sinon.createSandbox(); + sandbox.stub(Sampling, "bucketSample").resolves(true); + sandbox.replaceGetter(ClientEnvironment, "userId", () => 42); + + Assert.ok( + !manager.isInBucketAllocation(recipe.bucketConfig), + "fails for no bucket config" + ); + + recipe = ExperimentFakes.recipe("foo2", { + bucketConfig: { randomizationUnit: "foo" }, + }); + + Assert.ok( + !manager.isInBucketAllocation(recipe.bucketConfig), + "fails for unknown randomizationUnit" + ); + + recipe = ExperimentFakes.recipe("foo3"); + + const result = await manager.isInBucketAllocation(recipe.bucketConfig); + + Assert.equal( + Sampling.bucketSample.callCount, + 1, + "it should call bucketSample" + ); + Assert.ok(result, "result should be true"); + const { args } = Sampling.bucketSample.firstCall; + Assert.equal(args[0][0], 42, "called with expected randomization id"); + Assert.equal( + args[0][1], + recipe.bucketConfig.namespace, + "called with expected namespace" + ); + Assert.equal( + args[1], + recipe.bucketConfig.start, + "called with expected start" + ); + Assert.equal( + args[2], + recipe.bucketConfig.count, + "called with expected count" + ); + Assert.equal( + args[3], + recipe.bucketConfig.total, + "called with expected total" + ); + + await assertEmptyStore(manager.store); + + sandbox.reset(); +}); + +add_task(async function enroll_in_reference_aw_experiment() { + cleanupStorePrefCache(); + + let dir = Services.dirsvc.get("CurWorkD", Ci.nsIFile).path; + let src = PathUtils.join( + dir, + "reference_aboutwelcome_experiment_content.json" + ); + const content = await IOUtils.readJSON(src); + // Create two dummy branches with the content from disk + const branches = ["treatment-a", "treatment-b"].map(slug => ({ + slug, + ratio: 1, + features: [ + { value: { ...content, enabled: true }, featureId: "aboutwelcome" }, + ], + })); + let recipe = ExperimentFakes.recipe("reference-aw", { branches }); + // Ensure we get enrolled + recipe.bucketConfig.count = recipe.bucketConfig.total; + + const manager = ExperimentFakes.manager(); + const enrollPromise = new Promise(resolve => + manager.store.on("update:reference-aw", resolve) + ); + await manager.onStartup(); + await manager.enroll(recipe, "enroll_in_reference_aw_experiment"); + await enrollPromise; + + Assert.ok(manager.store.get("reference-aw"), "Successful onboarding"); + let prefValue = Services.prefs.getStringPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome` + ); + Assert.ok( + prefValue, + "aboutwelcome experiment enrollment should be stored to prefs" + ); + // In case some regression causes us to store a significant amount of data + // in prefs. + Assert.ok(prefValue.length < 3498, "Make sure we don't bloat the prefs"); + + manager.unenroll(recipe.slug, "enroll_in_reference_aw_experiment:cleanup"); + manager.store._deleteForTests("aboutwelcome"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_forceEnroll_cleanup() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const fooEnrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + const barEnrollPromise = new Promise(resolve => + manager.store.on("update:optin-bar", resolve) + ); + let unenrollStub = sandbox.spy(manager, "unenroll"); + let existingRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "treatment", + ratio: 1, + features: [{ featureId: "force-enrollment", value: {} }], + }, + ], + }); + let forcedRecipe = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "treatment", + ratio: 1, + features: [{ featureId: "force-enrollment", value: {} }], + }, + ], + }); + + await manager.onStartup(); + await manager.enroll(existingRecipe, "test_forceEnroll_cleanup"); + await fooEnrollPromise; + + let setExperimentActiveSpy = sandbox.spy(manager, "setExperimentActive"); + manager.forceEnroll(forcedRecipe, forcedRecipe.branches[0]); + await barEnrollPromise; + + Assert.ok(unenrollStub.called, "Unenrolled from existing experiment"); + Assert.equal( + unenrollStub.firstCall.args[0], + existingRecipe.slug, + "Called with existing recipe slug" + ); + Assert.ok(setExperimentActiveSpy.calledOnce, "Activated forced experiment"); + Assert.equal( + setExperimentActiveSpy.firstCall.args[0].slug, + `optin-${forcedRecipe.slug}`, + "Called with forced experiment slug" + ); + Assert.equal( + manager.store.getExperimentForFeature("force-enrollment").slug, + `optin-${forcedRecipe.slug}`, + "Enrolled in forced experiment" + ); + + manager.unenroll(`optin-${forcedRecipe.slug}`, "test-cleanup"); + + await assertEmptyStore(manager.store); + + sandbox.restore(); +}); + +add_task(async function test_rollout_unenroll_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + let unenrollStub = sandbox.stub(manager, "unenroll").returns(true); + let enrollStub = sandbox.stub(manager, "_enroll").returns(true); + let rollout = ExperimentFakes.rollout("rollout_conflict"); + + // We want to force a conflict + sandbox.stub(manager.store, "getRolloutForFeature").returns(rollout); + + manager.forceEnroll(rollout, rollout.branch); + + Assert.ok(unenrollStub.calledOnce, "Should unenroll the conflicting rollout"); + Assert.ok( + unenrollStub.calledWith(rollout.slug, "force-enrollment"), + "Should call with expected slug" + ); + Assert.ok(enrollStub.calledOnce, "Should call enroll as expected"); + + await assertEmptyStore(manager.store); + + sandbox.restore(); +}); + +add_task(async function test_forceEnroll() { + const experiment1 = ExperimentFakes.recipe("experiment-1"); + const experiment2 = ExperimentFakes.recipe("experiment-2"); + const rollout1 = ExperimentFakes.recipe("rollout-1", { isRollout: true }); + const rollout2 = ExperimentFakes.recipe("rollout-2", { isRollout: true }); + + const TEST_CASES = [ + { + enroll: [experiment1, rollout1], + expected: [experiment1, rollout1], + }, + { + enroll: [rollout1, experiment1], + expected: [experiment1, rollout1], + }, + { + enroll: [experiment1, experiment2], + expected: [experiment2], + }, + { + enroll: [rollout1, rollout2], + expected: [rollout2], + }, + { + enroll: [experiment1, rollout1, rollout2, experiment2], + expected: [experiment2, rollout2], + }, + ]; + + async function forceEnroll(manager, recipe) { + const enrollmentPromise = new Promise(resolve => { + manager.store.on(`update:optin-${recipe.slug}`, resolve); + }); + + manager.forceEnroll(recipe, recipe.branches[0]); + + return enrollmentPromise; + } + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([experiment1, experiment2, rollout1, rollout2]); + sinon.stub(loader, "setTimer"); + + await loader.init(); + await manager.onStartup(); + + for (const { enroll, expected } of TEST_CASES) { + for (const recipe of enroll) { + await forceEnroll(manager, recipe); + } + + const activeSlugs = manager.store + .getAll() + .filter(enrollment => enrollment.active) + .map(r => r.slug); + + Assert.equal( + activeSlugs.length, + expected.length, + `Should be enrolled in ${expected.length} experiments and rollouts` + ); + + for (const { slug, isRollout } of expected) { + Assert.ok( + activeSlugs.includes(`optin-${slug}`), + `Should be enrolled in ${ + isRollout ? "rollout" : "experiment" + } with slug optin-${slug}` + ); + } + + for (const { slug } of expected) { + manager.unenroll(`optin-${slug}`); + manager.store._deleteForTests(`optin-${slug}`); + } + } + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_featureIds_is_stored() { + Services.prefs.setStringPref("messaging-system.log", "all"); + const recipe = ExperimentFakes.recipe("featureIds"); + // Ensure we get enrolled + recipe.bucketConfig.count = recipe.bucketConfig.total; + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + const { + enrollmentPromise, + doExperimentCleanup, + } = ExperimentFakes.enrollmentHelper(recipe, { manager }); + + await enrollmentPromise; + + Assert.ok(manager.store.addEnrollment.calledOnce, "experiment is stored"); + let [enrollment] = manager.store.addEnrollment.firstCall.args; + Assert.ok("featureIds" in enrollment, "featureIds is stored"); + Assert.deepEqual( + enrollment.featureIds, + ["testFeature"], + "Has expected value" + ); + + await doExperimentCleanup(); + + await assertEmptyStore(manager.store); +}); + +add_task(async function experiment_and_rollout_enroll_and_cleanup() { + let store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + let rolloutCleanup = await ExperimentFakes.enrollWithRollout( + { + featureId: "aboutwelcome", + value: { enabled: true }, + }, + { + manager, + } + ); + + let experimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "aboutwelcome", + value: { enabled: true }, + }, + { manager } + ); + + Assert.ok( + Services.prefs.getBoolPref(`${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`) + ); + Assert.ok( + Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled` + ) + ); + + await experimentCleanup(); + + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + Assert.ok( + Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled` + ) + ); + + await rolloutCleanup(); + + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + + await assertEmptyStore(manager.store); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js new file mode 100644 index 0000000000..17b34dc199 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js @@ -0,0 +1,111 @@ +"use strict"; +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); + +const TEST_CONFIG = { + slug: "test-experiment", + branches: [ + { + slug: "control", + ratio: 1, + }, + { + slug: "branchA", + ratio: 1, + }, + { + slug: "branchB", + ratio: 1, + }, + ], + namespace: "test-namespace", + start: 0, + count: 2000, + total: 10000, +}; +add_task(async function test_generateTestIds() { + let result = await ExperimentManager.generateTestIds(TEST_CONFIG); + + Assert.ok(result, "should return object"); + Assert.ok(result.notInExperiment, "should have a id for no experiment"); + Assert.ok(result.control, "should have id for control"); + Assert.ok(result.branchA, "should have id for branchA"); + Assert.ok(result.branchB, "should have id for branchB"); +}); + +add_task(async function test_generateTestIds_input_errors() { + const { slug, branches, namespace, start, count, total } = TEST_CONFIG; + await Assert.rejects( + ExperimentManager.generateTestIds({ + branches, + namespace, + start, + count, + total, + }), + /slug, namespace not in expected format/, + "should throw because of missing slug" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ slug, branches, start, count, total }), + /slug, namespace not in expected format/, + "should throw because of missing namespace" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + count, + total, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing start" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + start, + total, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing count" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + count, + start, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing total" + ); + + // Intentionally misspelled slug + let invalidBranches = [ + { slug: "a", ratio: 1 }, + { slugG: "b", ratio: 1 }, + ]; + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches: invalidBranches, + namespace, + start, + count, + total, + }), + /branches parameter not in expected format/, + "should throw because of invalid format for branches" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js new file mode 100644 index 0000000000..f368286fc6 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js @@ -0,0 +1,419 @@ +"use strict"; + +const { Sampling } = ChromeUtils.import( + "resource://gre/modules/components-utils/Sampling.jsm" +); + +/** + * onStartup() + * - should set call setExperimentActive for each active experiment + */ +add_task(async function test_onStartup_setExperimentActive_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const experiments = []; + sandbox.stub(manager, "setExperimentActive"); + sandbox.stub(manager.store, "init").resolves(); + sandbox.stub(manager.store, "getAll").returns(experiments); + sandbox + .stub(manager.store, "get") + .callsFake(slug => experiments.find(expt => expt.slug === slug)); + sandbox.stub(manager.store, "set"); + + const active = ["foo", "bar"].map(ExperimentFakes.experiment); + + const inactive = ["baz", "qux"].map(slug => + ExperimentFakes.experiment(slug, { active: false }) + ); + + [...active, ...inactive].forEach(exp => experiments.push(exp)); + + await manager.onStartup(); + + active.forEach(exp => + Assert.equal( + manager.setExperimentActive.calledWith(exp), + true, + `should call setExperimentActive for active experiment: ${exp.slug}` + ) + ); + + inactive.forEach(exp => + Assert.equal( + manager.setExperimentActive.calledWith(exp), + false, + `should not call setExperimentActive for inactive experiment: ${exp.slug}` + ) + ); +}); + +add_task(async function test_onStartup_setRolloutActive_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.stub(manager, "setExperimentActive"); + sandbox.stub(manager.store, "init").resolves(); + + const active = ["foo", "bar"].map(ExperimentFakes.rollout); + sandbox.stub(manager.store, "getAll").returns(active); + sandbox + .stub(manager.store, "get") + .callsFake(slug => active.find(e => e.slug === slug)); + sandbox.stub(manager.store, "set"); + + await manager.onStartup(); + + active.forEach(r => + Assert.equal( + manager.setExperimentActive.calledWith(r), + true, + `should call setExperimentActive for rollout: ${r.slug}` + ) + ); +}); + +add_task(async function test_startup_unenroll() { + Services.prefs.setBoolPref("app.shield.optoutstudies.enabled", false); + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + let recipe = ExperimentFakes.experiment("startup_unenroll", { + experimentType: "unittest", + source: "test", + }); + // Test initializing ExperimentManager with an active + // recipe in the store. If the user has opted out it should + // unenroll. + await store.init(); + let enrollmentPromise = new Promise(resolve => + store.on(`update:${recipe.slug}`, resolve) + ); + store.addEnrollment(recipe); + await enrollmentPromise; + + const manager = ExperimentFakes.manager(store); + const unenrollSpy = sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + Assert.ok( + unenrollSpy.calledOnce, + "Unenrolled from active experiment if user opt out is true" + ); + Assert.ok( + unenrollSpy.calledWith("startup_unenroll", "studies-opt-out"), + "Called unenroll for expected recipe" + ); + + Services.prefs.clearUserPref("app.shield.optoutstudies.enabled"); +}); + +/** + * onRecipe() + * - should add recipe slug to .session[source] + * - should call .enroll() if the recipe hasn't been seen before; + * - should call .update() if the Enrollment already exists in the store; + * - should skip enrollment if recipe.isEnrollmentPaused is true + */ +add_task(async function test_onRecipe_track_slug() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + const fooRecipe = ExperimentFakes.recipe("foo"); + + await manager.onStartup(); + // The first time a recipe has seen; + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.sessions.get("test").has("foo"), + true, + "should add slug to sessions[test]" + ); +}); + +add_task(async function test_onRecipe_enroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + sandbox.stub(Sampling, "bucketSample").resolves(true); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + const fooRecipe = ExperimentFakes.recipe("foo"); + const experimentUpdate = new Promise(resolve => + manager.store.on(`update:${fooRecipe.slug}`, resolve) + ); + await manager.onStartup(); + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.enroll.calledWith(fooRecipe), + true, + "should call .enroll() the first time a recipe is seen" + ); + await experimentUpdate; + Assert.equal( + manager.store.has("foo"), + true, + "should add recipe to the store" + ); + + manager.unenroll(fooRecipe.slug, "test-cleanup"); +}); + +add_task(async function test_onRecipe_update() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + + const fooRecipe = ExperimentFakes.recipe("foo"); + const experimentUpdate = new Promise(resolve => + manager.store.on(`update:${fooRecipe.slug}`, resolve) + ); + + await manager.onStartup(); + await manager.onRecipe(fooRecipe, "test"); + // onRecipe calls enroll which saves the experiment in the store + // but none of them wait on disk operations to finish + await experimentUpdate; + // Call again after recipe has already been enrolled + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(fooRecipe), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + + manager.unenroll(fooRecipe.slug, "test-cleanup"); +}); + +add_task(async function test_onRecipe_rollout_update() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "unenroll"); + sandbox.spy(manager, "updateEnrollment"); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + + const fooRecipe = { + ...ExperimentFakes.recipe("foo"), + isRollout: true, + }; + // Rollouts should only have 1 branch + fooRecipe.branches = fooRecipe.branches.slice(0, 1); + const experimentUpdate = new Promise(resolve => + manager.store.on(`update:${fooRecipe.slug}`, resolve) + ); + + await manager.onStartup(); + await manager.onRecipe(fooRecipe, "test"); + // onRecipe calls enroll which saves the experiment in the store + // but none of them wait on disk operations to finish + await experimentUpdate; + // Call again after recipe has already been enrolled + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(fooRecipe), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + Assert.ok( + manager.updateEnrollment.alwaysReturned(true), + "updateEnrollment will confirm the enrolled branch still exists in the recipe and exit" + ); + Assert.ok( + manager.unenroll.notCalled, + "Should not call if the branches did not change" + ); + + // We call again but this time we change the branch slug + // Has to be a deep clone otherwise you're changing the + // value found in the experiment store + let recipeClone = Cu.cloneInto(fooRecipe, {}); + recipeClone.branches[0].slug = "control-v2"; + await manager.onRecipe(recipeClone, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(recipeClone), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + Assert.ok( + manager.unenroll.called, + "updateEnrollment will unenroll because the branch slug changed" + ); + Assert.ok( + manager.unenroll.calledWith(fooRecipe.slug, "branch-removed"), + "updateEnrollment will unenroll because the branch slug changed" + ); +}); + +add_task(async function test_onRecipe_isEnrollmentPaused() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + await manager.onStartup(); + + const pausedRecipe = ExperimentFakes.recipe("xyz", { + isEnrollmentPaused: true, + }); + await manager.onRecipe(pausedRecipe, "test"); + Assert.equal( + manager.enroll.calledWith(pausedRecipe), + false, + "should skip enrollment for recipes that are paused" + ); + Assert.equal( + manager.store.has("xyz"), + false, + "should not add recipe to the store" + ); + + const fooRecipe = ExperimentFakes.recipe("foo"); + const updatedRecipe = ExperimentFakes.recipe("foo", { + isEnrollmentPaused: true, + }); + let enrollmentPromise = new Promise(resolve => + manager.store.on(`update:${fooRecipe.slug}`, resolve) + ); + await manager.enroll(fooRecipe, "test"); + await enrollmentPromise; + await manager.onRecipe(updatedRecipe, "test"); + Assert.equal( + manager.updateEnrollment.calledWith(updatedRecipe), + true, + "should still update existing recipes, even if enrollment is paused" + ); +}); + +/** + * onFinalize() + * - should unenroll experiments that weren't seen in the current session + */ + +add_task(async function test_onFinalize_unenroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + // Add an experiment to the store without calling .onRecipe + // This simulates an enrollment having happened in the past. + let recipe0 = ExperimentFakes.experiment("foo", { + experimentType: "unittest", + userFacingName: "foo", + userFacingDescription: "foo", + lastSeen: new Date().toJSON(), + source: "test", + }); + await manager.store.addEnrollment(recipe0); + + const recipe1 = ExperimentFakes.recipe("bar"); + // Unique features to prevent overlap + recipe1.branches[0].features[0].featureId = "red"; + recipe1.branches[1].features[0].featureId = "red"; + await manager.onRecipe(recipe1, "test"); + const recipe2 = ExperimentFakes.recipe("baz"); + recipe2.branches[0].features[0].featureId = "green"; + recipe2.branches[1].features[0].featureId = "green"; + await manager.onRecipe(recipe2, "test"); + + // Finalize + manager.onFinalize("test"); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("foo", "recipe-not-seen"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); + Assert.equal( + manager.sessions.has("test"), + false, + "should clear sessions[test]" + ); +}); + +add_task(async function test_onFinalize_unenroll_mismatch() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + // Add an experiment to the store without calling .onRecipe + // This simulates an enrollment having happened in the past. + let recipe0 = ExperimentFakes.experiment("foo", { + experimentType: "unittest", + userFacingName: "foo", + userFacingDescription: "foo", + lastSeen: new Date().toJSON(), + source: "test", + }); + await manager.store.addEnrollment(recipe0); + + const recipe1 = ExperimentFakes.recipe("bar"); + // Unique features to prevent overlap + recipe1.branches[0].features[0].featureId = "red"; + recipe1.branches[1].features[0].featureId = "red"; + await manager.onRecipe(recipe1, "test"); + const recipe2 = ExperimentFakes.recipe("baz"); + recipe2.branches[0].features[0].featureId = "green"; + recipe2.branches[1].features[0].featureId = "green"; + await manager.onRecipe(recipe2, "test"); + + // Finalize + manager.onFinalize("test", { recipeMismatches: [recipe0.slug] }); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("foo", "targeting-mismatch"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); + Assert.equal( + manager.sessions.has("test"), + false, + "should clear sessions[test]" + ); +}); + +add_task(async function test_onFinalize_rollout_unenroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + let rollout = ExperimentFakes.rollout("rollout"); + await manager.store.addEnrollment(rollout); + + manager.onFinalize("NimbusTestUtils"); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("rollout", "recipe-not-seen"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js new file mode 100644 index 0000000000..7ae802fbb7 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js @@ -0,0 +1,3152 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { + _ExperimentFeature: ExperimentFeature, + NimbusFeatures, +} = ChromeUtils.import("resource://nimbus/ExperimentAPI.jsm"); + +const { PrefUtils } = ChromeUtils.import( + "resource://normandy/lib/PrefUtils.jsm" +); + +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.import( + "resource://normandy/lib/TelemetryEvents.jsm" +); + +/** + * Pick a single entry from an object and return a new object containing only + * that entry. + * + * @param {object} obj The object to copy the value from. + * @param {string} key The key whose value is to be copied. + * + * @returns {object} An object with the property `key` set to `obj[key]`. + */ +function pick(obj, key) { + return { [key]: obj[key] }; +} + +const PREF_FEATURES = [ + new ExperimentFeature("test-set-pref", { + description: "Test feature that sets a pref on the default branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup: false, + variables: { + foo: { + type: "string", + description: "Test variable", + setPref: "nimbus.test-only.foo", + }, + }, + }), + new ExperimentFeature("test-set-user-pref", { + description: "Test feature that sets a pref on the user branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup: true, + variables: { + bar: { + type: "string", + description: "Test variable", + setPref: "nimbus.test-only.bar", + }, + }, + }), +]; + +const DEFAULT_VALUE = "default-value"; +const USER_VALUE = "user-value"; +const EXPERIMENT_VALUE = "experiment-value"; +const ROLLOUT_VALUE = "rollout-value"; +const OVERWRITE_VALUE = "overwrite-value"; + +const USER = "user"; +const DEFAULT = "default"; +const ROLLOUT = "rollout"; +const EXPERIMENT = "experiment"; + +const PREFS = { + [DEFAULT]: "nimbus.test-only.foo", + [USER]: "nimbus.test-only.bar", +}; + +const FEATURE_IDS = { + [DEFAULT]: "test-set-pref", + [USER]: "test-set-user-pref", +}; + +const CONFIGS = { + [DEFAULT]: { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: { + foo: ROLLOUT_VALUE, + }, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: { + foo: EXPERIMENT_VALUE, + }, + }, + }, + [USER]: { + [ROLLOUT]: { + featureId: FEATURE_IDS[USER], + value: { + bar: ROLLOUT_VALUE, + }, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[USER], + value: { + bar: EXPERIMENT_VALUE, + }, + }, + }, +}; + +/** + * Set the given pref values on their respective branches (if they are not + * null). + */ +function setPrefs(pref, { defaultBranchValue = null, userBranchValue = null }) { + if (defaultBranchValue !== null) { + Services.prefs + .getDefaultBranch(null) + .setStringPref(pref, defaultBranchValue); + } + + if (userBranchValue !== null) { + Services.prefs.setStringPref(pref, userBranchValue); + } +} + +function assertExpectedPrefValues(pref, branch, expected, visible, msg) { + info(`Assert pref ${pref} on branch ${branch} matches ${expected} ${msg}`); + const hasBranchValue = expected !== null; + const hasVisibleValue = visible !== null; + + function hasValueMsg(hasValue) { + return `Expected pref "${pref}" on the ${branch} branch to${ + hasValue ? " " : " not " + }have a value ${msg}`; + } + + let prefBranch; + switch (branch) { + case USER: + Assert.equal( + Services.prefs.prefHasUserValue(pref), + hasBranchValue, + hasValueMsg(hasBranchValue) + ); + prefBranch = Services.prefs; + break; + + case DEFAULT: + Assert.equal( + Services.prefs.prefHasDefaultValue(pref), + hasBranchValue, + hasValueMsg(hasBranchValue) + ); + prefBranch = Services.prefs.getDefaultBranch(null); + break; + + default: + Assert.ok(false, "invalid pref branch"); + } + + if (hasBranchValue) { + Assert.equal( + prefBranch.getStringPref(pref), + expected, + `Expected pref "${pref} on the ${branch} branch to be ${JSON.stringify( + expected + )} ${msg}` + ); + } + + if (hasVisibleValue) { + Assert.equal( + Services.prefs.getStringPref(pref), + visible, + `Expected pref "${pref}" to be ${JSON.stringify(visible)} ${msg}` + ); + } else { + Assert.ok( + !Services.prefs.prefHasUserValue(pref) && + !Services.prefs.prefHasDefaultValue(pref), + `Expected pref "${pref} to not be set ${msg}` + ); + } +} + +/** + * Assert the store has no active experiments or rollouts. + */ +function assertEmptyStore(store) { + Assert.deepEqual( + store.getAllActive(), + [], + "There should be no experiments active." + ); + + Assert.deepEqual( + store.getAllRollouts(), + [], + "There should be no rollouts active" + ); +} + +/** + * Remove the experiment store. + */ +async function cleanupStore(store) { + // We need to call finalize first to ensure that any pending saves from + // JSONFile.saveSoon overwrite files on disk. + await store._store.finalize(); + await IOUtils.remove(store._store.path); +} + +/** + * Assert the manager has no active pref observers. + */ +function assertNoObservers(manager) { + Assert.equal( + manager._prefs.size, + 0, + "There should be no active pref observers" + ); + Assert.equal( + manager._prefsBySlug.size, + 0, + "There should be no active pref observers" + ); +} + +/** + * Remove all pref observers on the given ExperimentManager. + */ +function removePrefObservers(manager) { + for (const [name, entry] of manager._prefs.entries()) { + Services.prefs.removeObserver(name, entry.observer); + } + + manager._prefs.clear(); + manager._prefsBySlug.clear(); +} + +add_setup(function setup() { + do_get_profile(); + Services.fog.initializeFOG(); + + const cleanupFeatures = ExperimentTestUtils.addTestFeatures(...PREF_FEATURES); + registerCleanupFunction(cleanupFeatures); +}); + +add_task(async function test_enroll_setPref_rolloutsAndExperiments() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + assertEmptyStore(store); + + /** + * Test that prefs are set correctly before and after enrollment and + * unenrollment. + * + * @param {object} options + * @param {string} options.pref + * The name of the pref. + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {string[]} options.enrollOrder + * The order to do the enrollments. Must only contain + * "experiment" and "rollout" as values. + * + * @param {string[]} options.unenrollOrder + * The order to undo the enrollments. Must only contain + * "experiment" and "rollout" as values. + * + * @param {(string|null)[]} options.expectedValues + * The expected values of the preft on the given branch at each point: + * + * * before enrollment; + * * one entry each each after enrolling in `options.enrollOrder[i]`; and + * * one entry each each after unenrolling in `options.unenrollOrder[i]`. + * + * A value of null indicates that the pref should not be set on that + * branch. + * + * @param {(string|null)[]?} options.visibleValues + * The expected values returned by + * `Services.prefs.getStringPref` (i.e., the user branch if set, + * falling back to the default branch if not), in the same + * order as |options.expectedValues|. + * + * If undefined, then it will default `options.expectedValues`. + */ + async function doBaseTest({ + pref, + branch, + configs, + userBranchValue = undefined, + defaultBranchValue = undefined, + enrollOrder, + unenrollOrder, + expectedValues, + visibleValues = undefined, + }) { + if (visibleValues === undefined) { + visibleValues = expectedValues; + } + + const cleanupFns = {}; + let i = 0; + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + "before enrollment" + ); + i++; + + for (const enrollmentKind of enrollOrder) { + const isRollout = enrollmentKind === ROLLOUT; + cleanupFns[ + enrollmentKind + ] = await ExperimentFakes.enrollWithFeatureConfig( + configs[enrollmentKind], + { manager, isRollout } + ); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} enrollment` + ); + i++; + } + + for (const enrollmentKind of unenrollOrder) { + await cleanupFns[enrollmentKind](); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} unenrollment` + ); + i++; + } + + assertEmptyStore(store); + Services.prefs.deleteBranch(pref); + } + + // Tests for a feature that would set a pref on the default branch, but the variable is omitted. + { + const branch = DEFAULT; + const pref = PREFS[branch]; + + const configs = { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + }; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in a rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in an experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + // Test for a feature that would set a pref on the user branch, but the variable is omitted. + { + const branch = USER; + const pref = PREFS[branch]; + + const configs = { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + }; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in a rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + visibleValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in an experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + visibleValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + // Tests for a feature that sets prefs on the default branch. + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in rollout then experiment; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in rollout then experiment; unenroll in same order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in same order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + } + + // Tests for a feature that sets prefs on the user branch. + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, null], + visibleValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + // Enroll in experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, null], + visibleValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + // Enroll in rollout then experiment; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in rollout then experiment; unenroll in same order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in same order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + } + + await cleanupStore(store); +}); + +add_task(async function test_restorePrefs_experimentAndRollout() { + /** + * Test that prefs are set correctly after restoring from a saved store file + * and unrnollment. + * + * This test sets up some enrollments and saves them to disk. + * + * A browser restart will be simulated by creating a new ExperimentStore and + * ExperimentManager to restore the saved enrollments. + * + * @param {object} options + * @param {string} options.pref + * The name of the pref. + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {string[]} options.unenrollOrder + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {(string|null)[]} options.expectedValues + * The expected values of the preft on the given branch at each point: + * + * * before enrollment; + * * one entry each each after enrolling in `options.enrollOrder[i]`; and + * * one entry each each after unenrolling in `options.unenrollOrder[i]`. + * + * A value of null indicates that the pref should not be set on that + * branch. + * + * @param {(string|null)[]?} options.visibleValues + * The expected values returned by + * Services.prefs.getStringPref (i.e., the user branch if set, + * falling back to the default branch if not), in the same + * order as `options.expectedValues`. + * + * If undefined, then it will default to `options.expectedValues`. + */ + async function doBaseTest({ + featureId, + pref, + branch, + configs, + defaultBranchValue = null, + userBranchValue = null, + unenrollOrder, + expectedValues, + visibleValues = undefined, + }) { + if (![USER, DEFAULT].includes(branch)) { + Assert.ok(false, `invalid branch ${branch}`); + } + + if (visibleValues === undefined) { + visibleValues = expectedValues; + } + + // Set the initial conditions. + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + // Enroll in some experiments and save the state to disk. + { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + assertEmptyStore(store); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout: enrollmentKind === ROLLOUT, + }); + } + + store._store.saveSoon(); + await store._store.finalize(); + + // User branch prefs persist through restart, so we only want to delete + // the prefs if we changed the default branch. + if (branch === "default") { + Services.prefs.deleteBranch(pref); + } + + removePrefObservers(manager); + assertNoObservers(manager); + } + + // Restore the default branch value as it was before "restarting". + setPrefs(pref, { defaultBranchValue }); + // If this is not a user branch pref, restore the user branch value. User + // branch values persist through restart, so we don't want to overwrite a + // value we just set. + if (branch === "default") { + setPrefs(pref, { userBranchValue }); + } + + const sandbox = sinon.createSandbox(); + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + const setPrefSpy = sandbox.spy(PrefUtils, "setPref"); + + await manager.onStartup(); + + if (branch === DEFAULT) { + Assert.ok(setPrefSpy.calledOnce, "Should have called setPref once total"); + Assert.ok( + setPrefSpy.calledOnceWith(pref, expectedValues[0], { branch }), + `Should have only called setPref with correct args (called with: ${JSON.stringify( + setPrefSpy.getCall(0).args + )}) expected ${JSON.stringify([pref, expectedValues[0], { branch }])})` + ); + } else if (branch === USER) { + Assert.ok( + setPrefSpy.notCalled, + "Should have not called setPref for a user branch pref" + ); + } + + assertExpectedPrefValues( + pref, + branch, + expectedValues[0], + visibleValues[0], + "after manager startup" + ); + + const slugs = { + [ROLLOUT]: store.getRolloutForFeature(featureId)?.slug, + [EXPERIMENT]: store.getExperimentForFeature(featureId)?.slug, + }; + + let i = 1; + for (const enrollmentKind of unenrollOrder) { + manager.unenroll(slugs[enrollmentKind]); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} unenrollment` + ); + + i++; + } + + for (const enrollmentKind of unenrollOrder) { + // The unenrollment happened normally, not through a cleanup function. + store._deleteForTests(slugs[enrollmentKind]); + } + + assertNoObservers(manager); + assertEmptyStore(store); + await cleanupStore(store); + + Services.prefs.deleteBranch(pref); + sandbox.restore(); + } + + { + const branch = DEFAULT; + const featureId = FEATURE_IDS[branch]; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ featureId, pref, branch, ...args }); + + // Tests with no prefs set beforehand. + await doTest({ + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + // Tests where the default branch is set beforehand. + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + // Tests where the user branch is set beforehand. + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Tests with both branches set beforehand + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + { + const branch = USER; + const featureId = FEATURE_IDS[branch]; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => + doBaseTest({ featureId, pref, branch, configs, ...args }); + + // Tests with no prefs set beforehand. + await doTest({ + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, null], + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, null], + }); + + await doTest({ + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, null], + }); + + await doTest({ + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, null], + }); + + // Tests with the default branch set beforehand. + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, null], + visibleValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + // Tests with the user branch set beforehand. + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + // Tests with both branches set beforehand + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + } +}); + +add_task(async function test_prefChange() { + /** + * Test that pref tampering causes unenrollment. + * + * This test sets up some enrollments and then modifies the given `pref` on a + * branch specified by `setBranch` and checks that unenrollments happen as + * appropriate. + * + * @param {object} options + * + * @param {string} options.pref + * The name of the pref. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string} options.setBranch + * The branch that the test will set (either "user" or "default"). + * + * @param {string[]} options.expectedEnrollments + * The list of enrollment kinds (e.g., "rollout" or "experiment") that + * should be active after setting the pref on the requested branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after setting the pref on + * the requested branch. + * + * A value of null indicates that the pref should not be set on the + * default branch. + * + * @param {string} options.expectedUser + * The expected value of the user branch after setting the pref on the + * requested branch. + * + * A value of null indicates that the pref should not be set on the + * user branch. + */ + async function doBaseTest({ + pref, + defaultBranchValue = null, + userBranchValue = null, + configs, + setBranch, + expectedEnrollments = [], + expectedDefault = null, + expectedUser = null, + }) { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + const cleanup = {}; + const slugs = {}; + + await manager.onStartup(); + + assertEmptyStore(store); + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + info(`Enrolling in ${Array.from(Object.keys(configs)).join(", ")} ...`); + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + cleanup[enrollmentKind] = await ExperimentFakes.enrollWithFeatureConfig( + config, + { + manager, + isRollout, + } + ); + + const enrollments = isRollout + ? store.getAllRollouts() + : store.getAllActive(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + info( + `Overwriting ${pref} with "${OVERWRITE_VALUE}" on ${setBranch} branch` + ); + + PrefUtils.setPref(pref, OVERWRITE_VALUE, { branch: setBranch }); + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + if (expectedUser === null) { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getStringPref(pref), + expectedUser, + `Expected the value of ${pref} on the user branch to match the expected value` + ); + } + + for (const enrollmentKind of expectedEnrollments) { + const enrollment = store.get(slugs[enrollmentKind]); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(enrollment.active, "It should still be active"); + } + + for (const enrollmentKind of Object.keys(configs)) { + if (!expectedEnrollments.includes(enrollmentKind)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(!enrollment.active, "It should not be active"); + + store._deleteForTests(slug); + } + } + + for (const enrollmentKind of expectedEnrollments) { + await cleanup[enrollmentKind](); + } + + assertNoObservers(manager); + assertEmptyStore(store); + await cleanupStore(store); + + Services.prefs.deleteBranch(pref); + } + + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enrolled in rollout, set default branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enrolled in rollout, set user branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: USER, + expectedDefault: ROLLOUT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: ROLLOUT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in experiment, set default branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enrolled in experiment, set user branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enroll in both, set default branch. + await doTest({ + configs, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enroll in both, set user branch. + await doTest({ + configs, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + } + + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enrolled in rollout, set default branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + // Enrolled in rollout, set user branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in experiment, set default branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + // Enrolled in experiment, set user branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in both, set default branch. + await doTest({ + configs, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + // Enrolled in both, set user branch. + await doTest({ + configs, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + } +}); + +add_task(async function test_deleteBranch() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + assertEmptyStore(store); + + const cleanup = []; + cleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[USER][EXPERIMENT], { + manager, + }), + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[USER][ROLLOUT], { + manager, + isRollout: true, + }), + await ExperimentFakes.enrollWithFeatureConfig( + CONFIGS[DEFAULT][EXPERIMENT], + { manager } + ), + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[DEFAULT][ROLLOUT], { + manager, + isRollout: true, + }) + ); + + Services.prefs.deleteBranch(PREFS[USER]); + Services.prefs.deleteBranch(PREFS[DEFAULT]); + + // deleteBranch does not trigger pref observers! + Assert.equal( + store.getAll().length, + 4, + "nsIPrefBranch::deleteBranch does not trigger unenrollment" + ); + + for (const cleanupFn of cleanup) { + await cleanupFn(); + } + + assertNoObservers(manager); + assertEmptyStore(store); + await cleanupStore(store); +}); + +add_task(async function test_clearUserPref() { + /** + * Test that nsIPrefBranch::clearUserPref() correctly interacts with pref + * tampering logic. + * + * This test sets up some enrollments and then clears the pref specified and + * checks that unenrollments happen as * appropriate. + * + * @param {object} options + * + * @param {string} options.pref + * The name of the pref. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {boolean} options.expectedEnrolled + * Whether or not the enrollments defined in `configs` should still be + * active after clearing the user branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after clearing the user branch. + * + * A value of null indicates that the pref should not be set on the default + * branch. + */ + async function doBaseTest({ + pref, + defaultBranchValue = null, + userBranchValue = null, + configs, + expectedEnrolled, + expectedDefault = null, + }) { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + assertEmptyStore(store); + + const cleanup = []; + const slugs = {}; + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + cleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout, + }) + ); + + const enrollments = isRollout + ? store.getAllRollouts() + : store.getAllActive(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + Services.prefs.clearUserPref(pref); + + for (const enrollmentKind of Object.keys(configs)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + + if (expectedEnrolled) { + Assert.ok(enrollment.active, "It should be active"); + } else { + Assert.ok(!enrollment.active, "It should not be active"); + } + } + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref}` + ); + + if (expectedEnrolled) { + for (const cleanupFn of Object.values(cleanup)) { + await cleanupFn(); + } + } else { + for (const slug of Object.values(slugs)) { + store._deleteForTests(slug); + } + } + + assertNoObservers(manager); + assertEmptyStore(store); + await cleanupStore(store); + + Services.prefs.deleteBranch(pref); + } + + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout. + await doTest({ + configs: pick(configs, ROLLOUT), + expectedEnrolled: true, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + + // Enroll in experiment. + await doTest({ + configs: pick(configs, EXPERIMENT), + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + + // Enroll in both. + await doTest({ + configs, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + } + + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + const doTest = args => + doBaseTest({ pref, branch, expectedEnrolled: false, ...args }); + + // Because this pref is set on the user branch, clearing the user branch has + // the same effect for every suite of configs. + for (const selectedConfig of [ + pick(configs, ROLLOUT), + pick(configs, EXPERIMENT), + configs, + ]) { + await doTest({ + configs: selectedConfig, + }); + + await doTest({ + configs: selectedConfig, + defaultBranchValue: DEFAULT_VALUE, + expectedDefault: DEFAULT_VALUE, + }); + + await doTest({ + configs: selectedConfig, + userBranchValue: USER_VALUE, + }); + + await doTest({ + configs: selectedConfig, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedDefault: DEFAULT_VALUE, + }); + } + } +}); + +// Test that unenrollment doesn't happen if a pref changes but it wasn't set. +add_task(async function test_prefChanged_noPrefSet() { + const featureId = "test-set-pref-2"; + const pref = "nimbus.test-only.baz"; + + function featureFactory(isEarlyStartup) { + return new ExperimentFeature(featureId, { + description: "Test feature that sets a pref", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup, + variables: { + baz: { + type: "string", + description: "Test variable", + setPref: pref, + }, + qux: { + type: "string", + description: "Test variable", + }, + }, + }); + } + + const config = { + featureId, + value: { + qux: "qux", + }, + }; + + for (const isEarlyStartup of [true, false]) { + const feature = featureFactory(isEarlyStartup); + const cleanupFeature = ExperimentTestUtils.addTestFeatures(feature); + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + await manager.onStartup(); + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const isRollout of [true, false]) { + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + const doEnrollmentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + config, + { manager, isRollout } + ); + + PrefUtils.setPref(pref, OVERWRITE_VALUE, { branch }); + + const enrollments = await store.getAll(); + Assert.equal( + enrollments.length, + 1, + "There should be one enrollment" + ); + Assert.ok(enrollments[0].active, "The enrollment should be active"); + + Assert.equal( + PrefUtils.getPref(pref, { branch }), + OVERWRITE_VALUE, + `The value of ${pref} on the ${branch} branch should be the expected value` + ); + + if (branch === USER) { + if (defaultBranchValue) { + Assert.equal( + PrefUtils.getPref(pref, { branch: DEFAULT }), + defaultBranchValue, + "The default branch should have the expected value" + ); + } else { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + "The default branch should not have a value" + ); + } + } else if (userBranchValue) { + Assert.equal( + PrefUtils.getPref(pref, { branch: USER }), + userBranchValue, + "The user branch should have the expected value" + ); + } else { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + "The user branch should not have a value" + ); + } + + assertNoObservers(manager); + + await doEnrollmentCleanup(); + assertEmptyStore(store); + + Services.prefs.deleteBranch(pref); + } + } + } + } + + cleanupFeature(); + await cleanupStore(store); + } +}); + +add_task(async function test_restorePrefs_manifestChanged() { + TelemetryEvents.init(); + + const LEGACY_FILTER = { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + }; + + const BOGUS_PREF = "nimbus.test-only.bogus"; + + const REMOVE_FEATURE = "remove-feature"; + const REMOVE_PREF_VARIABLE = "remove-pref-variable"; + const REMOVE_OTHER_VARIABLE = "remove-other-variable"; + const REMOVE_SETPREF = "remove-setpref"; + const CHANGE_SETPREF = "change-setpref"; + + const OPERATIONS = [ + REMOVE_FEATURE, + REMOVE_PREF_VARIABLE, + REMOVE_OTHER_VARIABLE, + REMOVE_SETPREF, + CHANGE_SETPREF, + ]; + + const REASONS = { + [REMOVE_FEATURE]: "invalid-feature", + [REMOVE_PREF_VARIABLE]: "pref-variable-missing", + [REMOVE_SETPREF]: "pref-variable-no-longer", + [CHANGE_SETPREF]: "pref-variable-changed", + }; + + const featureId = "test-set-pref-temp"; + const pref = "nimbus.test-only.baz"; + + // Return a new object so we can modified the returned value. + function featureFactory(isEarlyStartup) { + return new ExperimentFeature(featureId, { + description: "Test feature that sets a pref on the default branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup, + variables: { + baz: { + type: "string", + description: "Test variable", + setPref: pref, + }, + qux: { + type: "string", + description: "Test variable", + }, + }, + }); + } + + /* + * Test that enrollments end when the manifest is sufficiently changed and + * that the appropriate telemetry is submitted. + * + * This test sets up some enrollments and saves them to disk. Then the + * manifest will be modified according to `operation`. + * + * A browser restart will be simulated by creating a new ExperimentStore and + * ExperimentManager to restore the saved enrollments. + * + * @param {object} options + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string} options.operation + * The operation that will be performed on the manifest. + * + * See `OPERATIONS` above. + * + * @param {string[]} options.expectedEnrollments + * The list of enrollment kinds (e.g., "rollout" or "experiment") that + * should be active after setting the pref on the requested branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after restoring enrollments. + * + * A value of null indicates that the pref should not be set on the + * default branch. + * + * @param {string} options.expectedUser + * The expected value of the user branch after restoring enrollments. + * + * A value of null indicates that the pref should not be set on the + * user branch. + */ + async function doBaseTest({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + expectedEnrollments = [], + expectedDefault = null, + expectedUser = null, + }) { + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); + + const feature = featureFactory(branch === USER); + const cleanupFeatures = ExperimentTestUtils.addTestFeatures(feature); + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + const slugs = {}; + let userPref = null; + + // Enroll in some experiments and save the state to disk. + { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + assertEmptyStore(store); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout, + }); + + const enrollments = isRollout + ? store.getAllRollouts() + : store.getAllActive(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + store._store.saveSoon(); + await store._store.finalize(); + + // User branch prefs persist through restart, so we only want to delete + // the prefs if we changed the default branch. + if (branch === "user") { + userPref = PrefUtils.getPref(pref, { branch }); + } + + Services.prefs.deleteBranch(pref); + + removePrefObservers(manager); + assertNoObservers(manager); + } + + // Restore the default branch value as it was before "restarting". + setPrefs(pref, { + defaultBranchValue, + userBranchValue: userPref ?? userBranchValue, + }); + + // Mangle the manifest. + switch (operation) { + case REMOVE_FEATURE: + cleanupFeatures(); + break; + + case REMOVE_PREF_VARIABLE: + delete NimbusFeatures[featureId].manifest.variables.baz; + break; + + case REMOVE_OTHER_VARIABLE: + delete NimbusFeatures[featureId].manifest.variables.qux; + break; + + case REMOVE_SETPREF: + delete NimbusFeatures[featureId].manifest.variables.baz.setPref; + break; + + case CHANGE_SETPREF: + NimbusFeatures[featureId].manifest.variables.baz.setPref = BOGUS_PREF; + break; + + default: + Assert.ok(false, "invalid operation"); + } + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + for (const enrollmentKind of expectedEnrollments) { + const enrollment = store.get(slugs[enrollmentKind]); + + Assert.ok( + enrollment !== null, + `An experiment of kind ${enrollmentKind} should exist` + ); + Assert.ok(enrollment.active, "It should still be active"); + } + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref} value: ${PrefUtils.getPref( + pref, + { branch: "default" } + )}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + if (expectedUser === null) { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref} value: ${PrefUtils.getPref( + pref, + { branch: "user" } + )}` + ); + } else { + Assert.equal( + Services.prefs.getStringPref(pref), + expectedUser, + `Expected the value of ${pref} on the user branch to match the expected value` + ); + } + + if (operation === CHANGE_SETPREF) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(BOGUS_PREF), + "The new pref should not have a value on the default branch" + ); + Assert.ok( + !Services.prefs.prefHasUserValue(BOGUS_PREF), + "The new pref should not have a value on the user branch" + ); + } + + for (const enrollmentKind of Object.keys(configs)) { + if (!expectedEnrollments.includes(enrollmentKind)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(!enrollment.active, "It should not be active"); + + store._deleteForTests(slug); + } + } + + const gleanEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + if (expectedEnrollments.length === 0) { + const expectedEvents = [EXPERIMENT, ROLLOUT] + .filter(enrollmentKind => Object.hasOwn(slugs, enrollmentKind)) + .map(enrollmentKind => ({ + reason: REASONS[operation], + experiment: slugs[enrollmentKind], + })); + + // Extract only the values we care about. + const processedEvents = gleanEvents.map(event => ({ + reason: event.extra.reason, + experiment: event.extra.experiment, + })); + + Assert.deepEqual( + processedEvents, + expectedEvents, + "Glean should have the expected unenrollment events" + ); + + const expectedLegacyEvents = expectedEvents.map(extra => ({ + value: extra.experiment, + extra: pick(extra, "reason"), + })); + + TelemetryTestUtils.assertEvents(expectedLegacyEvents, LEGACY_FILTER); + } else { + Assert.equal( + gleanEvents, + undefined, + "Glean should have no unenrollment events" + ); + + TelemetryTestUtils.assertEvents([], LEGACY_FILTER); + } + + for (const enrollmentKind of expectedEnrollments) { + const slug = slugs[enrollmentKind]; + manager.unenroll(slug); + store._deleteForTests(slug); + } + + assertEmptyStore(store); + await cleanupStore(store); + + assertNoObservers(manager); + Services.prefs.deleteBranch(pref); + + if (operation !== REMOVE_FEATURE) { + // If we try to remove the feature twice, we will throw an exception. + cleanupFeatures(); + } + } + + // Test only qux set. These tests should not cause any unenrollments. + { + const quxConfigs = { + [EXPERIMENT]: { + featureId, + value: { + qux: EXPERIMENT_VALUE, + }, + }, + [ROLLOUT]: { + featureId, + value: { + qux: ROLLOUT_VALUE, + }, + }, + }; + + const doTest = ({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + }) => + doBaseTest({ + branch, + configs, + defaultBranchValue, + userBranchValue, + operation, + expectedEnrollments: Object.keys(configs), + expectedDefault: defaultBranchValue, + expectedUser: userBranchValue, + }); + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const specifiedConfigs of [ + pick(quxConfigs, ROLLOUT), + pick(quxConfigs, EXPERIMENT), + quxConfigs, + ]) { + for (const operation of OPERATIONS) { + await doTest({ + branch, + defaultBranchValue, + userBranchValue, + configs: specifiedConfigs, + operation, + }); + } + } + } + } + } + } + + // Test only baz set. All operations except REMOVE_OTHER_VARIABLE will trigger + // unenrollment. + { + const bazConfigs = { + [EXPERIMENT]: { + featureId, + value: { + baz: EXPERIMENT_VALUE, + }, + }, + [ROLLOUT]: { + featureId, + value: { + baz: ROLLOUT_VALUE, + }, + }, + }; + + const doTest = ({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + }) => { + const expectedEnrollments = + operation === REMOVE_OTHER_VARIABLE ? Object.keys(configs) : []; + + function expectedPref(forBranch, originalValue) { + if (forBranch === branch) { + if (expectedEnrollments.includes(EXPERIMENT)) { + return EXPERIMENT_VALUE; + } else if (expectedEnrollments.includes(ROLLOUT)) { + return ROLLOUT_VALUE; + } + } + return originalValue; + } + + const expectedDefault = expectedPref(DEFAULT, defaultBranchValue); + const expectedUser = expectedPref(USER, userBranchValue); + + return doBaseTest({ + branch, + configs, + defaultBranchValue, + userBranchValue, + operation, + expectedEnrollments, + expectedDefault, + expectedUser, + }); + }; + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const specifiedConfigs of [ + pick(bazConfigs, ROLLOUT), + pick(bazConfigs, EXPERIMENT), + bazConfigs, + ]) { + for (const operation of OPERATIONS) { + await doTest({ + branch, + defaultBranchValue, + userBranchValue, + configs: specifiedConfigs, + operation, + }); + } + } + } + } + } + } + + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js new file mode 100644 index 0000000000..904060eb73 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js @@ -0,0 +1,510 @@ +"use strict"; + +const { TelemetryEvents } = ChromeUtils.import( + "resource://normandy/lib/TelemetryEvents.jsm" +); +const { TelemetryEnvironment } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryEnvironment.sys.mjs" +); +const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled"; +const UPLOAD_ENABLED_PREF = "datareporting.healthreport.uploadEnabled"; + +const globalSandbox = sinon.createSandbox(); +globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); +globalSandbox.spy(TelemetryEvents, "sendEvent"); +registerCleanupFunction(() => { + globalSandbox.restore(); +}); + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +/** + * Normal unenrollment for experiments: + * - set .active to false + * - set experiment inactive in telemetry + * - send unrollment event + */ +add_task(async function test_set_inactive() { + const manager = ExperimentFakes.manager(); + + await manager.onStartup(); + await manager.store.addEnrollment(ExperimentFakes.experiment("foo")); + + manager.unenroll("foo", "some-reason"); + + Assert.equal( + manager.store.get("foo").active, + false, + "should set .active to false" + ); +}); + +add_task(async function test_unenroll_opt_out() { + globalSandbox.reset(); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + + Assert.equal( + manager.store.get(experiment.slug).active, + false, + "should set .active to false" + ); + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + experiment.slug, + { + reason: "studies-opt-out", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "studies-opt-out", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + experiment.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + // reset pref + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_unenroll_rollout_opt_out() { + globalSandbox.reset(); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + const manager = ExperimentFakes.manager(); + const rollout = ExperimentFakes.rollout("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(rollout); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + + Assert.equal( + manager.store.get(rollout.slug).active, + false, + "should set .active to false" + ); + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + rollout.slug, + { + reason: "studies-opt-out", + branch: rollout.branch.slug, + enrollmentId: rollout.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + rollout.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct rollout slug" + ); + Assert.equal( + rollout.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "studies-opt-out", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + rollout.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + // reset pref + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_unenroll_uploadPref() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const recipe = ExperimentFakes.recipe("foo"); + + await manager.onStartup(); + await ExperimentFakes.enrollmentHelper(recipe, { manager }).enrollmentPromise; + + Assert.equal( + manager.store.get(recipe.slug).active, + true, + "Should set .active to true" + ); + + Services.prefs.setBoolPref(UPLOAD_ENABLED_PREF, false); + + Assert.equal( + manager.store.get(recipe.slug).active, + false, + "Should set .active to false" + ); + Services.prefs.clearUserPref(UPLOAD_ENABLED_PREF); +}); + +add_task(async function test_setExperimentInactive_called() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Because `manager.store.addEnrollment()` sidesteps telemetry recording + // we will also call on the Glean experiment API directly to test that + // `manager.unenroll()` does in fact call `Glean.setExperimentActive()` + Services.fog.setExperimentActive( + experiment.slug, + experiment.branch.slug, + null + ); + + // Test Glean experiment API interaction + Assert.notEqual( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "experiment should be active before unenroll" + ); + + manager.unenroll("foo", "some-reason"); + + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledWith("foo"), + "should call TelemetryEnvironment.setExperimentInactive with slug" + ); + + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "experiment should be inactive after unenroll" + ); +}); + +add_task(async function test_send_unenroll_event() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + manager.unenroll("foo", "some-reason"); + + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + "foo", // slug + { + reason: "some-reason", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "some-reason", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + experiment.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); +}); + +add_task(async function test_undefined_reason() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + manager.unenroll("foo"); + + const options = TelemetryEvents.sendEvent.firstCall?.args[3]; + Assert.ok( + "reason" in options, + "options object with .reason should be the fourth param" + ); + Assert.equal( + options.reason, + "unknown", + "should include unknown as the reason if none was supplied" + ); + + // Check that the Glean unenrollment event was recorded. + let unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event reason matches the expected reason + Assert.equal( + "unknown", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct (unknown) reason" + ); +}); + +/** + * Normal unenrollment for rollouts: + * - remove stored enrollment and synced data (prefs) + * - set rollout inactive in telemetry + * - send unrollment event + */ + +add_task(async function test_remove_rollouts() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + + sinon.stub(store, "get").returns(rollout); + sinon.spy(store, "updateExperiment"); + + await manager.onStartup(); + + manager.unenroll("foo", "some-reason"); + + Assert.ok( + manager.store.updateExperiment.calledOnce, + "Called to set the rollout as !active" + ); + Assert.ok( + manager.store.updateExperiment.calledWith(rollout.slug, { active: false }), + "Called with expected parameters" + ); +}); + +add_task(async function test_remove_rollout_onFinalize() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + + sinon.stub(store, "getAllRollouts").returns([rollout]); + sinon.stub(store, "get").returns(rollout); + sinon.spy(manager, "unenroll"); + sinon.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + manager.onFinalize("NimbusTestUtils"); + + // Check that there aren't any Glean unenroll_failed events + var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue(); + Assert.equal( + undefined, + unenrollFailedEvents, + "no Glean unenroll_failed events when removing rollout" + ); + + Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail"); + Assert.ok(manager.unenroll.calledOnce, "Should unenroll recipe not seen"); + Assert.ok(manager.unenroll.calledWith(rollout.slug, "recipe-not-seen")); +}); + +add_task(async function test_rollout_telemetry_events() { + globalSandbox.restore(); + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); + globalSandbox.spy(TelemetryEvents, "sendEvent"); + + sinon.stub(store, "getAllRollouts").returns([rollout]); + sinon.stub(store, "get").returns(rollout); + sinon.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + manager.onFinalize("NimbusTestUtils"); + + // Check that there aren't any Glean unenroll_failed events + var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue(); + Assert.equal( + undefined, + unenrollFailedEvents, + "no Glean unenroll_failed events when removing rollout" + ); + + Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail"); + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledOnce, + "Should unenroll recipe not seen" + ); + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledWith(rollout.slug), + "Should set rollout to inactive." + ); + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData(rollout.slug), + "Should set rollout to inactive" + ); + + Assert.ok( + TelemetryEvents.sendEvent.calledWith( + "unenroll", + sinon.match.string, + rollout.slug, + sinon.match.object + ), + "Should send unenroll event for rollout." + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + rollout.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct rollout slug" + ); + Assert.equal( + rollout.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "recipe-not-seen", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + rollout.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + globalSandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentStore.js b/toolkit/components/nimbus/test/unit/test_ExperimentStore.js new file mode 100644 index 0000000000..a9f9788ac2 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentStore.js @@ -0,0 +1,869 @@ +"use strict"; + +const { ExperimentStore } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentStore.jsm" +); +const { FeatureManifest } = ChromeUtils.import( + "resource://nimbus/FeatureManifest.js" +); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; +const { cleanupStorePrefCache } = ExperimentFakes; + +add_task(async function test_sharedDataMap_key() { + const store = new ExperimentStore(); + + // Outside of tests we use sharedDataKey for the profile dir filepath + // where we store experiments + Assert.ok(store._sharedDataKey, "Make sure it's defined"); +}); + +add_task(async function test_usageBeforeInitialization() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + + Assert.equal(store.getAll().length, 0, "It should not fail"); + + await store.init(); + store.addEnrollment(experiment); + + Assert.equal( + store.getExperimentForFeature("purple"), + experiment, + "should return a matching experiment for the given feature" + ); +}); + +add_task(async function test_event_add_experiment() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const expected = ExperimentFakes.experiment("foo"); + const updateEventCbStub = sandbox.stub(); + + // Setup ExperimentManager and child store for ExperimentAPI + await store.init(); + + // Set update cb + store.on("update:foo", updateEventCbStub); + + // Add some data + store.addEnrollment(expected); + + Assert.equal(updateEventCbStub.callCount, 1, "Called once for add"); + + store.off("update:foo", updateEventCbStub); +}); + +add_task(async function test_event_updates_main() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo"); + const updateEventCbStub = sandbox.stub(); + + // Setup ExperimentManager and child store for ExperimentAPI + await store.init(); + + // Set update cb + store.on( + `update:${experiment.branch.features[0].featureId}`, + updateEventCbStub + ); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + Assert.equal( + updateEventCbStub.callCount, + 2, + "Should be called twice: add, update" + ); + Assert.equal( + updateEventCbStub.secondCall.args[1].active, + false, + "Should be called with updated experiment status" + ); + + store.off( + `update:${experiment.branch.features[0].featureId}`, + updateEventCbStub + ); +}); + +add_task(async function test_getExperimentForGroup() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + + await store.init(); + store.addEnrollment(ExperimentFakes.experiment("bar")); + store.addEnrollment(experiment); + + Assert.equal( + store.getExperimentForFeature("purple"), + experiment, + "should return a matching experiment for the given feature" + ); +}); + +add_task(async function test_hasExperimentForFeature() { + const store = ExperimentFakes.store(); + + await store.init(); + store.addEnrollment( + ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + feature: { featureId: "green" }, + }, + }) + ); + store.addEnrollment( + ExperimentFakes.experiment("foo2", { + branch: { + slug: "variant", + feature: { featureId: "yellow" }, + }, + }) + ); + store.addEnrollment( + ExperimentFakes.experiment("bar_expired", { + active: false, + branch: { + slug: "variant", + feature: { featureId: "purple" }, + }, + }) + ); + Assert.equal( + store.hasExperimentForFeature(), + false, + "should return false if the input is empty" + ); + + Assert.equal( + store.hasExperimentForFeature(undefined), + false, + "should return false if the input is undefined" + ); + + Assert.equal( + store.hasExperimentForFeature("green"), + true, + "should return true if there is an experiment with any of the given groups" + ); + + Assert.equal( + store.hasExperimentForFeature("purple"), + false, + "should return false if there is a non-active experiment with the given groups" + ); +}); + +add_task(async function test_getAll_getAllActive() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.experiment(slug, { active: false })) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux"], + ".getAll() should return all experiments" + ); + Assert.deepEqual( + store.getAllActive().map(e => e.slug), + ["qux"], + ".getAllActive() should return all experiments that are active" + ); +}); + +add_task(async function test_getAll_getAllActive_no_rollouts() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.experiment(slug, { active: false })) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + store.addEnrollment(ExperimentFakes.rollout("rol")); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux", "rol"], + ".getAll() should return all experiments and rollouts" + ); + Assert.deepEqual( + store.getAllActive().map(e => e.slug), + ["qux"], + ".getAllActive() should return all experiments that are active and no rollouts" + ); +}); + +add_task(async function test_getAllRollouts() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.rollout(slug)) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux"], + ".getAll() should return all experiments and rollouts" + ); + Assert.deepEqual( + store.getAllRollouts().map(e => e.slug), + ["foo", "bar", "baz"], + ".getAllRollouts() should return all rollouts" + ); +}); + +add_task(async function test_addEnrollment_experiment() { + const store = ExperimentFakes.store(); + const exp = ExperimentFakes.experiment("foo"); + + await store.init(); + store.addEnrollment(exp); + + Assert.equal(store.get("foo"), exp, "should save experiment by slug"); +}); + +add_task(async function test_addEnrollment_rollout() { + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo"); + + await store.init(); + store.addEnrollment(rollout); + + Assert.equal(store.get("foo"), rollout, "should save rollout by slug"); +}); + +add_task(async function test_updateExperiment() { + const features = [{ featureId: "cfr" }]; + const experiment = Object.freeze( + ExperimentFakes.experiment("foo", { features, active: true }) + ); + const store = ExperimentFakes.store(); + + await store.init(); + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + const actual = store.get("foo"); + Assert.equal(actual.active, false, "should change updated props"); + Assert.deepEqual( + actual.branch.features, + features, + "should not update other props" + ); +}); + +add_task(async function test_sync_access_before_init() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + + Assert.equal(store.getAll().length, 0, "Start with an empty store"); + + const syncAccessExp = ExperimentFakes.experiment("foo", { + features: [{ featureId: "newtab" }], + }); + await store.init(); + store.addEnrollment(syncAccessExp); + + let prefValue; + try { + prefValue = JSON.parse( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}newtab`) + ); + } catch (e) { + Assert.ok(false, "Failed to parse pref value"); + } + + Assert.ok(prefValue, "Parsed stored experiment"); + Assert.equal(prefValue.slug, syncAccessExp.slug, "Got back the experiment"); + + // New un-initialized store that should read the pref value + store = ExperimentFakes.store(); + + Assert.equal( + store.getExperimentForFeature("newtab").slug, + "foo", + "Returns experiment from pref" + ); +}); + +add_task(async function test_sync_access_update() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "aboutwelcome" }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { + branch: { + ...experiment.branch, + features: [ + { + featureId: "aboutwelcome", + value: { bar: "bar", enabled: true }, + }, + ], + }, + }); + + store = ExperimentFakes.store(); + let cachedExperiment = store.getExperimentForFeature("aboutwelcome"); + + Assert.ok(cachedExperiment, "Got back 1 experiment"); + Assert.deepEqual( + // `branch.feature` and not `features` because for sync access (early startup) + // experiments we only store the `isEarlyStartup` feature + cachedExperiment.branch.feature.value, + { bar: "bar", enabled: true }, + "Got updated value" + ); +}); + +add_task(async function test_sync_features_only() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "cfr" }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store = ExperimentFakes.store(); + + Assert.equal(store.getAll().length, 0, "cfr is not a sync access experiment"); +}); + +add_task(async function test_sync_features_remotely() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "cfr", isEarlyStartup: true }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store = ExperimentFakes.store(); + + Assert.ok( + Services.prefs.prefHasUserValue("nimbus.syncdatastore.cfr"), + "The cfr feature was stored as early access in prefs" + ); + Assert.equal(store.getAll().length, 0, "Featre restored from prefs"); +}); + +add_task(async function test_sync_access_unenroll() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "aboutwelcome" }], + active: true, + }); + + await store.init(); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + store = ExperimentFakes.store(); + let experiments = store.getAll(); + + Assert.equal(experiments.length, 0, "Unenrolled experiment is deleted"); +}); + +add_task(async function test_sync_access_unenroll_2() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment1 = ExperimentFakes.experiment("foo", { + features: [{ featureId: "newtab" }], + }); + let experiment2 = ExperimentFakes.experiment("bar", { + features: [{ featureId: "aboutwelcome" }], + }); + + await store.init(); + + store.addEnrollment(experiment1); + store.addEnrollment(experiment2); + + Assert.equal(store.getAll().length, 2, "2/2 experiments"); + + let other_store = ExperimentFakes.store(); + + Assert.ok( + other_store.getExperimentForFeature("aboutwelcome"), + "Fetches experiment from pref cache even before init (aboutwelcome)" + ); + + store.updateExperiment("bar", { active: false }); + + Assert.ok( + other_store.getExperimentForFeature("newtab").slug, + "Fetches experiment from pref cache even before init (newtab)" + ); + Assert.ok( + !other_store.getExperimentForFeature("aboutwelcome")?.slug, + "Experiment was updated and should not be found" + ); + + store.updateExperiment("foo", { active: false }); + Assert.ok( + !other_store.getExperimentForFeature("newtab")?.slug, + "Unenrolled from 2/2 experiments" + ); + + Assert.equal( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}newtab`, "").length, + 0, + "Cleared pref 1" + ); + Assert.equal( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}aboutwelcome`, "") + .length, + 0, + "Cleared pref 2" + ); +}); + +add_task(async function test_getRolloutForFeature_fromStore() { + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo"); + + await store.init(); + store.addEnrollment(rollout); + + Assert.deepEqual( + store.getRolloutForFeature(rollout.featureIds[0]), + rollout, + "Should return back the same rollout" + ); +}); + +add_task(async function test_getRolloutForFeature_fromSyncCache() { + let store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "early-startup", + features: [{ featureId: "aboutwelcome", value: { enabled: true } }], + }, + }); + let updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + + await store.init(); + store.addEnrollment(rollout); + await updatePromise; + // New uninitialized store will return data from sync cache + // before init + store = ExperimentFakes.store(); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Sync cache is set" + ); + Assert.equal( + store.getRolloutForFeature(rollout.featureIds[0]).slug, + rollout.slug, + "Should return back the same rollout" + ); + Assert.deepEqual( + store.getRolloutForFeature(rollout.featureIds[0]).branch.feature, + rollout.branch.features[0], + "Should return back the same feature" + ); + cleanupStorePrefCache(); +}); + +add_task(async function test_remoteRollout() { + let store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "early-startup", + features: [{ featureId: "aboutwelcome", value: { enabled: true } }], + }, + }); + let featureUpdateStub = sinon.stub(); + let updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + store.on("update:aboutwelcome", featureUpdateStub); + + await store.init(); + store.addEnrollment(rollout); + await updatePromise; + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Sync cache is set" + ); + + updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + store.updateExperiment(rollout.slug, { active: false }); + + // wait for it to be removed + await updatePromise; + + Assert.ok(featureUpdateStub.calledTwice, "Called for add and remove"); + Assert.ok( + store.get(rollout.slug), + "Rollout is still in the store just not active" + ); + Assert.ok( + !store.getRolloutForFeature("aboutwelcome"), + "Feature rollout should not exist" + ); + Assert.ok( + !Services.prefs.getStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`, + "" + ), + "Sync cache is cleared" + ); +}); + +add_task(async function test_syncDataStore_setDefault() { + cleanupStorePrefCache(); + const store = ExperimentFakes.store(); + + await store.init(); + + Assert.equal( + Services.prefs.getStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`, + "" + ), + "", + "Pref is empty" + ); + + let rollout = ExperimentFakes.rollout("foo", { + features: [{ featureId: "aboutwelcome", value: { remote: true } }], + }); + store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Stored in pref" + ); + + cleanupStorePrefCache(); +}); + +add_task(async function test_syncDataStore_getDefault() { + cleanupStorePrefCache(); + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("aboutwelcome-slug", { + branch: { + features: [ + { + featureId: "aboutwelcome", + value: { remote: true }, + }, + ], + }, + }); + + await store.init(); + await store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`) + ); + + let restoredRollout = store.getRolloutForFeature("aboutwelcome"); + + Assert.ok(restoredRollout); + Assert.ok( + restoredRollout.branch.features[0].value.remote, + "Restore data from pref" + ); + + cleanupStorePrefCache(); +}); + +add_task(async function test_addEnrollment_rollout() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const stub = sandbox.stub(); + const value = { bar: true }; + let rollout = ExperimentFakes.rollout("foo", { + features: [{ featureId: "aboutwelcome", value }], + }); + + store._onFeatureUpdate("aboutwelcome", stub); + + await store.init(); + store.addEnrollment(rollout); + + Assert.deepEqual( + store.getRolloutForFeature("aboutwelcome"), + rollout, + "should return the stored value" + ); + Assert.equal(stub.callCount, 1, "Called once on update"); + Assert.equal( + stub.firstCall.args[1], + "rollout-updated", + "Called for correct reason" + ); +}); + +add_task(async function test_storeValuePerPref_noVariables() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment metadata saved to prefs" + ); + + Assert.equal(branch.getChildList("").length, 0, "No variables to store"); + + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); +}); + +add_task(async function test_storeValuePerPref_withVariables() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { color: "purple", enabled: true }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + let val = Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`); + Assert.equal( + val.indexOf("color"), + -1, + `Experiment metadata does not contain variables ${val}` + ); + + Assert.equal(branch.getChildList("").length, 2, "Enabled and color"); + + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.equal(branch.getChildList("").length, 0, "Variables are also removed"); +}); + +add_task(async function test_storeValuePerPref_returnsSameValue() { + let store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { color: "purple", enabled: true }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + store = ExperimentFakes.store(); + const cachedExperiment = store.getExperimentForFeature("purple"); + // Cached experiment format only stores early access feature + cachedExperiment.branch.features = [cachedExperiment.branch.feature]; + delete cachedExperiment.branch.feature; + Assert.deepEqual(cachedExperiment, experiment, "Returns the same value"); + + // Cleanup + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.deepEqual(branch.getChildList(""), [], "Variables are also removed"); +}); + +add_task(async function test_storeValuePerPref_returnsSameValue_allTypes() { + let store = ExperimentFakes.store(); + // Add a fake feature that matches the variables we're testing + FeatureManifest.purple = { + variables: { + string: { type: "string" }, + bool: { type: "boolean" }, + array: { type: "json" }, + number1: { type: "int" }, + number2: { type: "int" }, + number3: { type: "int" }, + json: { type: "json" }, + }, + }; + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { + string: "string", + bool: true, + array: [1, 2, 3], + number1: 42, + number2: 0, + number3: -5, + json: { jsonValue: true }, + }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + store = ExperimentFakes.store(); + Assert.deepEqual( + store.getExperimentForFeature("purple").branch.feature.value, + experiment.branch.features[0].value, + "Returns the same value" + ); + + // Cleanup + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.deepEqual(branch.getChildList(""), [], "Variables are also removed"); + delete FeatureManifest.purple; +}); + +add_task(async function test_cleanupOldRecipes() { + let store = ExperimentFakes.store(); + let sandbox = sinon.createSandbox(); + let stub = sandbox.stub(store, "_removeEntriesByKeys"); + const experiment1 = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment2 = ExperimentFakes.experiment("bar", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment3 = ExperimentFakes.experiment("baz", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment4 = ExperimentFakes.experiment("faz", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + // Exp 2 is kept because it's recent (even though it's not active) + // Exp 4 is kept because it's active + experiment2.lastSeen = new Date().toISOString(); + experiment2.active = false; + experiment1.lastSeen = new Date("2020-01-01").toISOString(); + experiment1.active = false; + experiment3.active = false; + delete experiment3.lastSeen; + store._data = { + foo: experiment1, + bar: experiment2, + baz: experiment3, + faz: experiment4, + }; + + store._cleanupOldRecipes(); + + Assert.ok(stub.calledOnce, "Recipe cleanup called"); + Assert.equal( + stub.firstCall.args[0].length, + 2, + "We call to remove enrollments" + ); + Assert.equal( + stub.firstCall.args[0][0], + experiment1.slug, + "Should remove expired enrollment" + ); + Assert.equal( + stub.firstCall.args[0][1], + experiment3.slug, + "Should remove invalid enrollment" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js b/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js new file mode 100644 index 0000000000..78b7948199 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js @@ -0,0 +1,84 @@ +"use strict"; + +const { ExperimentFakes, ExperimentTestUtils } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); + +add_task(async function test_recipe_fake_validates() { + const recipe = ExperimentFakes.recipe("foo"); + Assert.ok( + await ExperimentTestUtils.validateExperiment(recipe), + "should produce a valid experiment recipe" + ); +}); + +add_task(async function test_enrollmentHelper() { + let recipe = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "control", + ratio: 1, + features: [{ featureId: "aboutwelcome", value: {} }], + }, + ], + }); + let manager = ExperimentFakes.manager(); + + Assert.deepEqual( + recipe.featureIds, + ["aboutwelcome"], + "Helper sets correct featureIds" + ); + + await manager.onStartup(); + + let { + enrollmentPromise, + doExperimentCleanup, + } = ExperimentFakes.enrollmentHelper(recipe, { manager }); + + await enrollmentPromise; + + Assert.ok(manager.store.getAllActive().length === 1, "Enrolled"); + Assert.equal( + manager.store.getAllActive()[0].slug, + recipe.slug, + "Has expected slug" + ); + Assert.ok( + Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"), + "Sync pref cache set" + ); + + await doExperimentCleanup(); + + Assert.ok(manager.store.getAll().length === 0, "Cleanup done"); + Assert.ok( + !Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"), + "Sync pref cache is cleared" + ); +}); + +add_task(async function test_enrollWithFeatureConfig() { + let manager = ExperimentFakes.manager(); + await manager.onStartup(); + let doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "enrollWithFeatureConfig", + value: { enabled: true }, + }, + { manager } + ); + + Assert.ok( + manager.store.hasExperimentForFeature("enrollWithFeatureConfig"), + "Enrolled successfully" + ); + + await doExperimentCleanup(); + + Assert.ok( + !manager.store.hasExperimentForFeature("enrollWithFeatureConfig"), + "Unenrolled successfully" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js new file mode 100644 index 0000000000..433aca60e7 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js @@ -0,0 +1,315 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { ExperimentManager } = ChromeUtils.import( + "resource://nimbus/lib/ExperimentManager.jsm" +); + +const { RemoteSettingsExperimentLoader } = ChromeUtils.import( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm" +); + +const ENABLED_PREF = "messaging-system.rsexperimentloader.enabled"; +const RUN_INTERVAL_PREF = "app.normandy.run_interval_seconds"; +const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled"; +const UPLOAD_PREF = "datareporting.healthreport.uploadEnabled"; +const DEBUG_PREF = "nimbus.debug"; + +add_task(async function test_real_exp_manager() { + equal( + RemoteSettingsExperimentLoader.manager, + ExperimentManager, + "should reference ExperimentManager singleton by default" + ); +}); + +add_task(async function test_lazy_pref_getters() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setIntPref(RUN_INTERVAL_PREF, 123456); + equal( + loader.intervalInSeconds, + 123456, + `should set intervalInSeconds to the value of ${RUN_INTERVAL_PREF}` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + equal( + loader.enabled, + true, + `should set enabled to the value of ${ENABLED_PREF}` + ); + Services.prefs.setBoolPref(ENABLED_PREF, false); + equal(loader.enabled, false); + + Services.prefs.clearUserPref(RUN_INTERVAL_PREF); + Services.prefs.clearUserPref(ENABLED_PREF); +}); + +add_task(async function test_init() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setBoolPref(ENABLED_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${ENABLED_PREF} pref is false` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.setTimer.calledOnce, "should call .setTimer"); + ok(loader.updateRecipes.calledOnce, "should call .updatpickeRecipes"); +}); + +add_task(async function test_init_with_opt_in() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${STUDIES_OPT_OUT_PREF} pref is false` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${ENABLED_PREF} pref is false` + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.setTimer.calledOnce, "should call .setTimer"); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); +}); + +add_task(async function test_updateRecipes() { + const loader = ExperimentFakes.rsLoader(); + + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "true", + }); + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "false", + }); + sinon.stub(loader, "setTimer"); + sinon.spy(loader, "updateRecipes"); + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([PASS_FILTER_RECIPE, FAIL_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should call .onRecipe only for recipes that pass" + ); + ok( + loader.manager.onRecipe.calledWith(PASS_FILTER_RECIPE, "rs-loader"), + "should call .onRecipe with argument data" + ); +}); + +add_task(async function test_updateRecipes_someMismatch() { + const loader = ExperimentFakes.rsLoader(); + + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "true", + }); + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "false", + }); + sinon.stub(loader, "setTimer"); + sinon.spy(loader, "updateRecipes"); + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([PASS_FILTER_RECIPE, FAIL_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should call .onRecipe only for recipes that pass" + ); + ok(loader.manager.onFinalize.calledOnce, "Should call onFinalize."); + ok( + loader.manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [FAIL_FILTER_RECIPE.slug], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with the recipes that failed targeting" + ); +}); + +add_task(async function test_updateRecipes_forFirstStartup() { + const loader = ExperimentFakes.rsLoader(); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "isFirstStartup", + }); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + sinon + .stub(loader.manager, "createTargetingContext") + .returns({ isFirstStartup: true }); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init({ isFirstStartup: true }); + + ok(loader.manager.onRecipe.calledOnce, "should pass the targeting filter"); +}); + +add_task(async function test_updateRecipes_forNoneFirstStartup() { + const loader = ExperimentFakes.rsLoader(); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "isFirstStartup", + }); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + sinon + .stub(loader.manager, "createTargetingContext") + .returns({ isFirstStartup: false }); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init({ isFirstStartup: true }); + + ok(loader.manager.onRecipe.notCalled, "should not pass the targeting filter"); +}); + +add_task(async function test_checkTargeting() { + const loader = ExperimentFakes.rsLoader(); + equal( + await loader.checkTargeting({}), + true, + "should return true if .targeting is not defined" + ); + equal( + await loader.checkTargeting({ + targeting: "'foo'", + slug: "test_checkTargeting", + }), + true, + "should return true for truthy expression" + ); + equal( + await loader.checkTargeting({ + targeting: "aPropertyThatDoesNotExist", + slug: "test_checkTargeting", + }), + false, + "should return false for falsey expression" + ); +}); + +add_task(async function test_checkExperimentSelfReference() { + const loader = ExperimentFakes.rsLoader(); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: + "experiment.slug == 'foo' && experiment.branches[0].slug == 'control'", + }); + + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "experiment.slug == 'bar'", + }); + + equal( + await loader.checkTargeting(PASS_FILTER_RECIPE), + true, + "Should return true for matching on slug name and branch" + ); + equal( + await loader.checkTargeting(FAIL_FILTER_RECIPE), + false, + "Should fail targeting" + ); +}); + +add_task(async function test_optIn_debug_disabled() { + info("Testing users cannot opt-in when nimbus.debug is false"); + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + const recipe = ExperimentFakes.recipe("foo"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + Services.prefs.setBoolPref(DEBUG_PREF, false); + Services.prefs.setBoolPref(UPLOAD_PREF, true); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + + await Assert.rejects( + loader.optInToExperiment({ + slug: recipe.slug, + branchSlug: recipe.branches[0].slug, + }), + /Could not opt in/ + ); + + Services.prefs.clearUserPref(DEBUG_PREF); + Services.prefs.clearUserPref(UPLOAD_PREF); + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_optIn_studies_disabled() { + info( + "Testing users cannot opt-in when telemetry is disabled or studies are disabled." + ); + + const prefs = [UPLOAD_PREF, STUDIES_OPT_OUT_PREF]; + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + const recipe = ExperimentFakes.recipe("foo"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + Services.prefs.setBoolPref(DEBUG_PREF, true); + + for (const pref of prefs) { + Services.prefs.setBoolPref(UPLOAD_PREF, true); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + + Services.prefs.setBoolPref(pref, false); + + await Assert.rejects( + loader.optInToExperiment({ + slug: recipe.slug, + branchSlug: recipe.branches[0].slug, + }), + /Could not opt in: studies are disabled/ + ); + } + + Services.prefs.clearUserPref(DEBUG_PREF); + Services.prefs.clearUserPref(UPLOAD_PREF); + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); diff --git a/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js new file mode 100644 index 0000000000..9fe8bafef6 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js @@ -0,0 +1,929 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.import( + "resource://testing-common/NimbusTestUtils.jsm" +); +const { FirstStartup } = ChromeUtils.importESModule( + "resource://gre/modules/FirstStartup.sys.mjs" +); +const { NimbusFeatures } = ChromeUtils.import( + "resource://nimbus/ExperimentAPI.jsm" +); +const { PanelTestProvider } = ChromeUtils.import( + "resource://activity-stream/lib/PanelTestProvider.jsm" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.import( + "resource://normandy/lib/TelemetryEvents.jsm" +); + +add_setup(async function setup() { + do_get_profile(); + Services.fog.initializeFOG(); +}); + +add_task(async function test_updateRecipes_activeExperiments() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: `"${recipe.slug}" in activeExperiments`, + }); + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([recipe]); + + await loader.init(); + + ok(onRecipe.calledOnce, "Should match active experiments"); +}); + +add_task(async function test_updateRecipes_isFirstRun() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + const PASS_FILTER_RECIPE = { ...recipe, targeting: "isFirstStartup" }; + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([recipe]); + + // Pretend to be in the first startup + FirstStartup._state = FirstStartup.IN_PROGRESS; + await loader.init(); + + Assert.ok(onRecipe.calledOnce, "Should match first run"); +}); + +add_task(async function test_updateRecipes_invalidFeatureId() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "invalid-feature-id", + value: { hello: "world" }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "invalid-feature-id", + value: { hello: "goodbye" }, + }, + ], + }, + ], + }); + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); +}); + +add_task(async function test_updateRecipes_invalidFeatureValue() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { + id: "test-spotlight-invalid-1", + }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { + id: "test-spotlight-invalid-2", + }, + }, + ], + }, + ], + }); + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); +}); + +add_task(async function test_updateRecipes_invalidRecipe() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo"); + delete badRecipe.slug; + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActive").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); +}); + +add_task(async function test_updateRecipes_invalidRecipeAfterUpdate() { + Services.fog.testResetFOG(); + + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const recipe = ExperimentFakes.recipe("foo"); + const badRecipe = { ...recipe }; + delete badRecipe.branches; + + sinon.stub(loader, "setTimer"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + sinon.stub(manager.store, "ready").resolves(); + sinon.spy(loader, "updateRecipes"); + + await loader.init(); + + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + ok( + loader.manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with no mismatches or invalid recipes" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + loader.manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + loader.manager.onFinalize.secondCall.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: ["foo"], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with an invalid recipe" + ); +}); + +add_task(async function test_updateRecipes_invalidBranchAfterUpdate() { + const message = await PanelTestProvider.getMessages().then(msgs => + msgs.find(m => m.id === "SPOTLIGHT_MESSAGE_93") + ); + + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { ...message }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { ...message }, + }, + ], + }, + ], + }); + + const badRecipe = { + ...recipe, + branches: [ + { ...recipe.branches[0] }, + { + ...recipe.branches[1], + features: [ + { + ...recipe.branches[1].features[0], + value: { ...message }, + }, + ], + }, + ], + }; + delete badRecipe.branches[1].features[0].value.template; + + sinon.stub(loader, "setTimer"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + sinon.stub(manager.store, "ready").resolves(); + sinon.spy(loader, "updateRecipes"); + + await loader.init(); + + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + ok( + loader.manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with no mismatches or invalid recipes" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + loader.manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + loader.manager.onFinalize.secondCall.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([["foo", [badRecipe.branches[0].slug]]]), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with an invalid branch" + ); +}); + +add_task(async function test_updateRecipes_simpleFeatureInvalidAfterUpdate() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("foo"); + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + ...recipe.branches[0], + features: [ + { + featureId: "testFeature", + value: { testInt: "abc123", enabled: true }, + }, + ], + }, + { + ...recipe.branches[1], + features: [ + { + featureId: "testFeature", + value: { testInt: 456, enabled: true }, + }, + ], + }, + ], + }); + + const EXPECTED_SCHEMA = { + $schema: "https://json-schema.org/draft/2019-09/schema", + title: "testFeature", + description: NimbusFeatures.testFeature.manifest.description, + type: "object", + properties: { + testInt: { + type: "integer", + }, + enabled: { + type: "boolean", + }, + testSetString: { + type: "string", + }, + }, + additionalProperties: true, + }; + + sinon.spy(loader, "updateRecipes"); + sinon.spy(loader, "_generateVariablesOnlySchema"); + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onFinalize"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + + await loader.init(); + ok(manager.onRecipe.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + ok( + loader.manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with nomismatches or invalid recipes" + ); + + ok( + loader._generateVariablesOnlySchema.calledOnce, + "Should have generated a schema for testFeature" + ); + + Assert.deepEqual( + loader._generateVariablesOnlySchema.returnValues[0], + EXPECTED_SCHEMA, + "should have generated a schema with three fields" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + loader.manager.onFinalize.secondCall.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([["foo", [badRecipe.branches[0].slug]]]), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with an invalid branch" + ); +}); + +add_task(async function test_updateRecipes_validationTelemetry() { + TelemetryEvents.init(); + + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); + + const invalidRecipe = ExperimentFakes.recipe("invalid-recipe"); + delete invalidRecipe.channel; + + const invalidBranch = ExperimentFakes.recipe("invalid-branch"); + invalidBranch.branches[0].features[0].value.testInt = "hello"; + invalidBranch.branches[1].features[0].value.testInt = "world"; + + const invalidFeature = ExperimentFakes.recipe("invalid-feature", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "unknown-feature", + value: { foo: "bar" }, + }, + { + featureId: "second-unknown-feature", + value: { baz: "qux" }, + }, + ], + }, + ], + }); + + const TEST_CASES = [ + { + recipe: invalidRecipe, + reason: "invalid-recipe", + events: [{}], + callCount: 1, + }, + { + recipe: invalidBranch, + reason: "invalid-branch", + events: invalidBranch.branches.map(branch => ({ branch: branch.slug })), + callCount: 2, + }, + { + recipe: invalidFeature, + reason: "invalid-feature", + events: invalidFeature.branches[0].features.map(feature => ({ + feature: feature.featureId, + })), + callCount: 2, + }, + ]; + + const LEGACY_FILTER = { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + }; + + for (const { recipe, reason, events, callCount } of TEST_CASES) { + info(`Testing validation failed telemetry for reason = "${reason}" ...`); + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActive").returns([]); + sinon.stub(manager.store, "getAllRollouts").returns([]); + + const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry"); + + await loader.init(); + + Assert.equal( + telemetrySpy.callCount, + callCount, + `Should call sendValidationFailedTelemetry ${callCount} times for reason ${reason}` + ); + + const gleanEvents = Glean.nimbusEvents.validationFailed + .testGetValue() + .map(event => { + event = { ...event }; + // We do not care about the timestamp. + delete event.timestamp; + return event; + }); + + const expectedGleanEvents = events.map(event => ({ + category: "nimbus_events", + name: "validation_failed", + extra: { + experiment: recipe.slug, + reason, + ...event, + }, + })); + + Assert.deepEqual( + gleanEvents, + expectedGleanEvents, + "Glean telemetry matches" + ); + + const expectedLegacyEvents = events.map(event => ({ + ...LEGACY_FILTER, + value: recipe.slug, + extra: { + reason, + ...event, + }, + LEGACY_FILTER, + })); + + TelemetryTestUtils.assertEvents(expectedLegacyEvents, LEGACY_FILTER, { + clear: true, + }); + + Services.fog.testResetFOG(); + } +}); + +add_task(async function test_updateRecipes_validationDisabled() { + Services.prefs.setBoolPref("nimbus.validation.enabled", false); + + const invalidRecipe = ExperimentFakes.recipe("invalid-recipe"); + delete invalidRecipe.channel; + + const invalidBranch = ExperimentFakes.recipe("invalid-branch"); + invalidBranch.branches[0].features[0].value.testInt = "hello"; + invalidBranch.branches[1].features[0].value.testInt = "world"; + + const invalidFeature = ExperimentFakes.recipe("invalid-feature", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "unknown-feature", + value: { foo: "bar" }, + }, + { + featureId: "second-unknown-feature", + value: { baz: "qux" }, + }, + ], + }, + ], + }); + + for (const recipe of [invalidRecipe, invalidBranch, invalidFeature]) { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActive").returns([]); + sinon.stub(manager.store, "getAllRollouts").returns([]); + + const finalizeStub = sinon.stub(manager, "onFinalize"); + const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry"); + + await loader.init(); + + Assert.equal( + telemetrySpy.callCount, + 0, + "Should not send validation failed telemetry" + ); + Assert.ok( + finalizeStub.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: false, + }), + "should call .onFinalize with no validation issues" + ); + } + Services.prefs.clearUserPref("nimbus.validation.enabled"); +}); + +add_task(async function test_updateRecipes_appId() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("background-task-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "backgroundTaskMessage", + value: {}, + }, + ], + }, + ], + }); + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + + info("Testing updateRecipes() with the default application ID"); + await loader.init(); + + Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called"); + Assert.ok( + manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); + + info("Testing updateRecipes() with a custom application ID"); + + Services.prefs.setStringPref( + "nimbus.appId", + "firefox-desktop-background-task" + ); + + await loader.updateRecipes(); + Assert.ok( + manager.onRecipe.calledWith(recipe, "rs-loader"), + `.onRecipe called with ${recipe.slug}` + ); + + Assert.ok( + manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); + + Services.prefs.clearUserPref("nimbus.appId"); +}); + +add_task(async function test_updateRecipes_withPropNotInManifest() { + // Need to randomize the slug so subsequent test runs don't skip enrollment + // due to a conflicting slug + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo" + Math.random(), { + arguments: {}, + branches: [ + { + features: [ + { + enabled: true, + featureId: "testFeature", + value: { + enabled: true, + testInt: 5, + testSetString: "foo", + additionalPropNotInManifest: 7, + }, + }, + ], + ratio: 1, + slug: "treatment-2", + }, + ], + channel: "nightly", + schemaVersion: "1.9.0", + targeting: "true", + }); + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + await loader.init(); + + ok( + loader.manager.onRecipe.calledWith(PASS_FILTER_RECIPE, "rs-loader"), + "should call .onRecipe with this recipe" + ); + equal(loader.manager.onRecipe.callCount, 1, "should only call onRecipe once"); +}); + +add_task(async function test_updateRecipes_recipeAppId() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("mobile-experiment", { + appId: "org.mozilla.firefox", + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "mobile-feature", + value: { + enabled: true, + }, + }, + ], + }, + ], + }); + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + + await loader.init(); + + Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called"); + Assert.ok( + manager.onFinalize.calledWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); +}); + +add_task(async function test_updateRecipes_featureValidationOptOut() { + const invalidTestRecipe = ExperimentFakes.recipe("invalid-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { + enabled: "true", + testInt: false, + }, + }, + ], + }, + ], + }); + + const message = await PanelTestProvider.getMessages().then(msgs => + msgs.find(m => m.id === "SPOTLIGHT_MESSAGE_93") + ); + delete message.template; + + const invalidMsgRecipe = ExperimentFakes.recipe("invalid-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: message, + }, + ], + }, + ], + }); + + for (const invalidRecipe of [invalidTestRecipe, invalidMsgRecipe]) { + const optOutRecipe = { + ...invalidMsgRecipe, + slug: "optout-recipe", + featureValidationOptOut: true, + }; + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([invalidRecipe, optOutRecipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActive").returns([]); + sinon.stub(manager.store, "getAllRollouts").returns([]); + + await loader.init(); + ok( + manager.onRecipe.calledOnceWith(optOutRecipe, "rs-loader"), + "should call .onRecipe for the opt-out recipe" + ); + ok( + manager.onFinalize.calledOnceWith("rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([[invalidRecipe.slug, ["control"]]]), + invalidFeatures: new Map(), + validationEnabled: true, + }), + "should call .onFinalize with only one invalid recipe" + ); + } +}); + +add_task(async function test_updateRecipes_invalidFeature_mismatch() { + info( + "Testing that we do not submit validation telemetry when the targeting does not match" + ); + const recipe = ExperimentFakes.recipe("recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "bogus", + value: { + bogus: "bogus", + }, + }, + ], + }, + ], + targeting: "false", + }); + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActive").returns([]); + sinon.stub(manager.store, "getAllRollouts").returns([]); + + const telemetrySpy = sinon.stub(manager, "sendValidationFailedTelemetry"); + const targetingSpy = sinon.spy(loader, "checkTargeting"); + + await loader.init(); + ok(targetingSpy.calledOnce, "Should have checked targeting for recipe"); + ok( + !(await targetingSpy.returnValues[0]), + "Targeting should not have matched" + ); + ok(manager.onRecipe.notCalled, "should not call .onRecipe for the recipe"); + ok( + telemetrySpy.notCalled, + "Should not have submitted validation failed telemetry" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_SharedDataMap.js b/toolkit/components/nimbus/test/unit/test_SharedDataMap.js new file mode 100644 index 0000000000..06abad981d --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_SharedDataMap.js @@ -0,0 +1,207 @@ +const { SharedDataMap } = ChromeUtils.import( + "resource://nimbus/lib/SharedDataMap.jsm" +); +const { FileTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/FileTestUtils.sys.mjs" +); +const { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); + +const PATH = FileTestUtils.getTempFile("shared-data-map").path; + +function with_sharedDataMap(test) { + let testTask = async () => { + const sandbox = sinon.createSandbox(); + const instance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: true, + }); + try { + await test({ instance, sandbox }); + } finally { + sandbox.restore(); + } + }; + + // Copy the name of the test function to identify the test + Object.defineProperty(testTask, "name", { value: test.name }); + add_task(testTask); +} + +with_sharedDataMap(async function test_set_notify({ instance, sandbox }) { + await instance.init(); + let updateStub = sandbox.stub(); + + instance.on("parent-store-update:foo", updateStub); + instance.set("foo", "bar"); + + Assert.equal(updateStub.callCount, 1, "Update event sent"); + Assert.equal(updateStub.firstCall.args[1], "bar", "Update event sent value"); +}); + +with_sharedDataMap(async function test_set_child_notify({ instance, sandbox }) { + await instance.init(); + + let updateStub = sandbox.stub(); + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + childInstance.on("child-store-update:foo", updateStub); + let childStoreUpdate = new Promise(resolve => + childInstance.on("child-store-update:foo", resolve) + ); + instance.set("foo", "bar"); + + await childStoreUpdate; + + Assert.equal(updateStub.callCount, 1, "Update event sent"); + Assert.equal(updateStub.firstCall.args[1], "bar", "Update event sent value"); +}); + +with_sharedDataMap(async function test_async({ instance, sandbox }) { + const spy = sandbox.spy(instance._store, "load"); + await instance.init(); + + instance.set("foo", "bar"); + + Assert.equal(spy.callCount, 1, "Should init async"); + Assert.equal(instance.get("foo"), "bar", "It should retrieve a string value"); +}); + +with_sharedDataMap(async function test_saveSoon({ instance, sandbox }) { + await instance.init(); + const stub = sandbox.stub(instance._store, "saveSoon"); + + instance.set("foo", "bar"); + + Assert.equal(stub.callCount, 1, "Should call save soon when setting a value"); +}); + +with_sharedDataMap(async function test_init_safe({ instance, sandbox }) { + let stub = sandbox.stub(instance._store, "load"); + sandbox.replaceGetter(instance._store, "data", () => { + throw new Error("expected xpcshell"); + }); + + try { + await instance.init(); + Assert.ok(stub.calledOnce, "Load should be called"); + } catch (e) { + Assert.ok(false, "Error should be caught in SharedDataMap"); + } +}); + +with_sharedDataMap(async function test_childInit({ instance, sandbox }) { + sandbox.stub(instance, "isParent").get(() => false); + const stubA = sandbox.stub(instance._store, "ensureDataReady"); + const stubB = sandbox.stub(instance._store, "load"); + + await instance.init(); + + Assert.equal( + stubA.callCount, + 0, + "It should not try to initialize sync from child" + ); + Assert.equal( + stubB.callCount, + 0, + "It should not try to initialize async from child" + ); +}); + +with_sharedDataMap(async function test_parentChildSync_synchronously({ + instance: parentInstance, + sandbox, +}) { + await parentInstance.init(); + parentInstance.set("foo", { bar: 1 }); + + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + await parentInstance.ready(); + await childInstance.ready(); + + await TestUtils.waitForCondition( + () => childInstance.get("foo"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("foo"), + parentInstance.get("foo"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_parentChildSync_async({ + instance: parentInstance, + sandbox, +}) { + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + await parentInstance.init(); + parentInstance.set("foo", { bar: 1 }); + + await parentInstance.ready(); + await childInstance.ready(); + + await TestUtils.waitForCondition( + () => childInstance.get("foo"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("foo"), + parentInstance.get("foo"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_earlyChildSync({ + instance: parentInstance, + sandbox, +}) { + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + Assert.equal(childInstance.has("baz"), false, "Should not fail"); + + await parentInstance.init(); + parentInstance.set("baz", { bar: 1 }); + + await TestUtils.waitForCondition( + () => childInstance.get("baz"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("baz"), + parentInstance.get("baz"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_updateStoreData({ instance, sandbox }) { + await instance.init(); + + Assert.ok(!instance.get("foo"), "No value initially"); + + instance.set("foo", "foo"); + instance.set("bar", "bar"); + instance._removeEntriesByKeys(["bar"]); + + Assert.ok(instance.get("foo"), "We keep one of the values"); + Assert.ok(!instance.get("bar"), "The other value is removed"); +}); diff --git a/toolkit/components/nimbus/test/unit/xpcshell.ini b/toolkit/components/nimbus/test/unit/xpcshell.ini new file mode 100644 index 0000000000..805da1b089 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/xpcshell.ini @@ -0,0 +1,27 @@ +[DEFAULT] +head = head.js +tags = nimbus +firefox-appdir = browser +support-files = + reference_aboutwelcome_experiment_content.json +skip-if = + toolkit == "android" + appname == "thunderbird" +run-sequentially = very high failure rate in parallel + +[test_ExperimentManager_context.js] +[test_ExperimentManager_enroll.js] +[test_ExperimentManager_lifecycle.js] +[test_ExperimentManager_unenroll.js] +[test_ExperimentManager_generateTestIds.js] +[test_ExperimentManager_prefs.js] +[test_ExperimentStore.js] +[test_NimbusTestUtils.js] +[test_SharedDataMap.js] +[test_ExperimentAPI.js] +[test_ExperimentAPI_ExperimentFeature.js] +[test_ExperimentAPI_ExperimentFeature_getAllVariables.js] +[test_ExperimentAPI_ExperimentFeature_getVariable.js] +[test_RemoteSettingsExperimentLoader.js] +[test_RemoteSettingsExperimentLoader_updateRecipes.js] +[test_ExperimentAPI_NimbusFeatures.js] |