diff options
Diffstat (limited to '')
| -rw-r--r-- | toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs | 493 |
1 files changed, 493 insertions, 0 deletions
diff --git a/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs new file mode 100644 index 0000000000..7be889ae06 --- /dev/null +++ b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs @@ -0,0 +1,493 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { ExperimentStore } from "resource://nimbus/lib/ExperimentStore.sys.mjs"; +import { FileTestUtils } from "resource://testing-common/FileTestUtils.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + FeatureManifest: "resource://nimbus/FeatureManifest.sys.mjs", + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + NormandyUtils: "resource://normandy/lib/NormandyUtils.sys.mjs", + _ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + _RemoteSettingsExperimentLoader: + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + NetUtil: "resource://gre/modules/NetUtil.jsm", +}); + +function fetchSchemaSync(uri) { + // Yes, this is doing a sync load, but this is only done *once* and we cache + // the result after *and* it is test-only. + const channel = lazy.NetUtil.newChannel({ + uri, + loadUsingSystemPrincipal: true, + }); + const stream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance( + Ci.nsIScriptableInputStream + ); + + stream.init(channel.open()); + + const available = stream.available(); + const json = stream.read(available); + stream.close(); + + return JSON.parse(json); +} + +XPCOMUtils.defineLazyGetter(lazy, "enrollmentSchema", () => { + return fetchSchemaSync( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); +}); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; + +const PATH = FileTestUtils.getTempFile("shared-data-map").path; + +async function fetchSchema(url) { + const response = await fetch(url); + const schema = await response.json(); + if (!schema) { + throw new Error(`Failed to load ${url}`); + } + return schema; +} + +export const ExperimentTestUtils = { + _validateSchema(schema, value, errorMsg) { + const result = lazy.JsonSchema.validate(value, schema, { + shortCircuit: false, + }); + if (result.errors.length) { + throw new Error( + `${errorMsg}: ${JSON.stringify(result.errors, undefined, 2)}` + ); + } + return value; + }, + + _validateFeatureValueEnum({ branch }) { + let { features } = branch; + for (let feature of features) { + // If we're not using a real feature skip this check + if (!lazy.FeatureManifest[feature.featureId]) { + return true; + } + let { variables } = lazy.FeatureManifest[feature.featureId]; + for (let varName of Object.keys(variables)) { + let varValue = feature.value[varName]; + if ( + varValue && + variables[varName].enum && + !variables[varName].enum.includes(varValue) + ) { + throw new Error( + `${varName} should have one of the following values: ${JSON.stringify( + variables[varName].enum + )} but has value '${varValue}'` + ); + } + } + } + return true; + }, + + /** + * Checks if an experiment is valid acording to existing schema + */ + async validateExperiment(experiment) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusExperiment.schema.json" + ); + + // Ensure that the `featureIds` field is properly set + const { branches } = experiment; + branches.forEach(branch => { + branch.features.map(({ featureId }) => { + if (!experiment.featureIds.includes(featureId)) { + throw new Error( + `Branch(${branch.slug}) contains feature(${featureId}) but that's not declared in recipe(${experiment.slug}).featureIds` + ); + } + }); + }); + + return this._validateSchema( + schema, + experiment, + `Experiment ${experiment.slug} not valid` + ); + }, + validateEnrollment(enrollment) { + // We still have single feature experiment recipes for backwards + // compatibility testing but we don't do schema validation + if (!enrollment.branch.features && enrollment.branch.feature) { + return true; + } + + return ( + this._validateFeatureValueEnum(enrollment) && + this._validateSchema( + lazy.enrollmentSchema, + enrollment, + `Enrollment ${enrollment.slug} is not valid` + ) + ); + }, + async validateRollouts(rollout) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); + + return this._validateSchema( + schema, + rollout, + `Rollout configuration ${rollout.slug} is not valid` + ); + }, + /** + * Add features for tests. + * + * These features will only be visible to the JS Nimbus client. The native + * Nimbus client will have no access. + * + * @params features A list of |_NimbusFeature|s. + * + * @returns A cleanup function to remove the features once the test has completed. + */ + addTestFeatures(...features) { + for (const feature of features) { + if (Object.hasOwn(lazy.NimbusFeatures, feature.featureId)) { + throw new Error( + `Cannot add feature ${feature.featureId} -- a feature with this ID already exists!` + ); + } + lazy.NimbusFeatures[feature.featureId] = feature; + } + return () => { + for (const { featureId } of features) { + delete lazy.NimbusFeatures[featureId]; + } + }; + }, +}; + +export const ExperimentFakes = { + manager(store) { + let sandbox = lazy.sinon.createSandbox(); + let manager = new lazy._ExperimentManager({ store: store || this.store() }); + // We want calls to `store.addEnrollment` to implicitly validate the + // enrollment before saving to store + let origAddExperiment = manager.store.addEnrollment.bind(manager.store); + sandbox.stub(manager.store, "addEnrollment").callsFake(async enrollment => { + await ExperimentTestUtils.validateEnrollment(enrollment); + return origAddExperiment(enrollment); + }); + + return manager; + }, + store() { + return new ExperimentStore("FakeStore", { + path: PATH, + isParent: true, + }); + }, + waitForExperimentUpdate(ExperimentAPI, slug) { + return new Promise(resolve => + ExperimentAPI._store.once(`update:${slug}`, resolve) + ); + }, + async enrollWithRollout( + featureConfig, + { manager = lazy.ExperimentAPI._manager, source } = {} + ) { + await manager.store.init(); + const rollout = this.rollout(`${featureConfig.featureId}-rollout`, { + branch: { + slug: `${featureConfig.featureId}-rollout-branch`, + features: [featureConfig], + }, + }); + if (source) { + rollout.source = source; + } + await ExperimentTestUtils.validateRollouts(rollout); + // After storing the remote configuration to store and updating the feature + // we want to flush so that NimbusFeature usage in content process also + // receives the update + await manager.store.addEnrollment(rollout); + manager.store._syncToChildren({ flush: true }); + + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, enrollment) => { + if (enrollment.slug === rollout.slug && !enrollment.active) { + manager.store._deleteForTests(rollout.slug); + resolve(); + } + }) + ); + + return () => { + let promise = unenrollCompleted(rollout.slug); + manager.unenroll(rollout.slug, "cleanup"); + return promise; + }; + }, + /** + * Enroll in an experiment branch with the given feature configuration. + * + * NB: It is unnecessary to await the enrollmentPromise. + * See bug 1773583 and bug 1829412. + */ + async enrollWithFeatureConfig( + featureConfig, + { manager = lazy.ExperimentAPI._manager, isRollout = false } = {} + ) { + await manager.store.ready(); + // Use id passed in featureConfig value to compute experimentId + // This help filter telemetry events (such as expose) in race conditions when telemetry + // from multiple experiments with same featureId co-exist in snapshot + let experimentId = `${featureConfig.featureId}${ + featureConfig?.value?.id ? "-" + featureConfig?.value?.id : "" + }-experiment-${Math.random()}`; + + let recipe = this.recipe(experimentId, { + bucketConfig: { + namespace: "mstest-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + branches: [ + { + slug: "control", + ratio: 1, + features: [featureConfig], + }, + ], + isRollout, + }); + let { enrollmentPromise, doExperimentCleanup } = this.enrollmentHelper( + recipe, + { manager } + ); + + await enrollmentPromise; + + return doExperimentCleanup; + }, + /** + * Enroll in the given recipe. + * + * NB: It is unnecessary to await the enrollmentPromise. + * See bug 1773583 and bug 1829412. + */ + enrollmentHelper( + recipe, + { manager = lazy.ExperimentAPI._manager, source = "enrollmentHelper" } = {} + ) { + if (!recipe?.slug) { + throw new Error("Enrollment helper expects a recipe"); + } + + let enrollmentPromise = new Promise(resolve => + manager.store.on(`update:${recipe.slug}`, (event, experiment) => { + if (experiment.active) { + manager.store._syncToChildren({ flush: true }); + resolve(experiment); + } + }) + ); + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(recipe.slug); + resolve(); + } + }) + ); + let doExperimentCleanup = async () => { + const experiment = manager.store.get(recipe.slug); + let promise = unenrollCompleted(experiment.slug); + manager.unenroll(experiment.slug, "cleanup"); + await promise; + }; + + if (!manager.store._isReady) { + throw new Error("Manager store not ready, call `manager.onStartup`"); + } + manager.enroll(recipe, source); + + return { enrollmentPromise, doExperimentCleanup }; + }, + async cleanupAll(slugs, { manager = lazy.ExperimentAPI._manager } = {}) { + function unenrollCompleted(slug) { + return new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(slug); + resolve(); + } + }) + ); + } + + for (const slug of slugs) { + let promise = unenrollCompleted(slug); + manager.unenroll(slug, "cleanup"); + await promise; + } + + if (manager.store.getAllActiveExperiments().length) { + throw new Error("Cleanup failed"); + } + }, + // Experiment store caches in prefs Enrollments for fast sync access + cleanupStorePrefCache() { + try { + Services.prefs.deleteBranch(SYNC_DATA_PREF_BRANCH); + Services.prefs.deleteBranch(SYNC_DEFAULTS_PREF_BRANCH); + } catch (e) { + // Expected if nothing is cached + } + }, + childStore() { + return new ExperimentStore("FakeStore", { isParent: false }); + }, + rsLoader() { + const loader = new lazy._RemoteSettingsExperimentLoader(); + // Replace RS client with a fake + Object.defineProperty(loader, "remoteSettingsClient", { + value: { get: () => Promise.resolve([]) }, + }); + // Replace xman with a fake + loader.manager = this.manager(); + + return loader; + }, + experiment(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "NimbusTestUtils", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: props?.branch?.features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, + rollout(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + isRollout: true, + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "rollout", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: (props?.branch?.features || props?.features)?.map( + f => f.featureId + ) || ["testFeature"], + ...props, + }; + }, + recipe(slug = lazy.NormandyUtils.generateUuid(), props = {}) { + return { + // This field is required for populating remote settings + id: lazy.NormandyUtils.generateUuid(), + schemaVersion: "1.7.0", + appName: "firefox_desktop", + appId: "firefox-desktop", + channel: "nightly", + slug, + isEnrollmentPaused: false, + probeSets: [], + startDate: null, + endDate: null, + proposedEnrollment: 7, + referenceBranch: "control", + application: "firefox-desktop", + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + ], + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 100, + total: 1000, + }, + userFacingName: "Nimbus recipe", + userFacingDescription: "NimbusTestUtils recipe", + featureIds: props?.branches?.[0].features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, +}; |