summaryrefslogtreecommitdiffstats
path: root/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs493
1 files changed, 493 insertions, 0 deletions
diff --git a/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs
new file mode 100644
index 0000000000..7be889ae06
--- /dev/null
+++ b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs
@@ -0,0 +1,493 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { ExperimentStore } from "resource://nimbus/lib/ExperimentStore.sys.mjs";
+import { FileTestUtils } from "resource://testing-common/FileTestUtils.sys.mjs";
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs",
+ FeatureManifest: "resource://nimbus/FeatureManifest.sys.mjs",
+ JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs",
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs",
+ NormandyUtils: "resource://normandy/lib/NormandyUtils.sys.mjs",
+ _ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs",
+ _RemoteSettingsExperimentLoader:
+ "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs",
+ sinon: "resource://testing-common/Sinon.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NetUtil: "resource://gre/modules/NetUtil.jsm",
+});
+
+function fetchSchemaSync(uri) {
+ // Yes, this is doing a sync load, but this is only done *once* and we cache
+ // the result after *and* it is test-only.
+ const channel = lazy.NetUtil.newChannel({
+ uri,
+ loadUsingSystemPrincipal: true,
+ });
+ const stream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(
+ Ci.nsIScriptableInputStream
+ );
+
+ stream.init(channel.open());
+
+ const available = stream.available();
+ const json = stream.read(available);
+ stream.close();
+
+ return JSON.parse(json);
+}
+
+XPCOMUtils.defineLazyGetter(lazy, "enrollmentSchema", () => {
+ return fetchSchemaSync(
+ "resource://nimbus/schemas/NimbusEnrollment.schema.json"
+ );
+});
+
+const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore;
+
+const PATH = FileTestUtils.getTempFile("shared-data-map").path;
+
+async function fetchSchema(url) {
+ const response = await fetch(url);
+ const schema = await response.json();
+ if (!schema) {
+ throw new Error(`Failed to load ${url}`);
+ }
+ return schema;
+}
+
+export const ExperimentTestUtils = {
+ _validateSchema(schema, value, errorMsg) {
+ const result = lazy.JsonSchema.validate(value, schema, {
+ shortCircuit: false,
+ });
+ if (result.errors.length) {
+ throw new Error(
+ `${errorMsg}: ${JSON.stringify(result.errors, undefined, 2)}`
+ );
+ }
+ return value;
+ },
+
+ _validateFeatureValueEnum({ branch }) {
+ let { features } = branch;
+ for (let feature of features) {
+ // If we're not using a real feature skip this check
+ if (!lazy.FeatureManifest[feature.featureId]) {
+ return true;
+ }
+ let { variables } = lazy.FeatureManifest[feature.featureId];
+ for (let varName of Object.keys(variables)) {
+ let varValue = feature.value[varName];
+ if (
+ varValue &&
+ variables[varName].enum &&
+ !variables[varName].enum.includes(varValue)
+ ) {
+ throw new Error(
+ `${varName} should have one of the following values: ${JSON.stringify(
+ variables[varName].enum
+ )} but has value '${varValue}'`
+ );
+ }
+ }
+ }
+ return true;
+ },
+
+ /**
+ * Checks if an experiment is valid acording to existing schema
+ */
+ async validateExperiment(experiment) {
+ const schema = await fetchSchema(
+ "resource://nimbus/schemas/NimbusExperiment.schema.json"
+ );
+
+ // Ensure that the `featureIds` field is properly set
+ const { branches } = experiment;
+ branches.forEach(branch => {
+ branch.features.map(({ featureId }) => {
+ if (!experiment.featureIds.includes(featureId)) {
+ throw new Error(
+ `Branch(${branch.slug}) contains feature(${featureId}) but that's not declared in recipe(${experiment.slug}).featureIds`
+ );
+ }
+ });
+ });
+
+ return this._validateSchema(
+ schema,
+ experiment,
+ `Experiment ${experiment.slug} not valid`
+ );
+ },
+ validateEnrollment(enrollment) {
+ // We still have single feature experiment recipes for backwards
+ // compatibility testing but we don't do schema validation
+ if (!enrollment.branch.features && enrollment.branch.feature) {
+ return true;
+ }
+
+ return (
+ this._validateFeatureValueEnum(enrollment) &&
+ this._validateSchema(
+ lazy.enrollmentSchema,
+ enrollment,
+ `Enrollment ${enrollment.slug} is not valid`
+ )
+ );
+ },
+ async validateRollouts(rollout) {
+ const schema = await fetchSchema(
+ "resource://nimbus/schemas/NimbusEnrollment.schema.json"
+ );
+
+ return this._validateSchema(
+ schema,
+ rollout,
+ `Rollout configuration ${rollout.slug} is not valid`
+ );
+ },
+ /**
+ * Add features for tests.
+ *
+ * These features will only be visible to the JS Nimbus client. The native
+ * Nimbus client will have no access.
+ *
+ * @params features A list of |_NimbusFeature|s.
+ *
+ * @returns A cleanup function to remove the features once the test has completed.
+ */
+ addTestFeatures(...features) {
+ for (const feature of features) {
+ if (Object.hasOwn(lazy.NimbusFeatures, feature.featureId)) {
+ throw new Error(
+ `Cannot add feature ${feature.featureId} -- a feature with this ID already exists!`
+ );
+ }
+ lazy.NimbusFeatures[feature.featureId] = feature;
+ }
+ return () => {
+ for (const { featureId } of features) {
+ delete lazy.NimbusFeatures[featureId];
+ }
+ };
+ },
+};
+
+export const ExperimentFakes = {
+ manager(store) {
+ let sandbox = lazy.sinon.createSandbox();
+ let manager = new lazy._ExperimentManager({ store: store || this.store() });
+ // We want calls to `store.addEnrollment` to implicitly validate the
+ // enrollment before saving to store
+ let origAddExperiment = manager.store.addEnrollment.bind(manager.store);
+ sandbox.stub(manager.store, "addEnrollment").callsFake(async enrollment => {
+ await ExperimentTestUtils.validateEnrollment(enrollment);
+ return origAddExperiment(enrollment);
+ });
+
+ return manager;
+ },
+ store() {
+ return new ExperimentStore("FakeStore", {
+ path: PATH,
+ isParent: true,
+ });
+ },
+ waitForExperimentUpdate(ExperimentAPI, slug) {
+ return new Promise(resolve =>
+ ExperimentAPI._store.once(`update:${slug}`, resolve)
+ );
+ },
+ async enrollWithRollout(
+ featureConfig,
+ { manager = lazy.ExperimentAPI._manager, source } = {}
+ ) {
+ await manager.store.init();
+ const rollout = this.rollout(`${featureConfig.featureId}-rollout`, {
+ branch: {
+ slug: `${featureConfig.featureId}-rollout-branch`,
+ features: [featureConfig],
+ },
+ });
+ if (source) {
+ rollout.source = source;
+ }
+ await ExperimentTestUtils.validateRollouts(rollout);
+ // After storing the remote configuration to store and updating the feature
+ // we want to flush so that NimbusFeature usage in content process also
+ // receives the update
+ await manager.store.addEnrollment(rollout);
+ manager.store._syncToChildren({ flush: true });
+
+ let unenrollCompleted = slug =>
+ new Promise(resolve =>
+ manager.store.on(`update:${slug}`, (event, enrollment) => {
+ if (enrollment.slug === rollout.slug && !enrollment.active) {
+ manager.store._deleteForTests(rollout.slug);
+ resolve();
+ }
+ })
+ );
+
+ return () => {
+ let promise = unenrollCompleted(rollout.slug);
+ manager.unenroll(rollout.slug, "cleanup");
+ return promise;
+ };
+ },
+ /**
+ * Enroll in an experiment branch with the given feature configuration.
+ *
+ * NB: It is unnecessary to await the enrollmentPromise.
+ * See bug 1773583 and bug 1829412.
+ */
+ async enrollWithFeatureConfig(
+ featureConfig,
+ { manager = lazy.ExperimentAPI._manager, isRollout = false } = {}
+ ) {
+ await manager.store.ready();
+ // Use id passed in featureConfig value to compute experimentId
+ // This help filter telemetry events (such as expose) in race conditions when telemetry
+ // from multiple experiments with same featureId co-exist in snapshot
+ let experimentId = `${featureConfig.featureId}${
+ featureConfig?.value?.id ? "-" + featureConfig?.value?.id : ""
+ }-experiment-${Math.random()}`;
+
+ let recipe = this.recipe(experimentId, {
+ bucketConfig: {
+ namespace: "mstest-utils",
+ randomizationUnit: "normandy_id",
+ start: 0,
+ count: 1000,
+ total: 1000,
+ },
+ branches: [
+ {
+ slug: "control",
+ ratio: 1,
+ features: [featureConfig],
+ },
+ ],
+ isRollout,
+ });
+ let { enrollmentPromise, doExperimentCleanup } = this.enrollmentHelper(
+ recipe,
+ { manager }
+ );
+
+ await enrollmentPromise;
+
+ return doExperimentCleanup;
+ },
+ /**
+ * Enroll in the given recipe.
+ *
+ * NB: It is unnecessary to await the enrollmentPromise.
+ * See bug 1773583 and bug 1829412.
+ */
+ enrollmentHelper(
+ recipe,
+ { manager = lazy.ExperimentAPI._manager, source = "enrollmentHelper" } = {}
+ ) {
+ if (!recipe?.slug) {
+ throw new Error("Enrollment helper expects a recipe");
+ }
+
+ let enrollmentPromise = new Promise(resolve =>
+ manager.store.on(`update:${recipe.slug}`, (event, experiment) => {
+ if (experiment.active) {
+ manager.store._syncToChildren({ flush: true });
+ resolve(experiment);
+ }
+ })
+ );
+ let unenrollCompleted = slug =>
+ new Promise(resolve =>
+ manager.store.on(`update:${slug}`, (event, experiment) => {
+ if (!experiment.active) {
+ // Removes recipe from file storage which
+ // (normally the users archive of past experiments)
+ manager.store._deleteForTests(recipe.slug);
+ resolve();
+ }
+ })
+ );
+ let doExperimentCleanup = async () => {
+ const experiment = manager.store.get(recipe.slug);
+ let promise = unenrollCompleted(experiment.slug);
+ manager.unenroll(experiment.slug, "cleanup");
+ await promise;
+ };
+
+ if (!manager.store._isReady) {
+ throw new Error("Manager store not ready, call `manager.onStartup`");
+ }
+ manager.enroll(recipe, source);
+
+ return { enrollmentPromise, doExperimentCleanup };
+ },
+ async cleanupAll(slugs, { manager = lazy.ExperimentAPI._manager } = {}) {
+ function unenrollCompleted(slug) {
+ return new Promise(resolve =>
+ manager.store.on(`update:${slug}`, (event, experiment) => {
+ if (!experiment.active) {
+ // Removes recipe from file storage which
+ // (normally the users archive of past experiments)
+ manager.store._deleteForTests(slug);
+ resolve();
+ }
+ })
+ );
+ }
+
+ for (const slug of slugs) {
+ let promise = unenrollCompleted(slug);
+ manager.unenroll(slug, "cleanup");
+ await promise;
+ }
+
+ if (manager.store.getAllActiveExperiments().length) {
+ throw new Error("Cleanup failed");
+ }
+ },
+ // Experiment store caches in prefs Enrollments for fast sync access
+ cleanupStorePrefCache() {
+ try {
+ Services.prefs.deleteBranch(SYNC_DATA_PREF_BRANCH);
+ Services.prefs.deleteBranch(SYNC_DEFAULTS_PREF_BRANCH);
+ } catch (e) {
+ // Expected if nothing is cached
+ }
+ },
+ childStore() {
+ return new ExperimentStore("FakeStore", { isParent: false });
+ },
+ rsLoader() {
+ const loader = new lazy._RemoteSettingsExperimentLoader();
+ // Replace RS client with a fake
+ Object.defineProperty(loader, "remoteSettingsClient", {
+ value: { get: () => Promise.resolve([]) },
+ });
+ // Replace xman with a fake
+ loader.manager = this.manager();
+
+ return loader;
+ },
+ experiment(slug, props = {}) {
+ return {
+ slug,
+ active: true,
+ enrollmentId: lazy.NormandyUtils.generateUuid(),
+ branch: {
+ slug: "treatment",
+ features: [
+ {
+ featureId: "testFeature",
+ value: { testInt: 123, enabled: true },
+ },
+ ],
+ ...props,
+ },
+ source: "NimbusTestUtils",
+ isEnrollmentPaused: true,
+ experimentType: "NimbusTestUtils",
+ userFacingName: "NimbusTestUtils",
+ userFacingDescription: "NimbusTestUtils",
+ lastSeen: new Date().toJSON(),
+ featureIds: props?.branch?.features?.map(f => f.featureId) || [
+ "testFeature",
+ ],
+ ...props,
+ };
+ },
+ rollout(slug, props = {}) {
+ return {
+ slug,
+ active: true,
+ enrollmentId: lazy.NormandyUtils.generateUuid(),
+ isRollout: true,
+ branch: {
+ slug: "treatment",
+ features: [
+ {
+ featureId: "testFeature",
+ value: { testInt: 123, enabled: true },
+ },
+ ],
+ ...props,
+ },
+ source: "NimbusTestUtils",
+ isEnrollmentPaused: true,
+ experimentType: "rollout",
+ userFacingName: "NimbusTestUtils",
+ userFacingDescription: "NimbusTestUtils",
+ lastSeen: new Date().toJSON(),
+ featureIds: (props?.branch?.features || props?.features)?.map(
+ f => f.featureId
+ ) || ["testFeature"],
+ ...props,
+ };
+ },
+ recipe(slug = lazy.NormandyUtils.generateUuid(), props = {}) {
+ return {
+ // This field is required for populating remote settings
+ id: lazy.NormandyUtils.generateUuid(),
+ schemaVersion: "1.7.0",
+ appName: "firefox_desktop",
+ appId: "firefox-desktop",
+ channel: "nightly",
+ slug,
+ isEnrollmentPaused: false,
+ probeSets: [],
+ startDate: null,
+ endDate: null,
+ proposedEnrollment: 7,
+ referenceBranch: "control",
+ application: "firefox-desktop",
+ branches: [
+ {
+ slug: "control",
+ ratio: 1,
+ features: [
+ {
+ featureId: "testFeature",
+ value: { testInt: 123, enabled: true },
+ },
+ ],
+ },
+ {
+ slug: "treatment",
+ ratio: 1,
+ features: [
+ {
+ featureId: "testFeature",
+ value: { testInt: 123, enabled: true },
+ },
+ ],
+ },
+ ],
+ bucketConfig: {
+ namespace: "nimbus-test-utils",
+ randomizationUnit: "normandy_id",
+ start: 0,
+ count: 100,
+ total: 1000,
+ },
+ userFacingName: "Nimbus recipe",
+ userFacingDescription: "NimbusTestUtils recipe",
+ featureIds: props?.branches?.[0].features?.map(f => f.featureId) || [
+ "testFeature",
+ ],
+ ...props,
+ };
+ },
+};