diff options
Diffstat (limited to 'toolkit/components/nimbus')
57 files changed, 20871 insertions, 0 deletions
diff --git a/toolkit/components/nimbus/ExperimentAPI.sys.mjs b/toolkit/components/nimbus/ExperimentAPI.sys.mjs new file mode 100644 index 0000000000..d31b99c8f8 --- /dev/null +++ b/toolkit/components/nimbus/ExperimentAPI.sys.mjs @@ -0,0 +1,672 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + ExperimentStore: "resource://nimbus/lib/ExperimentStore.sys.mjs", + FeatureManifest: "resource://nimbus/FeatureManifest.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", +}); + +const IS_MAIN_PROCESS = + Services.appinfo.processType === Services.appinfo.PROCESS_TYPE_DEFAULT; + +const COLLECTION_ID_PREF = "messaging-system.rsexperimentloader.collection_id"; +const COLLECTION_ID_FALLBACK = "nimbus-desktop-experiments"; +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "COLLECTION_ID", + COLLECTION_ID_PREF, + COLLECTION_ID_FALLBACK +); +const EXPOSURE_EVENT_CATEGORY = "normandy"; +const EXPOSURE_EVENT_METHOD = "expose"; +const EXPOSURE_EVENT_OBJECT = "nimbus_experiment"; + +function parseJSON(value) { + if (value) { + try { + return JSON.parse(value); + } catch (e) { + console.error(e); + } + } + return null; +} + +function featuresCompat(branch) { + if (!branch) { + return []; + } + let { features } = branch; + // In <=v1.5.0 of the Nimbus API, experiments had single feature + if (!features) { + features = [branch.feature]; + } + + return features; +} + +function getBranchFeature(enrollment, targetFeatureId) { + return featuresCompat(enrollment.branch).find( + ({ featureId }) => featureId === targetFeatureId + ); +} + +const experimentBranchAccessor = { + get: (target, prop) => { + // Offer an API where we can access `branch.feature.*`. + // This is a useful shorthand that hides the fact that + // even single-feature recipes are still represented + // as an array with 1 item + if (!(prop in target) && target.features) { + return target.features.find(f => f.featureId === prop); + } else if (target.feature?.featureId === prop) { + // Backwards compatibility for version 1.6.2 and older + return target.feature; + } + + return target[prop]; + }, +}; + +export const ExperimentAPI = { + /** + * @returns {Promise} Resolves when the API has synchronized to the main store + */ + ready() { + return this._store.ready(); + }, + + /** + * Returns an experiment, including all its metadata + * Sends exposure event + * + * @param {{slug?: string, featureId?: string}} options slug = An experiment identifier + * or feature = a stable identifier for a type of experiment + * @returns {{slug: string, active: bool}} A matching experiment if one is found. + */ + getExperiment({ slug, featureId } = {}) { + if (!slug && !featureId) { + throw new Error( + "getExperiment(options) must include a slug or a feature." + ); + } + let experimentData; + try { + if (slug) { + experimentData = this._store.get(slug); + } else if (featureId) { + experimentData = this._store.getExperimentForFeature(featureId); + } + } catch (e) { + console.error(e); + } + if (experimentData) { + return { + slug: experimentData.slug, + active: experimentData.active, + branch: new Proxy(experimentData.branch, experimentBranchAccessor), + }; + } + + return null; + }, + + /** + * Used by getExperimentMetaData and getRolloutMetaData + * + * @param {{slug: string, featureId: string}} options Enrollment identifier + * @param isRollout Is enrollment an experiment or a rollout + * @returns {object} Enrollment metadata + */ + getEnrollmentMetaData({ slug, featureId }, isRollout) { + if (!slug && !featureId) { + throw new Error( + "getExperiment(options) must include a slug or a feature." + ); + } + + let experimentData; + try { + if (slug) { + experimentData = this._store.get(slug); + } else if (featureId) { + if (isRollout) { + experimentData = this._store.getRolloutForFeature(featureId); + } else { + experimentData = this._store.getExperimentForFeature(featureId); + } + } + } catch (e) { + console.error(e); + } + if (experimentData) { + return { + slug: experimentData.slug, + active: experimentData.active, + branch: { slug: experimentData.branch.slug }, + }; + } + + return null; + }, + + /** + * Return experiment slug its status and the enrolled branch slug + * Does NOT send exposure event because you only have access to the slugs + */ + getExperimentMetaData(options) { + return this.getEnrollmentMetaData(options); + }, + + /** + * Return rollout slug its status and the enrolled branch slug + * Does NOT send exposure event because you only have access to the slugs + */ + getRolloutMetaData(options) { + return this.getEnrollmentMetaData(options, true); + }, + + /** + * Return FeatureConfig from first active experiment where it can be found + * @param {{slug: string, featureId: string }} + * @returns {Branch | null} + */ + getActiveBranch({ slug, featureId }) { + let experiment = null; + try { + if (slug) { + experiment = this._store.get(slug); + } else if (featureId) { + experiment = this._store.getExperimentForFeature(featureId); + } + } catch (e) { + console.error(e); + } + + if (!experiment) { + return null; + } + + // Default to null for feature-less experiments where we're only + // interested in exposure. + return experiment?.branch || null; + }, + + /** + * Deregisters an event listener. + * @param {string} eventName + * @param {function} callback + */ + off(eventName, callback) { + this._store.off(eventName, callback); + }, + + /** + * Returns the recipe for a given experiment slug + * + * This should noly be called from the main process. + * + * Note that the recipe is directly fetched from RemoteSettings, which has + * all the recipe metadata available without relying on the `this._store`. + * Therefore, calling this function does not require to call `this.ready()` first. + * + * @param slug {String} An experiment identifier + * @returns {Recipe|undefined} A matching experiment recipe if one is found + */ + async getRecipe(slug) { + if (!IS_MAIN_PROCESS) { + throw new Error( + "getRecipe() should only be called from the main process" + ); + } + + let recipe; + + try { + [recipe] = await this._remoteSettingsClient.get({ + // Do not sync the RS store, let RemoteSettingsExperimentLoader do that + syncIfEmpty: false, + filters: { slug }, + }); + } catch (e) { + // If an error occurs in .get(), an empty list is returned and the destructuring + // assignment will throw. + console.error(e); + recipe = undefined; + } + + return recipe; + }, + + /** + * Returns all the branches for a given experiment slug + * + * This should only be called from the main process. Like `getRecipe()`, + * calling this function does not require to call `this.ready()` first. + * + * @param slug {String} An experiment identifier + * @returns {[Branches]|undefined} An array of branches for the given slug + */ + async getAllBranches(slug) { + if (!IS_MAIN_PROCESS) { + throw new Error( + "getAllBranches() should only be called from the main process" + ); + } + + const recipe = await this.getRecipe(slug); + return recipe?.branches.map( + branch => new Proxy(branch, experimentBranchAccessor) + ); + }, + + recordExposureEvent({ featureId, experimentSlug, branchSlug }) { + Services.telemetry.setEventRecordingEnabled(EXPOSURE_EVENT_CATEGORY, true); + try { + Services.telemetry.recordEvent( + EXPOSURE_EVENT_CATEGORY, + EXPOSURE_EVENT_METHOD, + EXPOSURE_EVENT_OBJECT, + experimentSlug, + { + branchSlug, + featureId, + } + ); + } catch (e) { + console.error(e); + } + Glean.nimbusEvents.exposure.record({ + experiment: experimentSlug, + branch: branchSlug, + feature_id: featureId, + }); + }, +}; + +/** + * Singleton that holds lazy references to _ExperimentFeature instances + * defined by the FeatureManifest + */ +export const NimbusFeatures = {}; + +for (let feature in lazy.FeatureManifest) { + XPCOMUtils.defineLazyGetter(NimbusFeatures, feature, () => { + return new _ExperimentFeature(feature); + }); +} + +export class _ExperimentFeature { + constructor(featureId, manifest) { + this.featureId = featureId; + this.prefGetters = {}; + this.manifest = manifest || lazy.FeatureManifest[featureId]; + if (!this.manifest) { + console.error( + `No manifest entry for ${featureId}. Please add one to toolkit/components/nimbus/FeatureManifest.yaml` + ); + } + this._didSendExposureEvent = false; + const variables = this.manifest?.variables || {}; + + Object.keys(variables).forEach(key => { + const { type, fallbackPref } = variables[key]; + if (fallbackPref) { + XPCOMUtils.defineLazyPreferenceGetter( + this.prefGetters, + key, + fallbackPref, + null, + () => { + ExperimentAPI._store._emitFeatureUpdate( + this.featureId, + "pref-updated" + ); + }, + type === "json" ? parseJSON : val => val + ); + } + }); + } + + getSetPrefName(variable) { + return this.manifest?.variables?.[variable]?.setPref; + } + + getFallbackPrefName(variable) { + return this.manifest?.variables?.[variable]?.fallbackPref; + } + + /** + * Wait for ExperimentStore to load giving access to experiment features that + * do not have a pref cache + */ + ready() { + return ExperimentAPI.ready(); + } + + /** + * Lookup feature variables in experiments, rollouts, and fallback prefs. + * @param {{defaultValues?: {[variableName: string]: any}}} options + * @returns {{[variableName: string]: any}} The feature value + */ + getAllVariables({ defaultValues = null } = {}) { + let enrollment = null; + try { + enrollment = ExperimentAPI._store.getExperimentForFeature(this.featureId); + } catch (e) { + console.error(e); + } + let featureValue = this._getLocalizedValue(enrollment); + + if (typeof featureValue === "undefined") { + try { + enrollment = ExperimentAPI._store.getRolloutForFeature(this.featureId); + } catch (e) { + console.error(e); + } + featureValue = this._getLocalizedValue(enrollment); + } + + return { + ...this.prefGetters, + ...defaultValues, + ...featureValue, + }; + } + + getVariable(variable) { + if (!this.manifest?.variables?.[variable]) { + // Only throw in nightly/tests + if (Cu.isInAutomation || AppConstants.NIGHTLY_BUILD) { + throw new Error( + `Nimbus: Warning - variable "${variable}" is not defined in FeatureManifest.yaml` + ); + } + } + + // Next, check if an experiment is defined + let enrollment = null; + try { + enrollment = ExperimentAPI._store.getExperimentForFeature(this.featureId); + } catch (e) { + console.error(e); + } + let value = this._getLocalizedValue(enrollment, variable); + if (typeof value !== "undefined") { + return value; + } + + // Next, check for a rollout. + try { + enrollment = ExperimentAPI._store.getRolloutForFeature(this.featureId); + } catch (e) { + console.error(e); + } + value = this._getLocalizedValue(enrollment, variable); + if (typeof value !== "undefined") { + return value; + } + + // Return the default preference value + const prefName = this.getFallbackPrefName(variable); + return prefName ? this.prefGetters[variable] : undefined; + } + + getRollout() { + let remoteConfig = ExperimentAPI._store.getRolloutForFeature( + this.featureId + ); + if (!remoteConfig) { + return null; + } + + if (remoteConfig.branch?.features) { + return remoteConfig.branch?.features.find( + f => f.featureId === this.featureId + ); + } + + // This path is deprecated and will be removed in the future + if (remoteConfig.branch?.feature) { + return remoteConfig.branch.feature; + } + + return null; + } + + recordExposureEvent({ once = false } = {}) { + if (once && this._didSendExposureEvent) { + return; + } + + let enrollmentData = ExperimentAPI.getExperimentMetaData({ + featureId: this.featureId, + }); + if (!enrollmentData) { + enrollmentData = ExperimentAPI.getRolloutMetaData({ + featureId: this.featureId, + }); + } + + // Exposure only sent if user is enrolled in an experiment + if (enrollmentData) { + ExperimentAPI.recordExposureEvent({ + featureId: this.featureId, + experimentSlug: enrollmentData.slug, + branchSlug: enrollmentData.branch?.slug, + }); + this._didSendExposureEvent = true; + } + } + + onUpdate(callback) { + ExperimentAPI._store._onFeatureUpdate(this.featureId, callback); + } + + offUpdate(callback) { + ExperimentAPI._store._offFeatureUpdate(this.featureId, callback); + } + + /** + * The applications this feature applies to. + * + */ + get applications() { + return this.manifest.applications ?? ["firefox-desktop"]; + } + + debug() { + return { + variables: this.getAllVariables(), + experiment: ExperimentAPI.getExperimentMetaData({ + featureId: this.featureId, + }), + fallbackPrefs: Object.keys(this.prefGetters).map(prefName => [ + prefName, + this.prefGetters[prefName], + ]), + rollouts: this.getRollout(), + }; + } + + /** + * Do recursive locale substitution on the values, if applicable. + * + * If there are no localizations provided, the value will be returned as-is. + * + * If the value is an object containing an $l10n key, its substitution will be + * returned. + * + * Otherwise, the value will be recursively substituted. + * + * @param {unknown} values The values to perform substitutions upon. + * @param {Record<string, string>} localizations The localization + * substitutions for a specific locale. + * @param {Set<string>?} missingIds An optional set to collect all the IDs of + * all missing l10n entries. + * + * @returns {any} The values, potentially locale substituted. + */ + static substituteLocalizations( + values, + localizations, + missingIds = undefined + ) { + const result = _ExperimentFeature._substituteLocalizations( + values, + localizations, + missingIds + ); + + if (missingIds?.size) { + throw new ExperimentLocalizationError("l10n-missing-entry"); + } + + return result; + } + + /** + * The implementation of localization substitution. + * + * @param {unknown} values The values to perform substitutions upon. + * @param {Record<string, string>} localizations The localization + * substitutions for a specific locale. + * @param {Set<string>?} missingIds An optional set to collect all the IDs of + * all missing l10n entries. + * + * @returns {any} The values, potentially locale substituted. + */ + static _substituteLocalizations(values, localizations, missingIds) { + // If the recipe is not localized, we don't need to do anything. + // Likewise, if the value we are attempting to localize is not an object, + // there is nothing to localize. + if ( + typeof localizations === "undefined" || + typeof values !== "object" || + values === null + ) { + return values; + } + + if (Array.isArray(values)) { + return values.map(value => + _ExperimentFeature._substituteLocalizations( + value, + localizations, + missingIds + ) + ); + } + + const substituted = Object.assign({}, values); + + for (const [key, value] of Object.entries(values)) { + if ( + key === "$l10n" && + typeof value === "object" && + value !== null && + value?.id + ) { + if (!Object.hasOwn(localizations, value.id)) { + if (missingIds) { + missingIds.add(value.id); + break; + } else { + throw new ExperimentLocalizationError("l10n-missing-entry"); + } + } + + return localizations[value.id]; + } + + substituted[key] = _ExperimentFeature._substituteLocalizations( + value, + localizations, + missingIds + ); + } + + return substituted; + } + + /** + * Return a value (or all values) from an enrollment, potentially localized. + * + * @param {Enrollment} enrollment - The enrollment to query for the value or values. + * @param {string?} variable - The name of the variable to query for. If not + * provided, all variables will be returned. + * + * @returns {any} The value for the variable(s) in question. + */ + _getLocalizedValue(enrollment, variable = undefined) { + if (enrollment) { + const locale = Services.locale.appLocaleAsBCP47; + + if ( + typeof enrollment.localizations === "object" && + enrollment.localizations !== null && + (typeof enrollment.localizations[locale] !== "object" || + enrollment.localizations[locale] === null) + ) { + ExperimentAPI._manager.unenroll(enrollment.slug, "l10n-missing-locale"); + return undefined; + } + + const allValues = getBranchFeature(enrollment, this.featureId)?.value; + const value = + typeof variable === "undefined" ? allValues : allValues?.[variable]; + + if (typeof value !== "undefined") { + try { + return _ExperimentFeature.substituteLocalizations( + value, + enrollment.localizations?.[locale] + ); + } catch (e) { + // This should never happen. + if (e instanceof ExperimentLocalizationError) { + ExperimentAPI._manager.unenroll(enrollment.slug, e.reason); + } else { + throw e; + } + } + } + } + + return undefined; + } +} + +XPCOMUtils.defineLazyGetter(ExperimentAPI, "_manager", function () { + return lazy.ExperimentManager; +}); + +XPCOMUtils.defineLazyGetter(ExperimentAPI, "_store", function () { + return IS_MAIN_PROCESS + ? lazy.ExperimentManager.store + : new lazy.ExperimentStore(); +}); + +XPCOMUtils.defineLazyGetter( + ExperimentAPI, + "_remoteSettingsClient", + function () { + return lazy.RemoteSettings(lazy.COLLECTION_ID); + } +); + +class ExperimentLocalizationError extends Error { + constructor(reason) { + super(`Localized experiment error (${reason})`); + this.reason = reason; + } +} diff --git a/toolkit/components/nimbus/FeatureManifest.yaml b/toolkit/components/nimbus/FeatureManifest.yaml new file mode 100644 index 0000000000..2cae50d2fa --- /dev/null +++ b/toolkit/components/nimbus/FeatureManifest.yaml @@ -0,0 +1,1564 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Features must be added here to be accessible through the NimbusFeature API. + +"no-feature-firefox-desktop": + description: A dummy feature for experiments that target no feature. + owner: barret@mozilla.com + applications: + - firefox-desktop + - firefox-desktop-background-task + hasExposure: false + variables: {} + +testFeature: + description: Test only feature + owner: barret@mozilla.com + applications: + - firefox-desktop + - firefox-desktop-background-task + hasExposure: false + isEarlyStartup: true + variables: + enabled: + type: boolean + description: Whether or not this feature is enabled + testInt: + type: int + fallbackPref: nimbus.testing.testInt + description: Int pref used by platform API tests + testSetString: + type: string + setPref: nimbus.testing.testSetString + description: A string pref set by Nimbus tests + +nimbus-qa-1: + description: A feature for testing pref-setting on the default branch. + owner: barret@mozilla.com + hasExposure: false + variables: + value: + type: string + setPref: nimbus.qa.pref-1 + description: The value to set for the pref. + +nimbus-qa-2: + description: A feature for testing pref-setting on the user branch. + owner: barret@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + value: + type: string + setPref: nimbus.qa.pref-2 + description: The value to set for the pref. + +# `search` is for search engine experimentation features which do not require +# isEarlyStartup to be set. +search: + description: Search engine experimentation support and testing features. + owner: search-and-suggest-program@mozilla.com + hasExposure: false + variables: + extraParams: + type: json + description: >- + This allows extra parameters to be set for search engines requests including, + where calls to the suggestions API, the search engine configuration defines + those parameters. + + The use of this field should be coordinated with the Search team. + + The field value is an array of objects with key/value fields. For example: + + [ + {"key": "google_channel_row", "value": "foo"} + ] + + This is matched to a section in the search configuration: + + "extraParams": [ + { + "name": "channel", + "pref": "google_channel_row", + "condition": "pref" + } + ], + + In this case, the resulting URL for the appropriate search engine would have + `&channel=foo` added to the URL when doing searches. + + If the key is not referenced in the search configuration, then no parameter + will be added. Only the search team can update the configuration. + cbhStudyUs: + type: string + setPref: browser.search.param.google_channel_us + description: >- + NOTE: Please use `extraParams` rather than this field. + A string pref set by Nimbus tests for US search cohort + cbhStudyRow: + type: string + setPref: browser.search.param.google_channel_row + description: >- + NOTE: Please use `extraParams` rather than this field. + A string pref set by Nimbus tests for Rest of World (ROW) search cohort + richSuggestionsFeatureGate: + type: boolean + setPref: browser.urlbar.richSuggestions.featureGate + description: >- + Feature gate that controls whether Rich Suggestions are enabled. + serpEventTelemetryEnabled: + type: boolean + setPref: browser.search.serpEventTelemetry.enabled + description: Whether the Glean SERP event telemetry is enabled. + +# `searchConfiguration` is for search experiment features for items that require +# isEarlyStartup to be true. These items may require a reload of the search +# engine configuration, and an additional reload may happen during the startup +# process. +searchConfiguration: + description: Search experimentation support for the engine configuration + owner: search-and-suggest-program@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + experiment: + type: string + fallbackPref: browser.search.experiment + description: >- + Used to activate only matching configurations that contain the value in + `experiment` + seperatePrivateDefaultUIEnabled: + type: boolean + description: Whether the UI for the separate private default feature is enabled. + seperatePrivateDefaultUrlbarResultEnabled: + type: boolean + description: Whether the urlbar result for the separate private default is shown. + +urlbar: + description: The Address Bar + owner: search-and-suggest-program@mozilla.com + hasExposure: true + exposureDescription: >- + The timing of the exposure event depends on the experiment, but generally + the event is recorded once per app session when the user first encounters + the UI of the experiment in which they're enrolled. + variables: + addonsFeatureGate: + type: boolean + fallbackPref: browser.urlbar.addons.featureGate + description: >- + Feature gate that controls whether all aspects of the addons suggestion + feature are exposed to the user. + addonsShowLessFrequentlyCap: + type: int + description: >- + If defined and non-zero, this is the maximum number of times the user + will be able to click the "Show less frequently" command for addon + suggestions. If undefined or zero, the user will be able to click the + command without any limit. + addonsUITreatment: + type: string + enum: + - a + - b + description: >- + Define the UI type for addon suggestions. + In case of "a", display rating stars and review volume on the bottom of + the suggestion. In case of "b", display a label explaining that the + addon suggestion is a recommendation. + autoFillAdaptiveHistoryEnabled: + type: boolean + fallbackPref: browser.urlbar.autoFill.adaptiveHistory.enabled + description: Whether enabling adaptive history autofill. + autoFillAdaptiveHistoryMinCharsThreshold: + type: int + fallbackPref: browser.urlbar.autoFill.adaptiveHistory.minCharsThreshold + description: Minimum char length of the user's search string to trigger adaptive history autofill. + autoFillAdaptiveHistoryUseCountThreshold: + type: string + description: This value assumes float expression like "0.47". Threshold for use count of input history that we handle as adaptive history autofill. If the use count is this value or more, it will be a candidate. + bestMatchBlockingEnabled: + type: boolean + fallbackPref: browser.urlbar.bestMatch.blockingEnabled + description: Whether best match Suggest suggestions can be blocked. + bestMatchEnabled: + type: boolean + fallbackPref: browser.urlbar.bestMatch.enabled + description: Gate for the best match feature. If false, the best match preferences UI and best match suggestions will not be shown. If true, the preferences UI will be shown, and the user can turn best match suggestions on or off. + experimentType: + type: string + description: The type of the experiment (or rollout). If "best-match", then the Nimbus exposure event will be recorded when the user first triggers a best match (or would have triggered a best match, for users in the control group). If "modal", the event will be recorded when the user first triggers to show the onbording dialog. If empty, the event will be recorded when the user first triggers any type of Suggest suggestion. + enum: + - best-match + - modal + - "" + isBestMatchExperiment: + type: boolean + description: >- + Whether the experiment (or rollout) is related to best match. If true, then the Nimbus exposure event will be recorded when the user first triggers a best match (or would have triggered a best match, for users in the control group). Deprecated, please use `experimentType: "best-match"` instead. + merinoClientVariants: + type: string + fallbackPref: browser.urlbar.merino.clientVariants + description: >- + Comma separated list of client variants to report to the Merino server. + May impact server behavior. + merinoEnabled: + type: boolean + fallbackPref: browser.urlbar.merino.enabled + description: Whether Merino is enabled as a quick suggest source + merinoEndpointURL: + type: string + fallbackPref: browser.urlbar.merino.endpointURL + description: The Merino endpoint URL, not including parameters. + merinoProviders: + type: string + fallbackPref: browser.urlbar.merino.providers + description: >- + Comma-separated list of providers to request from the Merino server. + Merino will return suggestions only for these providers. + merinoTimeoutMs: + type: int + fallbackPref: browser.urlbar.merino.timeoutMs + description: Timeout for Merino fetches (ms) + exposureResults: + type: string + setPref: browser.urlbar.exposureResults + description: >- + Comma-separated list of result type combinations, that are used to determine if an exposure event should be fired. + showExposureResults: + type: boolean + setPref: browser.urlbar.showExposureResults + description: >- + Boolean used to determine if the results defined in `exposureResults` should be shown in search results. Should be false for Control branch of an experiment. + quickSuggestAllowPositionInSuggestions: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.allowPositionInSuggestions + description: Whether quick suggest results can be shown in position specified in the suggestions. + quickSuggestBlockingEnabled: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.blockingEnabled + description: Whether the usual non-best-match Suggest suggestions can be blocked. + quickSuggestDataCollectionEnabled: + type: boolean + description: Whether data collection should be enabled by default. If this variable is specified, it will override the value implied by the scenario. It will never override the user's local preference to disable (or enable) data collection, if the user has already toggled that preference. + quickSuggestEnabled: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.enabled + description: Gate for the Firefox Suggest feature as a whole. If false, the Firefox Suggest preferences UI and Suggest suggestions will not be shown. If true, the preferences UI will be shown, and the user can turn suggestions on or off. + quickSuggestImpressionCapsSponsoredEnabled: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.impressionCaps.sponsoredEnabled + description: Whether sponsored suggestions are subject to impression frequency caps. If false, sponsored suggestions can be shown an unlimited number of times over any given period. If true, sponsored suggestion impressions will be subject to the caps in the remote settings configuration. + quickSuggestImpressionCapsNonSponsoredEnabled: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.impressionCaps.nonSponsoredEnabled + description: Whether non-sponsored suggestions are subject to impression frequency caps. If false, non-sponsored suggestions can be shown an unlimited number of times over any given period. If true, non-sponsored suggestion impressions will be subject to the caps in the remote settings configuration. + quickSuggestNonSponsoredEnabled: + type: boolean + description: Whether non-sponsored suggestions should be enabled by default. If this variable is specified, it will override the value implied by the scenario. It will never override the user's local preference to disable (or enable) non-sponsored suggestions, if the user has already toggled that preference. + quickSuggestNonSponsoredIndex: + type: int + fallbackPref: browser.urlbar.quicksuggest.nonSponsoredIndex + description: >- + The index of non-sponsored QuickSuggest results within the general + group. A negative index is relative to the end of the group + quickSuggestOnboardingDialogVariation: + type: string + description: >- + Specify the messages/UI variation for QuickSuggest onboarding dialog. This value is case insensitive. + quickSuggestRemoteSettingsDataType: + type: string + description: The `type` of the suggestions data in remote settings. If not specified, "data" is used. + quickSuggestRemoteSettingsEnabled: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.remoteSettings.enabled + description: Whether Remote Settings is enabled as a quick suggest source + quickSuggestScenario: + # IMPORTANT: This should not have a fallbackPref. See UrlbarPrefs.jsm. + type: string + description: The Firefox Suggest scenario in which the user is enrolled + enum: + - history + - offline + - online + quickSuggestShouldShowOnboardingDialog: + type: boolean + fallbackPref: browser.urlbar.quicksuggest.shouldShowOnboardingDialog + description: Whether or not to show the QuickSuggest onboarding dialog + quickSuggestShowOnboardingDialogAfterNRestarts: + type: int + fallbackPref: browser.urlbar.quicksuggest.showOnboardingDialogAfterNRestarts + description: Show QuickSuggest onboarding dialog after N browser restarts + quickSuggestSponsoredEnabled: + type: boolean + description: Whether sponsored suggestions should be enabled by default. If this variable is specified, it will override the value implied by the scenario. It will never override the user's local preference to disable (or enable) sponsored suggestions, if the user has already toggled that preference. + quickSuggestSponsoredIndex: + type: int + fallbackPref: browser.urlbar.quicksuggest.sponsoredIndex + description: >- + The index of sponsored QuickSuggest results within the general group. A + negative index is relative to the end of the group + recordNavigationalSuggestionTelemetry: + type: boolean + description: Whether to record navigational suggestion telemetry. Defaults to false. + showSearchTermsFeatureGate: + type: boolean + fallbackPref: browser.urlbar.showSearchTerms.featureGate + description: Gate for the show search terms feature. If false, the preference#search will not show the search terms feature checkbox, and search terms will never persist in the urlbar. If true, the preference checkbox will be shown on preferences#search, and the user can choose to persist search terms on or off in the urlbar. + weatherFeatureGate: + type: boolean + fallbackPref: browser.urlbar.weather.featureGate + description: >- + Feature gate that controls whether all aspects of the weather suggestion + feature are exposed to the user. See also `weatherKeywords` and + `weatherKeywordsMinimumLength`. In summary: To enable the weather + suggestion, set `weatherFeatureGate` to true, `weatherKeywords` to an + array of full keyword strings, and `weatherKeywordsMinimumLength` to a + non-zero integer. To disable the weather suggestion, leave out all + weather-related variables. + weatherKeywords: + type: json + description: >- + An array of full keyword strings that will trigger the weather + suggestion when the user types them in the address bar. If absent or + null, Firefox will fall back to the weather keywords defined in remote + settings. If neither Nimbus nor remote settings defines any keywords, + the weather suggestion will be disabled. See also + `weatherKeywordsMinimumLength`. + weatherKeywordsMinimumLength: + type: int + description: >- + If defined and non-zero, the weather suggestion will be triggered by + typing any prefix of a full weather keyword when the prefix is at least + `weatherKeywordsMinimumLength` characters long. If this variable is + absent or zero, Firefox will fall back to the minimum length defined in + remote settings. If neither Nimbus nor remote settings defines a minimum + length, only full keywords will trigger the suggestion. See also + `weatherKeywords`. + weatherKeywordsMinimumLengthCap: + type: int + description: >- + If defined and non-zero, the user will not be able to increment the + minimum keyword length beyond this value. e.g., if this value is 6, the + current minimum length is 5, and the user clicks "Show less frequently", + then the minimum length will be incremented to 6, the "Show less + frequently" command will be hidden, and the user can continue to trigger + the weather suggestion by typing 6 characters, but they will not be able + to increment the minimum length any further. If this variable is absent + or zero, Firefox will fall back to the cap defined in remote settings. + If neither Nimbus nor remote settings defines a cap, no cap will be + used, and the user will be able to increment the minimum length without + any limit. + +frecency: + description: "The address bar ranking algorithm" + owner: search-and-suggest-program@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + originsAlternativeEnable: + description: >- + Use an alternative ranking algorithm for autofilling origins, that is + mainly domains of Web pages. When the user types the beginning of an + origin, we autofill the whole origin. Whether autofill happens depends + on the ranking algorithm. Bookmarks are always autofilled anyway. + type: boolean + setPref: "places.frecency.origins.alternative.featureGate" + originsDaysCutOff: + description: >- + The alternative ranking algorithm only considers pages visited in the + last N days, where N is controlled by this variable. + type: int + setPref: "places.frecency.origins.alternative.daysCutOff" + +aboutwelcome: + description: "The about:welcome page" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + Exposure is sent once per browsing session when the about:welcome URL is + first accessed. + isEarlyStartup: true + variables: + enabled: + type: boolean + fallbackPref: browser.aboutwelcome.enabled + description: >- + Should users see about:welcome? If this is false, users will see a + regular new tab instead. + id: + type: string + description: >- + Descriptive ID for the about:welcome content + screens: + type: json + fallbackPref: browser.aboutwelcome.screens + description: Content to show in the onboarding flow + languageMismatchEnabled: + type: boolean + fallbackPref: intl.multilingual.aboutWelcome.languageMismatchEnabled + description: >- + Suggest to change the language on about:welcome when there is a mismatch with + the OS. + transitions: + type: boolean + description: Enable transition effect between screens + showModal: + type: boolean + fallbackPref: browser.aboutwelcome.showModal + description: >- + Should users see window modal onboarding + backdrop: + type: string + fallbackPref: browser.aboutwelcome.backdrop + description: >- + Specify the color to be used to update the background color + +moreFromMozilla: + description: "New page on about:preferences to suggest more Mozilla products" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + Exposure is sent once per browsing session when the about:preferences URL is + first accessed. + variables: + enabled: + type: boolean + fallbackPref: browser.preferences.moreFromMozilla + description: Should users see the new more from Mozilla section. + template: + type: string + fallbackPref: browser.preferences.moreFromMozilla.template + description: UI template used to display Mozilla products. Possible values simple, advanced. Default is simple. + +abouthomecache: + description: "The startup about:home cache." + owner: omc@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + enabled: + type: boolean + fallbackPref: browser.startup.homepage.abouthome_cache.enabled + description: Is the feature enabled? + +newtab: + description: "The about:newtab page" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + Exposure is sent once per browsing session when the first newtab page loads + (either about:newtab or about:home). + isEarlyStartup: true + variables: + newTheme: + type: boolean + description: Enable the new theme + customizationMenuEnabled: + type: boolean + fallbackPref: browser.newtabpage.activity-stream.customizationMenu.enabled + description: Enable the customization panel inside of the newtab + prefsButtonIcon: + type: string + description: Icon url to use for the preferences button + topSitesContileEnabled: + type: boolean + fallbackPref: browser.topsites.contile.enabled + description: Enable the Contile integration for Sponsored Top Sites + topSitesUseAdditionalTilesFromContile: + type: boolean + description: Allow Contile to use additonal sponsored top sites + +pocketNewtab: + description: The Pocket section in newtab + owner: sdowne@getpocket.com + hasExposure: false + isEarlyStartup: true + variables: + spocPositions: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spoc-positions + description: CSV string of spoc position indexes on newtab Pocket grid + spocTopsitesPositions: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spoc-topsites-positions + description: CSV string of spoc position indexes on newtab topsites section + spocAdTypes: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spocAdTypes + description: CSV string of data to set the spoc content. + spocZoneIds: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spocZoneIds + description: CSV string of data to set the spoc content. + spocTopsitesAdTypes: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spocTopsitesAdTypes + description: CSV string of data to set the spoc content. + spocTopsitesZoneIds: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spocTopsitesZoneIds + description: CSV string of data to set the spoc content. + spocSiteId: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.spocSiteId + description: String ID to set the spoc content. + widgetPositions: + type: string + fallbackPref: browser.newtabpage.activity-stream.discoverystream.widget-positions + description: CSV string of widget position indexes on newtab grid + hybridLayout: + type: boolean + fallbackPref: browser.newtabpage.activity-stream.discoverystream.hybridLayout.enabled + description: Enable compact cards on newtab grid only for specific breakpoints + hideCardBackground: + type: boolean + fallbackPref: browser.newtabpage.activity-stream.discoverystream.hideCardBackground.enabled + description: Removes Pocket card background and borders. + fourCardLayout: + type: boolean + fallbackPref: browser.newtabpage.activity-stream.discoverystream.fourCardLayout.enabled + description: Enable four Pocket cards per row. + newFooterSection: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.newFooterSection.enabled + description: Enable an updated Pocket section topics footer + saveToPocketCard: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.saveToPocketCard.enabled + description: >- + A save to Pocket button inside the card, shown on the card thumbnail, on + hover. + saveToPocketCardRegions: + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.saveToPocketCardRegions + description: >- + CSV string of regions that support the save to Pocket button inside the card. + hideDescriptions: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.hideDescriptions.enabled + description: >- + Hide or display descriptions for Pocket stories on newtab. + hideDescriptionsRegions: + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.hideDescriptionsRegions + description: >- + CSV string of regions that hide descriptions for Pocket stories on newtab. + compactGrid: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.compactGrid.enabled + description: >- + Reduce the number of pixels between the Pocket cards on newtab. + compactImages: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.compactImages.enabled + description: >- + Reduce the height on Pocket card images on newtab. + imageGradient: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.imageGradient.enabled + description: >- + Add a gradient to the bottom of Pocket card images on newtab to blend the + image in with the card. + titleLines: + type: int + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.titleLines + description: >- + Changes the maximum number of lines a title can be for Pocket cards on newtab. + descLines: + type: int + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.descLines + description: >- + Changes the maximum number of lines a description can be for Pocket cards on newtab. + onboardingExperience: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.onboardingExperience.enabled + description: >- + Enables an onboarding experience for Pocket section on newtab. + essentialReadsHeader: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.essentialReadsHeader.enabled + description: >- + Updates the Pocket section header and title to say "Today’s Essential Reads", + moves the "Recommended by Pocket" header to the right side. + editorsPicksHeader: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.editorsPicksHeader.enabled + description: >- + Updates the Pocket section header and title to say "Editor’s Picks", if used with + essentialReadsHeader, creates a second section 2 rows down for editorsPicksHeader. + recentSavesEnabled: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.recentSaves.enabled + description: >- + Updates the Pocket section with a new header and 1 row of recently saved Pocket stories. + readTime: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.readTime.enabled + description: >- + Displays an estimated read time for Pocket cards on newtab. + newSponsoredLabel: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.newSponsoredLabel.enabled + description: >- + Updates the sponsored label position to below the image for Pocket cards on newtab. + sendToPocket: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.sendToPocket.enabled + description: >- + Decides what to do when a logged out user click "Save to Pocket" from a Pocket card. + recsPersonalized: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.recs.personalized + description: >- + Enables Pocket stories personalization. + spocsPersonalized: + type: boolean + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.spocs.personalized + description: >- + Enables Pocket sponsored content personalization. + spocsCacheTimeout: + type: int + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.spocs.cacheTimeout + description: >- + Set sponsored content cache timeout in minutes. + discoveryStreamConfig: + description: A JSON blob of discovery stream configuration. + type: string + setPref: "browser.newtabpage.activity-stream.discoverystream.config" + spocsEndpoint: + description: The URL for the spocs endpoint. + type: string + setPref: "browser.newtabpage.activity-stream.discoverystream.spocs-endpoint" + regionStoriesConfig: + description: A comma-separated list of region to get stories for. + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.region-stories-config + regionBffConfig: + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.region-bff-config + description: A comma-separated list of regions to get stories from the recommendations BFF. Also requires region-stories-config. + regionStoriesBlock: + description: A comma-separated list of regions that do not get stories, regardless of locale-list-config. + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.region-stories-block + localeListConfig: + description: A comma-separated list of locales that get stories, regardless of region-stories-config. + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.locale-list-config + regionSpocsConfig: + description: A comma-separated list of regions that get spocs by default. + type: string + fallbackPref: >- + browser.newtabpage.activity-stream.discoverystream.region-spocs-config + topSitesMaxSponsored: + # Defined under `pocketNewtab` as it needs to be used along with other variables + type: int + description: The maximum number of sponsored Top Sites to be displayed + topSitesContileMaxSponsored: + # Defined under `pocketNewtab` as it needs to be used along with other variables + type: int + description: The maximum number of sponsored Top Sites used from Contile + topSitesContileSovEnabled: + # Defined under `pocketNewtab` as it needs to be used along with other variables + description: Enable the Share-of-Voice feature for Sponsored Topsites. + type: boolean + fallbackPref: >- + browser.topsites.contile.sov.enabled + +saveToPocket: + description: The save to Pocket feature + owner: sdowne@getpocket.com + hasExposure: false + isEarlyStartup: true + variables: + emailButton: + type: boolean + fallbackPref: extensions.pocket.refresh.emailButton.enabled + description: Just for the new Pocket panels, enables the email signup button. + hideRecentSaves: + type: boolean + fallbackPref: extensions.pocket.refresh.hideRecentSaves.enabled + description: Hides the recently saved section in the home panel. + bffRecentSaves: + type: boolean + fallbackPref: "extensions.pocket.bffRecentSaves" + description: Use the new BFF Proxy Service instead of the legacy Pocket Service for Recent Saves + bffApi: + type: string + fallbackPref: "extensions.pocket.bffApi" + description: BFF Proxy Service domain + oAuthConsumerKeyBff: + type: string + fallbackPref: "extensions.pocket.oAuthConsumerKeyBff" + description: BFF Proxy Service OAuth Consumer Key + +password-autocomplete: + description: A special autocomplete UI for password fields. + owner: sgalich@mozilla.com + hasExposure: false + variables: + directMigrateSingleProfile: + type: boolean + description: Enable direct migration? + +shellService: + description: "Interface with OS, e.g., pinning and set default" + owner: desktop-integrations@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + disablePin: + type: boolean + description: Disable pin to taskbar feature + setDefaultBrowserUserChoice: + type: boolean + fallbackPref: browser.shell.setDefaultBrowserUserChoice + description: Should it set as default browser + setDefaultPDFHandler: + type: boolean + fallbackPref: browser.shell.setDefaultPDFHandler + description: Should setting it as the default browser set it as the default PDF handler. + setDefaultPDFHandlerOnlyReplaceBrowsers: + type: boolean + fallbackPref: browser.shell.setDefaultPDFHandler.onlyReplaceBrowsers + description: >- + Should setting it as the default PDF handler only replace existing PDF + handlers that are browsers, and not other PDF handlers such as Acrobat + Reader or Nitro PDF. + +upgradeDialog: + description: The dialog shown for major upgrades + owner: omc@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + enabled: + type: boolean + fallbackPref: browser.startup.upgradeDialog.enabled + description: Is the feature enabled? + +readerMode: + description: Firefox Reader Mode + owner: sdowne@getpocket.com + hasExposure: false + isEarlyStartup: true + variables: + pocketCTAVersion: + type: string + fallbackPref: reader.pocket.ctaVersion + description: >- + What version of Pocket CTA to show in Reader Mode (Empty string is no + CTA) +cfr: + description: "A Firefox Messaging System message for the cfr message channel" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +"moments-page": + description: "A Firefox Messaging System message for the moments-page message channel" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +infobar: + description: "A Firefox Messaging system message for the infobar message channel" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +spotlight: + description: "A Firefox Messaging System message for the spotlight message channel" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +# +# Before each Nightly leaves the station for Beta, we're currently +# experimenting with ensuring that we have enough fx-messages-* messages to +# accomodate all the rollout possibilities for Release that we're aware of, +# plus one for a buffer just-in-case. +# +# When adding an fxms-message-* feature id, be sure to also add it to the +# MESSAGING_EXPERIMENTS_DEFAULT_FEATURES list in ASRouter. + +# currently in use by 107+ exp/rollout pair of Import Bookmarks Infrequent +# Existing Users with 5 bookmarks: +# https://experimenter.services.mozilla.com/nimbus/import-infrequent-rollout-make-yourself-at-home/summary +fxms-message-1: + description: "A Firefox Messaging System message" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: 110+ exp/rollout pair of Fox Doodle: Pin experiment winner +# (Existing Users) +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-2: + description: "Firefox Messaging System message 2" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: 110+ exp/rollout pair of Fox Doodle: Set Default (Existing +# Users) +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-3: + description: "Firefox Messaging System message 3" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next potential planned use: 111+ exp/rollout pair for PDF annotations callout +# (https://mozilla-hub.atlassian.net/browse/FXE-83) experiment. +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-4: + description: "Firefox Messaging System message 4" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next potential planned use: 112+ holdback/rollout pair for default PDF handler +# (https://mozilla-hub.atlassian.net/browse/FXE-91) experiment. +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-5: + description: "Firefox Messaging System message 5" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next potential planned use: 114+ possible holdback/rollout pair (Device Migration): +# https://mozilla-hub.atlassian.net/browse/FXE-271 +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-6: + description: "Firefox Messaging System message 6" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: buffer, for the unexpected +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-7: + description: "Firefox Messaging System message 7" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + + +# next planned use: buffer, for the unexpected +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-8: + description: "Firefox Messaging System message 8" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: buffer, for the unexpected +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-9: + description: "Firefox Messaging System message 9" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: buffer, for the unexpected +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-10: + description: "Firefox Messaging System message 10" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +# next planned use: buffer, for the unexpected +# +# More info about the fx-message-* feature id prototype is available at: +# https://docs.google.com/document/d/1KdtaNycZL5j240nXofbibm9yKh7qVvu1-uZFLdmYM7Y/ +fxms-message-11: + description: "Firefox Messaging System message 11" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + "Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched." + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +pbNewtab: + description: "A Firefox Messaging System message for the pbNewtab message channel" + owner: omc@mozilla.com + hasExposure: true + exposureDescription: >- + Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched. + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/MessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json" + variables: {} + +backgroundTaskMessage: + description: "A Firefox Messaging System message for the background task message channel" + owner: nalexander@mozilla.com + applications: + - firefox-desktop-background-task + hasExposure: true + exposureDescription: >- + Exposure is sent if the message is about to be shown after trigger and targeting conditions on the message matched. + isEarlyStartup: false + schema: + uri: "resource://activity-stream/schemas/BackgroundTaskMessagingExperiment.schema.json" + path: "browser/components/newtab/content-src/asrouter/schemas/BackgroundTaskMessagingExperiment.schema.json" + variables: {} + +pictureinpicture: + description: Message for first time Picture-in-Picture users + owner: nbaumgardner@mozilla.com + hasExposure: true + exposureDescription: Exposure is sent when a user hovers over a video and Picture-in-Picture has not been used before + variables: + title: + type: string + description: The title to be used for the PiP toggle + message: + type: string + description: The message to be used in the PiP toggle + showIconOnly: + type: boolean + description: Whether to show the first time PiP toggle or show the PiP icon only + oldToggle: + type: boolean + description: Whether to show the control style (true) or variant style (false) for the first time PiP toggle + displayDuration: + type: int + description: Duration of PiP first time toggle display in days before switching to PiP icon toggle + +glean: + description: "The Glean data collection SDK within Firefox Desktop" + owner: glean-team@mozilla.com + hasExposure: false + variables: + finalInactive: + type: "boolean" + description: "Enables FOG early shutdown pings when true" + newtabPingEnabled: + type: "boolean" + fallbackPref: "browser.newtabpage.ping.enabled" + description: "Whether to submit the 'newtab' ping" + gleanMetricConfiguration: + type: json + description: >- + "A map of metric base-identifiers to booleans representing the state of the 'disabled' flag for that metric" + +majorRelease2022: + description: Major Release 2022 + owner: firefoxview@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + feltPrivacyPBMDarkTheme: + type: boolean + fallbackPref: "browser.theme.dark-private-windows" + description: "Use dark theme variant for PBM windows. This is only supported if the theme sets darkTheme data." + feltPrivacyPBMNewIndicator: + type: boolean + fallbackPref: "browser.privatebrowsing.enable-new-indicator" + description: "Enables the new private browsing indicator." + feltPrivacyPBMNewLogo: + type: boolean + fallbackPref: "browser.privatebrowsing.enable-new-logo" + description: "Enables the new about:privatebrowsing logo." + feltPrivacyShowPreferencesSection: + type: boolean + fallbackPref: "browser.privacySegmentation.preferences.show" + description: "Controls visibility of the privacy segmentation preferences section." + feltPrivacyWindowSeparation: + type: boolean + fallbackPref: "browser.privateWindowSeparation.enabled" + description: "Whether or not private browsing windows use a separate icon in the Windows taskbar" + colorwayCloset: + type: boolean + fallbackPref: "browser.theme.colorway-closet" + description: "Whether or not to show the colorway closet modal" + firefoxView: + type: boolean + fallbackPref: "browser.tabs.firefox-view" + description: "Whether or not to show the firefox view tab" + onboarding: + type: boolean + fallbackPref: "browser.majorrelease.onboarding" + description: "Whether or not to use the MR2022 onboarding settings." + +featureCallout: + description: "Prefs that control users' progress through Feature Callout tours" + owner: omc@mozilla.com + hasExposure: false + isEarlyStartup: false + variables: + pdfJsTourProgress: + description: A JSON String representing the intitial state of the pref that holds the progress of the PDF.js feature tour. + type: string + setPref: "browser.pdfjs.feature-tour" +browserLowMemoryPrefs: + description: Prefs which control the browser's behaviour under low memory. + owner: haftandilian@mozilla.com + hasExposure: false + variables: + lowMemoryResponseMask: + description: Control the response on macOS when under memory pressure. + type: int + setPref: "browser.lowMemoryResponseMask" + lowMemoryResponseOnWarn: + description: Controls which macOS memory-pressure levels trigger the browser low memory response. + type: boolean + setPref: "browser.lowMemoryResponseOnWarn" + tabsUnloadOnLowMemory: + description: Whether to unload tabs when available memory is running low. + type: boolean + setPref: "browser.tabs.unloadOnLowMemory" + +scriptLoaderPrefs: + description: Prefs that control the script loader. + owner: npierron@mozilla.com + hasExposure: false + variables: + delazificationStrategy: + description: >- + Selects which parsing/delazification strategy should be used while + parsing scripts off-main-thread. See DelazificationOption in + CompileOptions.h for values. + type: int + setPref: "dom.script_loader.delazification.strategy" + +echPrefs: + description: Prefs that control Encrypted Client Hello. + owner: djackson@mozilla.com + hasExposure: false + variables: + tlsEnabled: + description: Whether to enable ECH for connections using TLS + type: boolean + setPref: "network.dns.echconfig.enabled" + h3Enabled: + description: Whether to enable ECH for connections using H3/QUIC + type: boolean + setPref: "network.dns.http3_echconfig.enabled" + forceWaitHttpsRR: + description: Whether to force waiting for HTTPS DNS records, which ECH requires. + type: boolean + setPref: "network.dns.force_waiting_https_rr" + insecureFallback: + description: Whether to fallback to non-ECH connections if all ECH RRs fail. + type: boolean + setPref: "network.dns.echconfig.fallback_to_origin_when_all_failed" + tlsGreaseProb: + description: Probability of GREASEing a TLS connection with ECH (0-100). + type: int + setPref: "security.tls.ech.grease_probability" + h3GreaseEnabled: + description: Whether to apply GREASE settings to H3/QUIC connections. + type: boolean + setPref: "security.tls.ech.grease_http3" + disableGreaseOnFallback: + description: Whether to disable GREASE when retrying a connection. + type: boolean + setPref: "security.tls.ech.disable_grease_on_fallback" + greasePaddingSize: + description: Assumed echConfig padding length for GREASE extensions (1-255). + type: int + setPref: "security.tls.ech.grease_size" + +dohPrefs: + description: Prefs that control DNS over HTTPS. + owner: vgosu@mozilla.com + hasExposure: false + variables: + trrMode: + description: Has a value of 2 for TRR first, 3 for TRR only, 0 for off. + type: int + setPref: "network.trr.mode" + trrUri: + description: The URL of the DNS over HTTPS endpoint + type: string + setPref: "network.trr.uri" + dohMode: + description: Same as trrMode, but set by the DoHController module. + type: int + setPref: "doh-rollout.mode" + dohUri: + description: Same as trrUri, but set by the DoHController module. + type: string + setPref: "doh-rollout.uri" + enableFallbackWarningPage: + description: Whether DoH fallback warning page will be displayed when DoH doesn't work in TRR first mode. + type: boolean + setPref: "network.trr.display_fallback_warning" + showFallbackCheckbox: + description: Whether the checkbox to enable the fallback warning page is displayed in the settings UI. + type: boolean + setPref: "network.trr_ui.show_fallback_warning_option" + +dooh: + description: "DNS over Oblivious HTTP" + owner: vgosu@mozilla.com + hasExposure: false + variables: + ohttpEnabled: + description: Whether to use Oblivious HTTP for the resolution + type: boolean + setPref: "network.trr.use_ohttp" + ohttpRelayUri: + description: The URL of the Oblivious HTTP relay + type: string + setPref: "network.trr.ohttp.relay_uri" + ohttpConfigUri: + description: The URL used to fetch the configuration of the Oblivious HTTP gateway + type: string + setPref: "network.trr.ohttp.config_uri" + ohttpUri: + description: The URL of the Oblivious DNS over HTTPS target resource + type: string + setPref: "network.trr.ohttp.uri" + +networking: + description: "Firefox Networking (Necko)" + owner: vgosu@mozilla.com + hasExposure: false + variables: + ehPreloadEnabled: + description: Whether Early Hints preload is enabled + type: boolean + setPref: "network.early-hints.enabled" + ehPreconnectEnabled: + description: Whether Early Hints preconnect is enabled + type: boolean + setPref: "network.early-hints.preconnect.enabled" + +pingsender: + description: "In-product usage of the pingsender telemetry reporter." + owner: nalexander@mozilla.com + hasExposure: false + variables: + backgroundTaskEnabled: + type: "boolean" + fallbackPref: "toolkit.telemetry.shutdownPingSender.backgroundtask.enabled" + description: "Whether to use the `pingsender` background task to send shutdown telemetry" + +dapTelemetry: + description: DAP Telemetry + owner: simon@mozilla.com + hasExposure: false + isEarlyStartup: true # Data is sent on startup with a trigger in BrowserGlue.sys.mjs + variables: + enabled: + type: boolean + fallbackPref: toolkit.telemetry.dap_enabled + description: Whether to automatically send DAP measurements. + task1Enabled: + type: boolean + fallbackPref: toolkit.telemetry.dap_task1_enabled + description: Whether to send fake measurements for verification task 1. + +etpLevel2PBMPref: + description: The pref that controls the ETP level 2 list in the private browsing mode + owner: tihuang@mozilla.com + hasExposure: false + variables: + enabled: + description: Whether to enable ETP level 2 list in the private browsing mode. + type: boolean + setPref: "privacy.annotate_channels.strict_list.pbmode.enabled" + +fxaButtonVisibility: + description: Prefs to control the visibility of the Firefox Accounts toolbar button when not signed in. + owner: mconley@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + boolean: + description: True if the Firefox Accounts toolbar button should be visible when not signed in. + type: boolean + setPref: identity.fxaccounts.toolbar.defaultVisible + +legacyHeartbeat: + description: Normandy Heartbeat exposed to Nimbus + owner: barret@mozilla.com + hasExposure: false + schema: + uri: "resource://normandy/schemas/LegacyHeartbeat.schema.json" + path: "toolkit/components/normandy/schemas/LegacyHeartbeat.schema.json" + variables: + survey: + type: json + description: The Heartbeat survey parameters. + +queryStripping: + description: Query parameter stripping anti-tracking feature. + owner: pbz@mozilla.com + hasExposure: false + variables: + enabledNormalBrowsing: + type: boolean + setPref: privacy.query_stripping.enabled + description: Enables / disables URL query string stripping in normal browsing mode. + enabledPrivateBrowsing: + type: boolean + setPref: privacy.query_stripping.enabled.pbmode + description: Enables / disables URL query string stripping in private browsing mode. + allowList: + type: string + setPref: privacy.query_stripping.allow_list + description: >- + List of sites exempt from query stripping. This list will be merged with + records coming from RemoteSettings. + stripList: + type: string + setPref: privacy.query_stripping.strip_list + description: >- + List of query params to be stripped from URIs. This list will be merged + with records coming from RemoteSettings. + +fontvisibility: + description: Control Font Visibility in PBM + owner: tom@mozilla.com + hasExposure: false + variables: + enabledETP: + type: int + setPref: layout.css.font-visibility.trackingprotection + description: Set the Font Visibility level when Enhanced Tracking Protection is enabled + enabledStandard: + type: int + setPref: layout.css.font-visibility.standard + description: Set the Font Visibility level for normal browsing + enabledPBM: + type: int + setPref: layout.css.font-visibility.private + description: Set the Font Visibility level for private browsing (will override ETP) + +fingerprintingProtection: + description: Control Fingerprinting Protection + owner: tihuang@mozilla.com + hasExposure: false + variables: + enabledNormal: + type: boolean + setPref: privacy.fingerprintingProtection + description: Enables / disables fingerprinting protection in normal browsing mode. + enabledPrivate: + type: boolean + setPref: privacy.fingerprintingProtection.pbmode + description: Enables / disables fingerprinting protection in private browsing mode. + overrides: + type: string + setPref: privacy.fingerprintingProtection.overrides + description: >- + The protection overrides to add or remove fingerprinting protection + targets. Please check RFPTargets.inc for all supported targets. + +migrationWizard: + description: Prefs to control the Migration Wizard UI. + owner: mconley@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + useNewWizard: + description: True if the new migration wizard should be used. + type: boolean + setPref: browser.migrate.content-modal.enabled + showImportAll: + description: True if the "Variant 2" of the Migration Wizard browser / profile selection UI should be used. This is only meaningful in the new Migration Wizard. + type: boolean + setPref: browser.migrate.content-modal.import-all.enabled + showPreferencesEntrypoint: + description: True if an entrypoint to the migration wizard should be visible in about:preferences. + type: boolean + setPref: browser.migrate.preferences-entrypoint.enabled + aboutWelcomeBehavior: + description: >- + When migration is kicked off from about:welcome, there are + a few different behaviors that we want to test, controlled + by a preference that is instrumented for Nimbus. The pref + has the following possible states: + + "autoclose": + The user will be directed to the migration wizard in + about:preferences, but once the wizard is dismissed, + the tab will close. + + "embedded": + The migration wizard is embedded in about:welcome. + + "standalone": + The migration wizard will open in a new top-level content + window. + + "legacy": + The legacy wizard will be opened from about:welcome, even if + the new wizard is enabled by default. + + "default" / other + The user will be directed to the migration wizard in + about:preferences. The tab will not close once the + user closes the wizard. + type: string + setPref: browser.migrate.content-modal.about-welcome-behavior + +mixedContentUpgrading: + description: Prefs to control whether we upgrade mixed passive content (images, audio, video) from http to https + owner: fbraun@mozilla.com + hasExposure: false + variables: + boolean: + description: True if the mixed content upgrading pref is enabled + type: boolean + setPref: security.mixed_content.upgrade_display_content + +jsParallelParsing: + description: Pref to toggle JS parallel parsing. + owner: dpalmeiro@mozilla.com, nbp@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + enabled: + description: True to enable parallel parsing. + type: boolean + setPref: "javascript.options.parallel_parsing" + +gcParallelMarking: + description: Pref to toggle parallel marking in the GC. + owner: dpalmeiro@mozilla.com, jonco@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + enabled: + description: True to enable parallel marking. + type: boolean + setPref: "javascript.options.mem.gc_parallel_marking" + +jitThresholds: + description: Prefs that control jit tier thresholds. + owner: dpalmeiro@mozilla.com, jdemooij@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + blinterp_threshold: + description: Set the threshold to enable blinterp compilation. + type: int + setPref: "javascript.options.blinterp.threshold" + baseline_threshold: + description: Set the threshold to enable baseline compilation. + type: int + setPref: "javascript.options.baselinejit.threshold" + ion_threshold: + description: Set the threshold to enable ion compilation. + type: int + setPref: "javascript.options.ion.threshold" + ion_bailout_threshold: + description: Set the ion frequent bailout threshold. + type: int + setPref: "javascript.options.ion.frequent_bailout_threshold" + ion_offthread_compilation: + description: True to enable offthread ion compilations. + type: boolean + setPref: "javascript.options.ion.offthread_compilation" + inlining_max_length: + description: Set the max bytecode length considered for inlining. + type: int + setPref: "javascript.options.inlining_bytecode_max_length" + +jitHintsCache: + description: Pref to toggle the JIT hints cache. + owner: dpalmeiro@mozilla.com + isEarlyStartup: true + hasExposure: false + variables: + enabled: + description: True to enable the hints cache. + type: boolean + setPref: "javascript.options.jithints" + +raceCacheWithNetwork: + description: Prefs to toggle the race cache with network. + owner: dpalmeiro@mozilla.com, acreskey@mozilla.com + hasExposure: false + variables: + enabled: + description: True to enable the rcwn feature. + type: boolean + setPref: "network.http.rcwn.enabled" + +httpSpeculativeParallelLimit: + description: Prefs to control the http speculative parallel limit. + owner: dpalmeiro@mozilla.com, acreskey@mozilla.com + hasExposure: false + variables: + speculative_parallel_limit: + description: Maximum number of parallel speculative connections. + type: int + setPref: "network.http.speculative-parallel-limit" + +deviceMigration: + description: Prefs to control aspects of the new device migration experiment + owner: hjones@mozilla.com + hasExposure: false + isEarlyStartup: true + variables: + helpMenuHidden: + description: True if new help menu item should be hidden + type: boolean + fallbackPref: browser.device-migration.help-menu.hidden + +opaqueResponseBlocking: + description: Prefs to enable Opaque Response Blocking + owner: farre@mozilla.com + isEarlyStartup: true + hasExposure: true + exposureDescription: Exposure is sent when a response is blocked + variables: + enabled: + description: Whether ORB is enabled + type: boolean + setPref: "browser.opaqueResponseBlocking" + javascriptValidator: + description: Whether JavaScript validation for ORB is enabled + type: boolean + setPref: "browser.opaqueResponseBlocking.javascriptValidator" + filterFetchResponse: + description: Whether filtering of internal responses in the parent ORB is enabled + type: int + setPref: "browser.opaqueResponseBlocking.filterFetchResponse" + mediaExceptionsStrategy: + description: >- + If we partially or wholly allow audio and video MIME types in conflict with spec. + type: int + setPref: "browser.opaqueResponseBlocking.mediaExceptionsStrategy"
\ No newline at end of file diff --git a/toolkit/components/nimbus/docs/index.rst b/toolkit/components/nimbus/docs/index.rst new file mode 100644 index 0000000000..5940dd4692 --- /dev/null +++ b/toolkit/components/nimbus/docs/index.rst @@ -0,0 +1,8 @@ +Nimbus +======================== + +About +------ + +Nimbus is an automated experimentation system. +Our API docs are hosted with the `Experimenter Docs <https://mozilla.github.io/experimenter-docs/desktop-feature-api>`_. diff --git a/toolkit/components/nimbus/generate/generate_feature_manifest.py b/toolkit/components/nimbus/generate/generate_feature_manifest.py new file mode 100644 index 0000000000..f9ee13dd5a --- /dev/null +++ b/toolkit/components/nimbus/generate/generate_feature_manifest.py @@ -0,0 +1,152 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import sys +from pathlib import Path + +import jsonschema +import yaml + +HEADER_LINE = ( + "// This file was generated by generate_feature_manifest.py from FeatureManifest.yaml." + " DO NOT EDIT.\n" +) + +FEATURE_MANIFEST_SCHEMA = Path("schemas", "ExperimentFeatureManifest.schema.json") + +NIMBUS_FALLBACK_PREFS = ( + "constexpr std::pair<nsLiteralCString, nsLiteralCString>" + "NIMBUS_FALLBACK_PREFS[]{{{}}};" +) + + +def write_fm_headers(fd): + fd.write(HEADER_LINE) + + +def validate_feature_manifest(schema_path, manifest_path, manifest): + with open(schema_path, "r") as f: + schema = json.load(f) + + set_prefs = {} + fallback_prefs = {} + + for feature, feature_def in manifest.items(): + try: + jsonschema.validate(feature_def, schema) + + for variable, variable_def in feature_def.get("variables", {}).items(): + set_pref = variable_def.get("setPref") + if set_pref is not None: + if set_pref in set_prefs: + other_feature = set_prefs[set_pref][0] + other_variable = set_prefs[set_pref][1] + print("Multiple variables cannot declare the same setPref") + print( + f"{feature} variable {variable} wants to set pref {set_pref}" + ) + print( + f"{other_feature} variable {other_variable} wants to set pref " + f"{set_pref}" + ) + raise Exception("Set prefs are exclusive") + + set_prefs[set_pref] = (feature, variable) + + fallback_pref = variable_def.get("fallbackPref") + if fallback_pref is not None: + fallback_prefs[fallback_pref] = (feature, variable) + + conflicts = [ + ( + "setPref", + fallback_pref, + "fallbackPref", + set_prefs.get(fallback_pref), + ), + ("fallbackPref", set_pref, "setPref", fallback_prefs.get(set_pref)), + ] + + for kind, pref, other_kind, conflict in conflicts: + if conflict is not None: + print( + "The same pref cannot be specified in setPref and fallbackPref" + ) + print( + f"{feature} variable {variable} has specified {kind} {pref}" + ) + print( + f"{conflict[0]} variable {conflict[1]} has specified {other_kind} " + f"{pref}" + ) + raise Exception("Set prefs and fallback prefs cannot overlap") + + except Exception as e: + print("Error while validating FeatureManifest.yaml") + print(f"On key: {feature}") + print(f"Input file: {manifest_path}") + raise e + + +def generate_feature_manifest(fd, input_file): + write_fm_headers(fd) + + try: + with open(input_file, "r", encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + validate_feature_manifest( + Path(input_file).parent / FEATURE_MANIFEST_SCHEMA, input_file, manifest + ) + + fd.write(f"export const FeatureManifest = {json.dumps(manifest)};") + except (IOError) as e: + print(f"{input_file}: error:\n {e}\n") + sys.exit(1) + + +def platform_feature_manifest_array(features): + entries = [] + for (feature, featureData) in features.items(): + # Features have to be tagged isEarlyStartup to be accessible + # to Nimbus platform API + if not featureData.get("isEarlyStartup", False): + continue + entries.extend( + '{{ "{}_{}"_ns, "{}"_ns }}'.format( + feature, variable, variableData["fallbackPref"] + ) + for (variable, variableData) in featureData.get("variables", {}).items() + if variableData.get("fallbackPref", False) + ) + return NIMBUS_FALLBACK_PREFS.format(", ".join(entries)) + + +def generate_platform_feature_manifest(fd, input_file): + write_fm_headers(fd) + + def file_structure(data): + return "\n".join( + [ + "#ifndef mozilla_NimbusFeaturesManifest_h", + "#define mozilla_NimbusFeaturesManifest_h", + "#include <utility>", + '#include "mozilla/Maybe.h"', + '#include "nsStringFwd.h"', + "namespace mozilla {", + platform_feature_manifest_array(data), + '#include "./lib/NimbusFeatureManifest.inc.h"', + "} // namespace mozilla", + "#endif // mozilla_NimbusFeaturesManifest_h", + ] + ) + + try: + with open(input_file, "r", encoding="utf-8") as yaml_input: + data = yaml.safe_load(yaml_input) + fd.write(file_structure(data)) + except (IOError) as e: + print("{}: error:\n {}\n".format(input_file, e)) + sys.exit(1) diff --git a/toolkit/components/nimbus/jar.mn b/toolkit/components/nimbus/jar.mn new file mode 100644 index 0000000000..9404cc8c28 --- /dev/null +++ b/toolkit/components/nimbus/jar.mn @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +toolkit.jar: +% resource nimbus %res/nimbus/ + res/nimbus/lib/ (./lib/*.sys.mjs) + res/nimbus/ExperimentAPI.sys.mjs (./ExperimentAPI.sys.mjs) + res/nimbus/FeatureManifest.sys.mjs (FeatureManifest.sys.mjs) + res/nimbus/schemas/NimbusEnrollment.schema.json (./schemas/NimbusEnrollment.schema.json) + res/nimbus/schemas/NimbusExperiment.schema.json (./schemas/NimbusExperiment.schema.json) diff --git a/toolkit/components/nimbus/lib/ExperimentManager.sys.mjs b/toolkit/components/nimbus/lib/ExperimentManager.sys.mjs new file mode 100644 index 0000000000..cd34f5ad03 --- /dev/null +++ b/toolkit/components/nimbus/lib/ExperimentManager.sys.mjs @@ -0,0 +1,1344 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ClientEnvironment: "resource://normandy/lib/ClientEnvironment.sys.mjs", + ExperimentStore: "resource://nimbus/lib/ExperimentStore.sys.mjs", + FirstStartup: "resource://gre/modules/FirstStartup.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + NormandyUtils: "resource://normandy/lib/NormandyUtils.sys.mjs", + PrefUtils: "resource://normandy/lib/PrefUtils.sys.mjs", + Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetryEvents: "resource://normandy/lib/TelemetryEvents.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("ExperimentManager"); +}); + +const TELEMETRY_EVENT_OBJECT = "nimbus_experiment"; +const TELEMETRY_EXPERIMENT_ACTIVE_PREFIX = "nimbus-"; +const TELEMETRY_DEFAULT_EXPERIMENT_TYPE = "nimbus"; + +const UPLOAD_ENABLED_PREF = "datareporting.healthreport.uploadEnabled"; +const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled"; + +const STUDIES_ENABLED_CHANGED = "nimbus:studies-enabled-changed"; + +function featuresCompat(branch) { + if (!branch || (!branch.feature && !branch.features)) { + return []; + } + let { features } = branch; + // In <=v1.5.0 of the Nimbus API, experiments had single feature + if (!features) { + features = [branch.feature]; + } + + return features; +} + +function getFeatureFromBranch(branch, featureId) { + return featuresCompat(branch).find( + featureConfig => featureConfig.featureId === featureId + ); +} + +/** + * A module for processes Experiment recipes, choosing and storing enrollment state, + * and sending experiment-related Telemetry. + */ +export class _ExperimentManager { + constructor({ id = "experimentmanager", store } = {}) { + this.id = id; + this.store = store || new lazy.ExperimentStore(); + this.sessions = new Map(); + // By default, no extra context. + this.extraContext = {}; + Services.prefs.addObserver(UPLOAD_ENABLED_PREF, this); + Services.prefs.addObserver(STUDIES_OPT_OUT_PREF, this); + + // A Map from pref names to pref observers and metadata. See + // `_updatePrefObservers` for the full structure. + this._prefs = new Map(); + // A Map from enrollment slugs to a Set of prefs that enrollment is setting + // or would set (e.g., if the enrollment is a rollout and there wasn't an + // active experiment already setting it). + this._prefsBySlug = new Map(); + } + + get studiesEnabled() { + return ( + Services.prefs.getBoolPref(UPLOAD_ENABLED_PREF) && + Services.prefs.getBoolPref(STUDIES_OPT_OUT_PREF) + ); + } + + /** + * Creates a targeting context with following filters: + * + * * `activeExperiments`: an array of slugs of all the active experiments + * * `isFirstStartup`: a boolean indicating whether or not the current enrollment + * is performed during the first startup + * + * @returns {Object} A context object + * @memberof _ExperimentManager + */ + createTargetingContext() { + let context = { + isFirstStartup: lazy.FirstStartup.state === lazy.FirstStartup.IN_PROGRESS, + ...this.extraContext, + }; + Object.defineProperty(context, "activeExperiments", { + get: async () => { + await this.store.ready(); + return this.store.getAllActiveExperiments().map(exp => exp.slug); + }, + }); + Object.defineProperty(context, "activeRollouts", { + get: async () => { + await this.store.ready(); + return this.store.getAllActiveRollouts().map(rollout => rollout.slug); + }, + }); + return context; + } + + /** + * Runs on startup, including before first run. + * + * @param {object} extraContext extra targeting context provided by the + * ambient environment. + */ + async onStartup(extraContext = {}) { + await this.store.init(); + this.extraContext = extraContext; + + const restoredExperiments = this.store.getAllActiveExperiments(); + const restoredRollouts = this.store.getAllActiveRollouts(); + + for (const experiment of restoredExperiments) { + this.setExperimentActive(experiment); + if (this._restoreEnrollmentPrefs(experiment)) { + this._updatePrefObservers(experiment); + } + } + for (const rollout of restoredRollouts) { + this.setExperimentActive(rollout); + if (this._restoreEnrollmentPrefs(rollout)) { + this._updatePrefObservers(rollout); + } + } + + this.observe(); + } + + /** + * Runs every time a Recipe is updated or seen for the first time. + * @param {RecipeArgs} recipe + * @param {string} source + */ + async onRecipe(recipe, source) { + const { slug, isEnrollmentPaused } = recipe; + + if (!source) { + throw new Error("When calling onRecipe, you must specify a source."); + } + + if (!this.sessions.has(source)) { + this.sessions.set(source, new Set()); + } + this.sessions.get(source).add(slug); + + if (this.store.has(slug)) { + await this.updateEnrollment(recipe, source); + } else if (isEnrollmentPaused) { + lazy.log.debug(`Enrollment is paused for "${slug}"`); + } else if (!(await this.isInBucketAllocation(recipe.bucketConfig))) { + lazy.log.debug("Client was not enrolled because of the bucket sampling"); + } else { + await this.enroll(recipe, source); + } + } + + _checkUnseenEnrollments( + enrollments, + sourceToCheck, + recipeMismatches, + invalidRecipes, + invalidBranches, + invalidFeatures, + missingLocale, + missingL10nIds + ) { + for (const enrollment of enrollments) { + const { slug, source } = enrollment; + if (sourceToCheck !== source) { + continue; + } + if (!this.sessions.get(source)?.has(slug)) { + lazy.log.debug(`Stopping study for recipe ${slug}`); + try { + let reason; + if (recipeMismatches.includes(slug)) { + reason = "targeting-mismatch"; + } else if (invalidRecipes.includes(slug)) { + reason = "invalid-recipe"; + } else if (invalidBranches.has(slug) || invalidFeatures.has(slug)) { + reason = "invalid-branch"; + } else if (missingLocale.includes(slug)) { + reason = "l10n-missing-locale"; + } else if (missingL10nIds.has(slug)) { + reason = "l10n-missing-entry"; + } else { + reason = "recipe-not-seen"; + } + this.unenroll(slug, reason); + } catch (err) { + console.error(err); + } + } + } + } + + /** + * Removes stored enrollments that were not seen after syncing with Remote Settings + * Runs when the all recipes been processed during an update, including at first run. + * @param {string} sourceToCheck + * @param {object} options Extra context used in telemetry reporting + * @param {string[]} options.recipeMismatches + * The list of experiments that do not match targeting. + * @param {string[]} options.invalidRecipes + * The list of recipes that do not match + * @param {Map<string, string[]>} options.invalidBranches + * A mapping of experiment slugs to a list of branches that failed + * feature validation. + * @param {Map<string, string[]>} options.invalidFeatures + * The mapping of experiment slugs to a list of invalid feature IDs. + * @param {string[]} options.missingLocale + * The list of experiment slugs missing an entry in the localization + * table for the current locale. + * @param {Map<string, string[]>} options.missingL10nIds + * The mapping of experiment slugs to the IDs of localization entries + * missing from the current locale. + * @param {string | null} options.locale + * The current locale. + * @param {boolean} options.validationEnabled + * Whether or not schema validation was enabled. + */ + onFinalize( + sourceToCheck, + { + recipeMismatches = [], + invalidRecipes = [], + invalidBranches = new Map(), + invalidFeatures = new Map(), + missingLocale = [], + missingL10nIds = new Map(), + locale = null, + validationEnabled = true, + } = {} + ) { + if (!sourceToCheck) { + throw new Error("When calling onFinalize, you must specify a source."); + } + const activeExperiments = this.store.getAllActiveExperiments(); + const activeRollouts = this.store.getAllActiveRollouts(); + this._checkUnseenEnrollments( + activeExperiments, + sourceToCheck, + recipeMismatches, + invalidRecipes, + invalidBranches, + invalidFeatures, + missingLocale, + missingL10nIds + ); + this._checkUnseenEnrollments( + activeRollouts, + sourceToCheck, + recipeMismatches, + invalidRecipes, + invalidBranches, + invalidFeatures, + missingLocale, + missingL10nIds + ); + + // If schema validation is disabled, then we will never send these + // validation failed telemetry events + if (validationEnabled) { + for (const slug of invalidRecipes) { + this.sendValidationFailedTelemetry(slug, "invalid-recipe"); + } + for (const [slug, branches] of invalidBranches.entries()) { + for (const branch of branches) { + this.sendValidationFailedTelemetry(slug, "invalid-branch", { + branch, + }); + } + } + for (const [slug, featureIds] of invalidFeatures.entries()) { + for (const featureId of featureIds) { + this.sendValidationFailedTelemetry(slug, "invalid-feature", { + feature: featureId, + }); + } + } + } + + if (locale) { + for (const slug of missingLocale.values()) { + this.sendValidationFailedTelemetry(slug, "l10n-missing-locale", { + locale, + }); + } + + for (const [slug, ids] of missingL10nIds.entries()) { + this.sendValidationFailedTelemetry(slug, "l10n-missing-entry", { + l10n_ids: ids.join(","), + locale, + }); + } + } + + this.sessions.delete(sourceToCheck); + } + + /** + * Bucket configuration specifies a specific percentage of clients that can + * be enrolled. + * @param {BucketConfig} bucketConfig + * @returns {Promise<boolean>} + */ + isInBucketAllocation(bucketConfig) { + if (!bucketConfig) { + lazy.log.debug("Cannot enroll if recipe bucketConfig is not set."); + return false; + } + + let id; + if (bucketConfig.randomizationUnit === "normandy_id") { + id = lazy.ClientEnvironment.userId; + } else { + // Others not currently supported. + lazy.log.debug( + `Invalid randomizationUnit: ${bucketConfig.randomizationUnit}` + ); + return false; + } + + return lazy.Sampling.bucketSample( + [id, bucketConfig.namespace], + bucketConfig.start, + bucketConfig.count, + bucketConfig.total + ); + } + + /** + * Start a new experiment by enrolling the users + * + * @param {RecipeArgs} recipe + * @param {string} source + * @param {object} options + * @param {boolean} options.reenroll - Allow re-enrollment. Only allowed for rollouts. + * @returns {Promise<Enrollment>} The experiment object stored in the data store + * @rejects {Error} + * @memberof _ExperimentManager + */ + async enroll(recipe, source, { reenroll = false } = {}) { + let { slug, branches } = recipe; + + const enrollment = this.store.get(slug); + + if ( + enrollment && + (enrollment.isActive || !enrollment.isRollout || !reenroll) + ) { + this.sendFailureTelemetry("enrollFailed", slug, "name-conflict"); + throw new Error(`An experiment with the slug "${slug}" already exists.`); + } + + let storeLookupByFeature = recipe.isRollout + ? this.store.getRolloutForFeature.bind(this.store) + : this.store.hasExperimentForFeature.bind(this.store); + const branch = await this.chooseBranch(slug, branches); + const features = featuresCompat(branch); + for (let feature of features) { + if (storeLookupByFeature(feature?.featureId)) { + lazy.log.debug( + `Skipping enrollment for "${slug}" because there is an existing ${ + recipe.isRollout ? "rollout" : "experiment" + } for this feature.` + ); + this.sendFailureTelemetry("enrollFailed", slug, "feature-conflict"); + + return null; + } + } + + return this._enroll(recipe, branch, source); + } + + _enroll( + { + slug, + experimentType = TELEMETRY_DEFAULT_EXPERIMENT_TYPE, + userFacingName, + userFacingDescription, + featureIds, + isRollout, + localizations, + }, + branch, + source, + options = {} + ) { + const { prefs, prefsToSet } = this._getPrefsForBranch(branch, isRollout); + + /** @type {Enrollment} */ + const experiment = { + slug, + branch, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + experimentType, + source, + userFacingName, + userFacingDescription, + lastSeen: new Date().toJSON(), + featureIds, + prefs, + }; + + if (localizations) { + experiment.localizations = localizations; + } + + if (typeof isRollout !== "undefined") { + experiment.isRollout = isRollout; + } + + // Tag this as a forced enrollment. This prevents all unenrolling unless + // manually triggered from about:studies + if (options.force) { + experiment.force = true; + } + + if (isRollout) { + experiment.experimentType = "rollout"; + this.store.addEnrollment(experiment); + this.setExperimentActive(experiment); + } else { + this.store.addEnrollment(experiment); + this.setExperimentActive(experiment); + } + this.sendEnrollmentTelemetry(experiment); + + this._setEnrollmentPrefs(prefsToSet); + this._updatePrefObservers(experiment); + + lazy.log.debug( + `New ${isRollout ? "rollout" : "experiment"} started: ${slug}, ${ + branch.slug + }` + ); + + return experiment; + } + + forceEnroll(recipe, branch, source = "force-enrollment") { + /** + * If we happen to be enrolled in an experiment for the same feature + * we need to unenroll from that experiment. + * If the experiment has the same slug after unenrollment adding it to the + * store will overwrite the initial experiment. + */ + const features = featuresCompat(branch); + for (let feature of features) { + const isRollout = recipe.isRollout ?? false; + let enrollment = isRollout + ? this.store.getRolloutForFeature(feature?.featureId) + : this.store.getExperimentForFeature(feature?.featureId); + if (enrollment) { + lazy.log.debug( + `Existing ${ + isRollout ? "rollout" : "experiment" + } found for the same feature ${feature.featureId}, unenrolling.` + ); + + this.unenroll(enrollment.slug, source); + } + } + + recipe.userFacingName = `${recipe.userFacingName} - Forced enrollment`; + + const slug = `optin-${recipe.slug}`; + const enrollment = this._enroll( + { + ...recipe, + slug, + }, + branch, + source, + { force: true } + ); + + Services.obs.notifyObservers(null, "nimbus:enrollments-updated", slug); + + return enrollment; + } + + /** + * Update an enrollment that was already set + * + * @param {RecipeArgs} recipe + * @returns {boolean} whether the enrollment is still active + */ + async updateEnrollment(recipe, source) { + /** @type Enrollment */ + const enrollment = this.store.get(recipe.slug); + + // Don't update experiments that were already unenrolled. + if (enrollment.active === false && !recipe.isRollout) { + lazy.log.debug(`Enrollment ${recipe.slug} has expired, aborting.`); + return false; + } + + if (recipe.isRollout) { + if (!(await this.isInBucketAllocation(recipe.bucketConfig))) { + lazy.log.debug( + `No longer meet bucketing for "${recipe.slug}"; unenrolling...` + ); + this.unenroll(recipe.slug, "bucketing"); + return false; + } else if ( + !enrollment.active && + enrollment.unenrollReason !== "individual-opt-out" + ) { + lazy.log.debug(`Re-enrolling in rollout "${recipe.slug}`); + return !!(await this.enroll(recipe, source, { reenroll: true })); + } + } + + // Stay in the same branch, don't re-sample every time. + const branch = recipe.branches.find( + branch => branch.slug === enrollment.branch.slug + ); + + if (!branch) { + // Our branch has been removed. Unenroll. + this.unenroll(recipe.slug, "branch-removed"); + return false; + } + + return true; + } + + /** + * Stop an enrollment that is currently active + * + * @param {string} slug + * The slug of the enrollment to stop. + * @param {string} reason + * An optional reason for the unenrollment. + * + * This will be reported in telemetry. + */ + unenroll(slug, reason = "unknown") { + const enrollment = this.store.get(slug); + if (!enrollment) { + this.sendFailureTelemetry("unenrollFailed", slug, "does-not-exist"); + throw new Error(`Could not find an experiment with the slug "${slug}"`); + } + + this._unenroll(enrollment, { reason }); + } + + /** + * Stop an enrollment that is currently active. + * + * @param {Enrollment} enrollment + * The enrollment to end. + * + * @param {object} options + * @param {string} options.reason + * An optional reason for the unenrollment. + * + * This will be reported in telemetry. + * + * @param {object?} options.changedPref + * If the unenrollment was due to pref change, this will contain the + * information about the pref that changed. + * + * @param {string} options.changedPref.name + * The name of the pref that caused the unenrollment. + * + * @param {string} options.changedPref.branch + * The branch that was changed ("user" or "default"). + */ + _unenroll( + enrollment, + { reason = "unknown", changedPref = undefined, duringRestore = false } = {} + ) { + const { slug } = enrollment; + + if (!enrollment.active) { + this.sendFailureTelemetry("unenrollFailed", slug, "already-unenrolled"); + throw new Error( + `Cannot stop experiment "${slug}" because it is already expired` + ); + } + + lazy.TelemetryEnvironment.setExperimentInactive(slug); + // We also need to set the experiment inactive in the Glean Experiment API + Services.fog.setExperimentInactive(slug); + this.store.updateExperiment(slug, { + active: false, + unenrollReason: reason, + }); + + lazy.TelemetryEvents.sendEvent("unenroll", TELEMETRY_EVENT_OBJECT, slug, { + reason, + branch: enrollment.branch.slug, + enrollmentId: + enrollment.enrollmentId || lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + }); + // Sent Glean event equivalent + Glean.nimbusEvents.unenrollment.record({ + experiment: slug, + branch: enrollment.branch.slug, + enrollment_id: + enrollment.enrollmentId || lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + reason, + }); + + this._unsetEnrollmentPrefs(enrollment, { changedPref, duringRestore }); + + lazy.log.debug(`Recipe unenrolled: ${slug}`); + } + + /** + * Unenroll from all active studies if user opts out. + */ + observe(aSubject, aTopic, aPrefName) { + if (!this.studiesEnabled) { + for (const { slug } of this.store.getAllActiveExperiments()) { + this.unenroll(slug, "studies-opt-out"); + } + for (const { slug } of this.store.getAllActiveRollouts()) { + this.unenroll(slug, "studies-opt-out"); + } + } + + Services.obs.notifyObservers(null, STUDIES_ENABLED_CHANGED); + } + + /** + * Send Telemetry for undesired event + * + * @param {string} eventName + * @param {string} slug + * @param {string} reason + */ + sendFailureTelemetry(eventName, slug, reason) { + lazy.TelemetryEvents.sendEvent(eventName, TELEMETRY_EVENT_OBJECT, slug, { + reason, + }); + if (eventName == "enrollFailed") { + Glean.nimbusEvents.enrollFailed.record({ + experiment: slug, + reason, + }); + } else if (eventName == "unenrollFailed") { + Glean.nimbusEvents.unenrollFailed.record({ + experiment: slug, + reason, + }); + } + } + + sendValidationFailedTelemetry(slug, reason, extra) { + lazy.TelemetryEvents.sendEvent( + "validationFailed", + TELEMETRY_EVENT_OBJECT, + slug, + { + reason, + ...extra, + } + ); + Glean.nimbusEvents.validationFailed.record({ + experiment: slug, + reason, + ...extra, + }); + } + + /** + * + * @param {Enrollment} experiment + */ + sendEnrollmentTelemetry({ slug, branch, experimentType, enrollmentId }) { + lazy.TelemetryEvents.sendEvent("enroll", TELEMETRY_EVENT_OBJECT, slug, { + experimentType, + branch: branch.slug, + enrollmentId: + enrollmentId || lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + }); + Glean.nimbusEvents.enrollment.record({ + experiment: slug, + branch: branch.slug, + enrollment_id: + enrollmentId || lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + experiment_type: experimentType, + }); + } + + /** + * Sets Telemetry when activating an experiment. + * + * @param {Enrollment} experiment + */ + setExperimentActive(experiment) { + lazy.TelemetryEnvironment.setExperimentActive( + experiment.slug, + experiment.branch.slug, + { + type: `${TELEMETRY_EXPERIMENT_ACTIVE_PREFIX}${experiment.experimentType}`, + enrollmentId: + experiment.enrollmentId || + lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + } + ); + // Report the experiment to the Glean Experiment API + Services.fog.setExperimentActive(experiment.slug, experiment.branch.slug, { + type: `${TELEMETRY_EXPERIMENT_ACTIVE_PREFIX}${experiment.experimentType}`, + enrollmentId: + experiment.enrollmentId || lazy.TelemetryEvents.NO_ENROLLMENT_ID_MARKER, + }); + } + + /** + * Generate Normandy UserId respective to a branch + * for a given experiment. + * + * @param {string} slug + * @param {Array<{slug: string; ratio: number}>} branches + * @param {string} namespace + * @param {number} start + * @param {number} count + * @param {number} total + * @returns {Promise<{[branchName: string]: string}>} An object where + * the keys are branch names and the values are user IDs that will enroll + * a user for that particular branch. Also includes a `notInExperiment` value + * that will not enroll the user in the experiment if not 100% enrollment. + */ + async generateTestIds(recipe) { + // Older recipe structure had bucket config values at the top level while + // newer recipes group them into a bucketConfig object + const { slug, branches, namespace, start, count, total } = { + ...recipe, + ...recipe.bucketConfig, + }; + const branchValues = {}; + const includeNot = count < total; + + if (!slug || !namespace) { + throw new Error(`slug, namespace not in expected format`); + } + + if (!(start < total && count <= total)) { + throw new Error("Must include start, count, and total as integers"); + } + + if ( + !Array.isArray(branches) || + branches.filter(branch => branch.slug && branch.ratio).length !== + branches.length + ) { + throw new Error("branches parameter not in expected format"); + } + + while (Object.keys(branchValues).length < branches.length + includeNot) { + const id = lazy.NormandyUtils.generateUuid(); + const enrolls = await lazy.Sampling.bucketSample( + [id, namespace], + start, + count, + total + ); + // Does this id enroll the user in the experiment + if (enrolls) { + // Choose a random branch + const { slug: pickedBranch } = await this.chooseBranch( + slug, + branches, + id + ); + + if (!Object.keys(branchValues).includes(pickedBranch)) { + branchValues[pickedBranch] = id; + lazy.log.debug(`Found a value for "${pickedBranch}"`); + } + } else if (!branchValues.notInExperiment) { + branchValues.notInExperiment = id; + } + } + return branchValues; + } + + /** + * Choose a branch randomly. + * + * @param {string} slug + * @param {Branch[]} branches + * @returns {Promise<Branch>} + * @memberof _ExperimentManager + */ + async chooseBranch(slug, branches, userId = lazy.ClientEnvironment.userId) { + const ratios = branches.map(({ ratio = 1 }) => ratio); + + // It's important that the input be: + // - Unique per-user (no one is bucketed alike) + // - Unique per-experiment (bucketing differs across multiple experiments) + // - Differs from the input used for sampling the recipe (otherwise only + // branches that contain the same buckets as the recipe sampling will + // receive users) + const input = `${this.id}-${userId}-${slug}-branch`; + + const index = await lazy.Sampling.ratioSample(input, ratios); + return branches[index]; + } + + /** + * Generate the list of prefs a recipe will set. + * + * @params {object} branch The recipe branch that will be enrolled. + * @params {boolean} isRollout Whether or not this recipe is a rollout. + * + * @returns {object} An object with the following keys: + * + * `prefs`: + * The full list of prefs that this recipe would set, + * if there are no conflicts. This will include prefs + * that, for example, will not be set because this + * enrollment is a rollout and there is an active + * experiment that set the same pref. + * + * `prefsToSet`: + * Prefs that should be set once enrollment is + * complete. + */ + _getPrefsForBranch(branch, isRollout = false) { + const prefs = []; + const prefsToSet = []; + + const getConflictingEnrollment = this._makeEnrollmentCache(isRollout); + + for (const { featureId, value: featureValue } of featuresCompat(branch)) { + const feature = lazy.NimbusFeatures[featureId]; + + if (!feature) { + continue; + } + + // It is possible to enroll in both an experiment and a rollout, so we + // need to check if we have another enrollment for the same feature. + const conflictingEnrollment = getConflictingEnrollment(featureId); + + const prefBranch = + feature.manifest.isEarlyStartup ?? false ? "user" : "default"; + + for (const [variable, value] of Object.entries(featureValue)) { + const prefName = feature.getSetPrefName(variable); + + if (prefName) { + let originalValue; + const conflictingPref = conflictingEnrollment?.prefs?.find( + p => p.name === prefName + ); + + if (conflictingPref) { + // If there is another enrollment that has already set the pref we + // care about, we use its stored originalValue. + originalValue = conflictingPref.originalValue; + } else if ( + prefBranch === "user" && + !Services.prefs.prefHasUserValue(prefName) + ) { + // If there is a default value set, then attempting to read the user + // branch would result in returning the default branch value. + originalValue = null; + } else { + originalValue = lazy.PrefUtils.getPref(prefName, { + branch: prefBranch, + }); + } + + prefs.push({ + name: prefName, + branch: prefBranch, + featureId, + variable, + originalValue, + }); + + // An experiment takes precedence if there is already a pref set. + if (!isRollout || !conflictingPref) { + prefsToSet.push({ name: prefName, value, prefBranch }); + } + } + } + } + + return { prefs, prefsToSet }; + } + + /** + * Set a list of prefs from enrolling in an experiment or rollout. + * + * The ExperimentManager's pref observers will be disabled while setting each + * pref so as not to accidentally unenroll an existing rollout that an + * experiment would override. + * + * @param {object[]} prefsToSet + * A list of objects containing the prefs to set. + * + * Each object has the following properties: + * + * * `name`: The name of the pref. + * * `value`: The value of the pref. + * * `prefBranch`: The branch to set the pref on (either "user" or "default"). + */ + _setEnrollmentPrefs(prefsToSet) { + for (const { name, value, prefBranch } of prefsToSet) { + const entry = this._prefs.get(name); + + // If another enrollment exists that has set this pref, temporarily + // disable the pref observer so as not to cause unenrollment. + if (entry) { + entry.enrollmentChanging = true; + } + + lazy.PrefUtils.setPref(name, value, { branch: prefBranch }); + + if (entry) { + entry.enrollmentChanging = false; + } + } + } + + /** + * Unset prefs set during this enrollment. + * + * If this enrollment is an experiment and there is an existing rollout that + * would set a pref that was covered by this enrollment, the pref will be + * updated to that rollout's value. + * + * Otherwise, it will be set to the original value from before the enrollment + * began. + * + * @param {Enrollment} enrollment + * The enrollment that has ended. + * + * @param {object} options + * + * @param {object?} options.changedPref + * If provided, a changed pref that caused the unenrollment that + * triggered unsetting these prefs. This is provided as to not + * overwrite a changed pref with an original value. + * + * @param {string} options.changedPref.name + * The name of the changed pref. + * + * @param {string} options.changedPref.branch + * The branch that was changed ("user" or "default"). + * + * @param {boolean} options.duringRestore + * The unenrollment was caused during restore. + */ + _unsetEnrollmentPrefs(enrollment, { changedPref, duringRestore } = {}) { + if (!enrollment.prefs?.length) { + return; + } + + const getConflictingEnrollment = this._makeEnrollmentCache( + enrollment.isRollout + ); + + for (const pref of enrollment.prefs) { + this._removePrefObserver(pref.name, enrollment.slug); + + if ( + changedPref?.name == pref.name && + changedPref.branch === pref.branch + ) { + // Resetting the original value would overwite the pref the user just + // set. Skip it. + continue; + } + + let newValue = pref.originalValue; + + // If we are unenrolling from an experiment during a restore, we must + // ignore any potential conflicting rollout in the store, because its + // hasn't gone through `_restoreEnrollmentPrefs`, which might also cause + // it to unenroll. + // + // Both enrollments will have the same `originalValue` stored, so it will + // always be restored. + if (!duringRestore || enrollment.isRollout) { + const conflictingEnrollment = getConflictingEnrollment(pref.featureId); + const conflictingPref = conflictingEnrollment?.prefs?.find( + p => p.name === pref.name + ); + + if (conflictingPref) { + if (enrollment.isRollout) { + // If we are unenrolling from a rollout, we have an experiment that + // has set the pref. Since experiments take priority, we do not unset + // it. + continue; + } else { + // If we are an unenrolling from an experiment, we have a rollout that would + // set the same pref, so we update the pref to that value instead of + // the original value. + newValue = getFeatureFromBranch( + conflictingEnrollment.branch, + pref.featureId + ).value[pref.variable]; + } + } + } + + // If another enrollment exists that has set this pref, temporarily + // disable the pref observer so as not to cause unenrollment when we + // update the pref to its value. + const entry = this._prefs.get(pref.name); + if (entry) { + entry.enrollmentChanging = true; + } + + lazy.PrefUtils.setPref(pref.name, newValue, { + branch: pref.branch, + }); + + if (entry) { + entry.enrollmentChanging = false; + } + } + } + + /** + * Restore the prefs set by an enrollment. + * + * @param {object} enrollment The enrollment. + * @param {object} enrollment.branch The branch that was enrolled. + * @param {object[]} enrollment.prefs The prefs that are set by the enrollment. + * @param {object[]} enrollment.isRollout The prefs that are set by the enrollment. + * + * @returns {boolean} Whether the restore was successful. If false, the + * enrollment has ended. + */ + _restoreEnrollmentPrefs(enrollment) { + const { branch, prefs = [], isRollout } = enrollment; + + if (!prefs?.length) { + return false; + } + + const featuresById = Object.assign( + ...featuresCompat(branch).map(f => ({ [f.featureId]: f })) + ); + + for (const { name, featureId, variable } of prefs) { + // If the feature no longer exists, unenroll. + if (!Object.hasOwn(lazy.NimbusFeatures, featureId)) { + this._unenroll(enrollment, { + reason: "invalid-feature", + duringRestore: true, + }); + return false; + } + + const variables = lazy.NimbusFeatures[featureId].manifest.variables; + + // If the feature is missing a variable that set a pref, unenroll. + if (!Object.hasOwn(variables, variable)) { + this._unenroll(enrollment, { + reason: "pref-variable-missing", + duringRestore: true, + }); + return false; + } + + const variableDef = variables[variable]; + + // If the variable is no longer a pref-setting variable, unenroll. + if (!Object.hasOwn(variableDef, "setPref")) { + this._unenroll(enrollment, { + reason: "pref-variable-no-longer", + duringRestore: true, + }); + return false; + } + + // If the variable is setting a different preference, unenroll. + if (variableDef.setPref !== name) { + this._unenroll(enrollment, { + reason: "pref-variable-changed", + duringRestore: true, + }); + return false; + } + } + + for (const { name, branch: prefBranch, featureId, variable } of prefs) { + // User prefs are already persisted. + if (prefBranch === "user") { + continue; + } + + // If we are a rollout, we need to check for an existing experiment that + // has set the same pref. If so, we do not need to set the pref because + // experiments take priority. + if (isRollout) { + const conflictingEnrollment = + this.store.getExperimentForFeature(featureId); + const conflictingPref = conflictingEnrollment?.prefs?.find( + p => p.name === name + ); + + if (conflictingPref) { + continue; + } + } + + const value = featuresById[featureId].value[variable]; + + if (prefBranch !== "user") { + lazy.PrefUtils.setPref(name, value, { branch: prefBranch }); + } + } + + return true; + } + + /** + * Make a cache to look up enrollments of the oppposite kind by feature ID. + * + * @param {boolean} isRollout Whether or not the current enrollment is a + * rollout. If true, the cache will return + * experiments. If false, the cache will return + * rollouts. + * + * @returns {function} The cache, as a callable function. + */ + _makeEnrollmentCache(isRollout) { + const getOtherEnrollment = ( + isRollout + ? this.store.getExperimentForFeature + : this.store.getRolloutForFeature + ).bind(this.store); + + const conflictingEnrollments = {}; + return featureId => { + if (!Object.hasOwn(conflictingEnrollments, featureId)) { + conflictingEnrollments[featureId] = getOtherEnrollment(featureId); + } + + return conflictingEnrollments[featureId]; + }; + } + + /** + * Update the set of observers with prefs set by the given enrollment. + * + * @param {Enrollment} enrollment + * The enrollment that is setting prefs. + */ + _updatePrefObservers({ slug, prefs }) { + if (!prefs?.length) { + return; + } + + for (const pref of prefs) { + const { name } = pref; + + if (!this._prefs.has(name)) { + const observer = () => this._onExperimentPrefChanged(pref); + const entry = { + slugs: new Set([slug]), + enrollmentChanging: false, + observer, + }; + + Services.prefs.addObserver(name, observer); + + this._prefs.set(name, entry); + } else { + this._prefs.get(name).slugs.add(slug); + } + + if (!this._prefsBySlug.has(slug)) { + this._prefsBySlug.set(slug, new Set([name])); + } else { + this._prefsBySlug.get(slug).add(name); + } + } + } + + /** + * Remove an entry for the pref observer for the given pref and slug. + * + * If there are no more enrollments listening to a pref, the observer will be removed. + * + * This is called when an enrollment is ending. + * + * @param {string} name The name of the pref. + * @param {string} slug The slug of the enrollment that is being unenrolled. + */ + _removePrefObserver(name, slug) { + // Update the pref observer that the current enrollment is no longer + // involved in the pref. + // + // If no enrollments have a variable setting the pref, then we can remove + // the observers. + const entry = this._prefs.get(name); + + // If this is happening due to a pref change, the observers will already be removed. + if (entry) { + entry.slugs.delete(slug); + if (entry.slugs.size == 0) { + Services.prefs.removeObserver(name, entry.observer); + this._prefs.delete(name); + } + } + + const bySlug = this._prefsBySlug.get(slug); + if (bySlug) { + bySlug.delete(name); + if (bySlug.size == 0) { + this._prefsBySlug.delete(slug); + } + } + } + + /** + * Handle a change to a pref set by enrollments by ending those enrollments. + * + * @param {object} pref + * Information about the pref that was changed. + * + * @param {string} pref.name + * The name of the pref that was changed. + * + * @param {string} pref.branch + * The branch enrollments set the pref on. + * + * @param {string} pref.featureId + * The feature ID of the feature containing the variable that set the + * pref. + * + * @param {string} pref.variable + * The variable in the given feature whose value determined the pref's + * value. + */ + _onExperimentPrefChanged(pref) { + const entry = this._prefs.get(pref.name); + // If this was triggered while we are enrolling or unenrolling from an + // experiment, then we don't want to unenroll from the rollout because the + // experiment's value is taking precendence. + // + // Otherwise, all enrollments that set the variable corresponding to this + // pref must be unenrolled. + if (entry.enrollmentChanging) { + return; + } + + // Copy the `Set` into an `Array` because we modify the set later in + // `_removePrefObserver` and we need to iterate over it multiple times. + const slugs = Array.from(entry.slugs); + + // Remove all pref observers set by enrollments. We are potentially about + // to set these prefs during unenrollment, so we don't want to trigger + // them and cause nested unenrollments. + for (const slug of slugs) { + const toRemove = Array.from(this._prefsBySlug.get(slug) ?? []); + for (const name of toRemove) { + this._removePrefObserver(name, slug); + } + } + + // Unenroll from the rollout first to save calls to setPref. + const enrollments = Array.from(slugs).map(slug => this.store.get(slug)); + + // There is a maximum of two enrollments (one experiment and one rollout). + if (enrollments.length == 2) { + // Order enrollments so that we unenroll from the rollout first. + if (!enrollments[0].isRollout) { + enrollments.reverse(); + } + } + + // We want to know what branch was changed so we can know if we should + // restore prefs. (e.g., if we have a pref set on the user branch and the + // user branch changed, we do not want to then overwrite the user's choice). + + // This is not complicated if a pref simply changed. However, we also must + // detect `nsIPrefBranch::clearUserPref()`, which wipes out the user branch + // and leaves the default branch untouched. That is where this gets + // complicated: + + let branch; + if (Services.prefs.prefHasUserValue(pref.name)) { + // If there is a user branch value, then the user branch changed. + branch = "user"; + } else if (!Services.prefs.prefHasDefaultValue(pref.name)) { + // If there is not default branch value, then the user branch must have + // been cleared becuase you cannot clear the default branch. + branch = "user"; + } else if (pref.branch === "default") { + const feature = getFeatureFromBranch( + enrollments.at(-1).branch, + pref.featureId + ); + const expectedValue = feature.value[pref.variable]; + const value = lazy.PrefUtils.getPref(pref.name, { branch: pref.branch }); + + if (value === expectedValue) { + // If the pref was set on the default branch and still matches the + // expected value, then the user branch must have been cleared. + branch = "user"; + } else { + branch = "default"; + } + } else { + // If the pref was set on the user branch and we don't have a user branch + // value, then the user branch must have been cleared. + branch = "user"; + } + + const changedPref = { + name: pref.name, + branch, + }; + + for (const enrollment of enrollments) { + this._unenroll(enrollment, { reason: "changed-pref", changedPref }); + } + } +} + +export const ExperimentManager = new _ExperimentManager(); diff --git a/toolkit/components/nimbus/lib/ExperimentStore.sys.mjs b/toolkit/components/nimbus/lib/ExperimentStore.sys.mjs new file mode 100644 index 0000000000..43f47524e2 --- /dev/null +++ b/toolkit/components/nimbus/lib/ExperimentStore.sys.mjs @@ -0,0 +1,485 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { SharedDataMap } from "resource://nimbus/lib/SharedDataMap.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + FeatureManifest: "resource://nimbus/FeatureManifest.sys.mjs", + PrefUtils: "resource://normandy/lib/PrefUtils.sys.mjs", +}); + +const IS_MAIN_PROCESS = + Services.appinfo.processType === Services.appinfo.PROCESS_TYPE_DEFAULT; + +// This branch is used to store experiment data +const SYNC_DATA_PREF_BRANCH = "nimbus.syncdatastore."; +// This branch is used to store remote rollouts +const SYNC_DEFAULTS_PREF_BRANCH = "nimbus.syncdefaultsstore."; +let tryJSONParse = data => { + try { + return JSON.parse(data); + } catch (e) {} + + return null; +}; +XPCOMUtils.defineLazyGetter(lazy, "syncDataStore", () => { + let experimentsPrefBranch = Services.prefs.getBranch(SYNC_DATA_PREF_BRANCH); + let defaultsPrefBranch = Services.prefs.getBranch(SYNC_DEFAULTS_PREF_BRANCH); + return { + _tryParsePrefValue(branch, pref) { + try { + return tryJSONParse(branch.getStringPref(pref, "")); + } catch (e) { + /* This is expected if we don't have anything stored */ + } + + return null; + }, + _trySetPrefValue(branch, pref, value) { + try { + branch.setStringPref(pref, JSON.stringify(value)); + } catch (e) { + console.error(e); + } + }, + _trySetTypedPrefValue(pref, value) { + let variableType = typeof value; + switch (variableType) { + case "boolean": + Services.prefs.setBoolPref(pref, value); + break; + case "number": + Services.prefs.setIntPref(pref, value); + break; + case "string": + Services.prefs.setStringPref(pref, value); + break; + case "object": + Services.prefs.setStringPref(pref, JSON.stringify(value)); + break; + } + }, + _clearBranchChildValues(prefBranch) { + const variablesBranch = Services.prefs.getBranch(prefBranch); + const prefChildList = variablesBranch.getChildList(""); + for (let variable of prefChildList) { + variablesBranch.clearUserPref(variable); + } + }, + /** + * Given a branch pref returns all child prefs and values + * { childPref: value } + * where value is parsed to the appropriate type + * + * @returns {Object[]} + */ + _getBranchChildValues(prefBranch, featureId) { + const branch = Services.prefs.getBranch(prefBranch); + const prefChildList = branch.getChildList(""); + let values = {}; + if (!prefChildList.length) { + return null; + } + for (const childPref of prefChildList) { + let prefName = `${prefBranch}${childPref}`; + let value = lazy.PrefUtils.getPref(prefName); + // Try to parse string values that could be stringified objects + if ( + lazy.FeatureManifest[featureId]?.variables[childPref]?.type === "json" + ) { + let parsedValue = tryJSONParse(value); + if (parsedValue) { + value = parsedValue; + } + } + values[childPref] = value; + } + + return values; + }, + get(featureId) { + let metadata = this._tryParsePrefValue(experimentsPrefBranch, featureId); + if (!metadata) { + return null; + } + let prefBranch = `${SYNC_DATA_PREF_BRANCH}${featureId}.`; + metadata.branch.feature.value = this._getBranchChildValues( + prefBranch, + featureId + ); + + return metadata; + }, + getDefault(featureId) { + let metadata = this._tryParsePrefValue(defaultsPrefBranch, featureId); + if (!metadata) { + return null; + } + let prefBranch = `${SYNC_DEFAULTS_PREF_BRANCH}${featureId}.`; + metadata.branch.feature.value = this._getBranchChildValues( + prefBranch, + featureId + ); + + return metadata; + }, + set(featureId, value) { + /* If the enrollment branch has variables we store those separately + * in pref branches of appropriate type: + * { featureId: "foo", value: { enabled: true } } + * gets stored as `${SYNC_DATA_PREF_BRANCH}foo.enabled=true` + */ + if (value.branch?.feature?.value) { + for (let variable of Object.keys(value.branch.feature.value)) { + let prefName = `${SYNC_DATA_PREF_BRANCH}${featureId}.${variable}`; + this._trySetTypedPrefValue( + prefName, + value.branch.feature.value[variable] + ); + } + this._trySetPrefValue(experimentsPrefBranch, featureId, { + ...value, + branch: { + ...value.branch, + feature: { + ...value.branch.feature, + value: null, + }, + }, + }); + } else { + this._trySetPrefValue(experimentsPrefBranch, featureId, value); + } + }, + setDefault(featureId, enrollment) { + /* We store configuration variables separately in pref branches of + * appropriate type: + * (feature: "foo") { variables: { enabled: true } } + * gets stored as `${SYNC_DEFAULTS_PREF_BRANCH}foo.enabled=true` + */ + let { feature } = enrollment.branch; + for (let variable of Object.keys(feature.value)) { + let prefName = `${SYNC_DEFAULTS_PREF_BRANCH}${featureId}.${variable}`; + this._trySetTypedPrefValue(prefName, feature.value[variable]); + } + this._trySetPrefValue(defaultsPrefBranch, featureId, { + ...enrollment, + branch: { + ...enrollment.branch, + feature: { + ...enrollment.branch.feature, + value: null, + }, + }, + }); + }, + getAllDefaultBranches() { + return defaultsPrefBranch.getChildList("").filter( + // Filter out remote defaults variable prefs + pref => !pref.includes(".") + ); + }, + delete(featureId) { + const prefBranch = `${SYNC_DATA_PREF_BRANCH}${featureId}.`; + this._clearBranchChildValues(prefBranch); + try { + experimentsPrefBranch.clearUserPref(featureId); + } catch (e) {} + }, + deleteDefault(featureId) { + let prefBranch = `${SYNC_DEFAULTS_PREF_BRANCH}${featureId}.`; + this._clearBranchChildValues(prefBranch); + try { + defaultsPrefBranch.clearUserPref(featureId); + } catch (e) {} + }, + }; +}); + +const DEFAULT_STORE_ID = "ExperimentStoreData"; + +/** + * Returns all feature ids associated with the branch provided. + * Fallback for when `featureIds` was not persisted to disk. Can be removed + * after bug 1725240 has reached release. + * + * @param {Branch} branch + * @returns {string[]} + */ +function getAllBranchFeatureIds(branch) { + return featuresCompat(branch).map(f => f.featureId); +} + +function featuresCompat(branch) { + if (!branch || (!branch.feature && !branch.features)) { + return []; + } + let { features } = branch; + // In <=v1.5.0 of the Nimbus API, experiments had single feature + if (!features) { + features = [branch.feature]; + } + + return features; +} + +export class ExperimentStore extends SharedDataMap { + static SYNC_DATA_PREF_BRANCH = SYNC_DATA_PREF_BRANCH; + static SYNC_DEFAULTS_PREF_BRANCH = SYNC_DEFAULTS_PREF_BRANCH; + + constructor(sharedDataKey, options = { isParent: IS_MAIN_PROCESS }) { + super(sharedDataKey || DEFAULT_STORE_ID, options); + } + + async init() { + await super.init(); + + this.getAllActiveExperiments().forEach(({ slug, branch, featureIds }) => { + (featureIds || getAllBranchFeatureIds(branch)).forEach(featureId => + this._emitFeatureUpdate(featureId, "feature-experiment-loaded") + ); + }); + this.getAllActiveRollouts().forEach(({ featureIds }) => { + featureIds.forEach(featureId => + this._emitFeatureUpdate(featureId, "feature-rollout-loaded") + ); + }); + + Services.tm.idleDispatchToMainThread(() => this._cleanupOldRecipes()); + } + + /** + * Given a feature identifier, find an active experiment that matches that feature identifier. + * This assumes, for now, that there is only one active experiment per feature per browser. + * Does not activate the experiment (send an exposure event) + * + * @param {string} featureId + * @returns {Enrollment|undefined} An active experiment if it exists + * @memberof ExperimentStore + */ + getExperimentForFeature(featureId) { + return ( + this.getAllActiveExperiments().find( + experiment => + experiment.featureIds?.includes(featureId) || + // Supports <v1.3.0, which was when .featureIds was added + getAllBranchFeatureIds(experiment.branch).includes(featureId) + // Default to the pref store if data is not yet ready + ) || lazy.syncDataStore.get(featureId) + ); + } + + /** + * Check if an active experiment already exists for a feature. + * Does not activate the experiment (send an exposure event) + * + * @param {string} featureId + * @returns {boolean} Does an active experiment exist for that feature? + * @memberof ExperimentStore + */ + hasExperimentForFeature(featureId) { + if (!featureId) { + return false; + } + return !!this.getExperimentForFeature(featureId); + } + + /** + * @returns {Enrollment[]} + */ + getAll() { + let data = []; + try { + data = Object.values(this._data || {}); + } catch (e) { + console.error(e); + } + + return data; + } + + /** + * Returns all active experiments + * @returns {Enrollment[]} + */ + getAllActiveExperiments() { + return this.getAll().filter( + enrollment => enrollment.active && !enrollment.isRollout + ); + } + + /** + * Returns all active rollouts + * @returns {Enrollment[]} + */ + getAllActiveRollouts() { + return this.getAll().filter( + enrollment => enrollment.active && enrollment.isRollout + ); + } + + /** + * Query the store for the remote configuration of a feature + * @param {string} featureId The feature we want to query for + * @returns {{Rollout}|undefined} Remote defaults if available + */ + getRolloutForFeature(featureId) { + return ( + this.getAllActiveRollouts().find(r => r.featureIds.includes(featureId)) || + lazy.syncDataStore.getDefault(featureId) + ); + } + + /** + * Check if an active rollout already exists for a feature. + * Does not active the experiment (send an exposure event). + * + * @param {string} featureId + * @returns {boolean} Does an active rollout exist for that feature? + */ + hasRolloutForFeature(featureId) { + if (!featureId) { + return false; + } + return !!this.getRolloutForFeature(featureId); + } + + /** + * Remove inactive enrollments older than 6 months + */ + _cleanupOldRecipes() { + // Roughly six months + const threshold = 15552000000; + const nowTimestamp = new Date().getTime(); + const recipesToRemove = this.getAll().filter( + experiment => + !experiment.active && + // Flip the comparison here to catch scenarios in which lastSeen is + // invalid or undefined. The result with be a comparison with NaN + // which is always false + !(nowTimestamp - new Date(experiment.lastSeen).getTime() < threshold) + ); + this._removeEntriesByKeys(recipesToRemove.map(r => r.slug)); + } + + _emitUpdates(enrollment) { + this.emit(`update:${enrollment.slug}`, enrollment); + const featureIds = + enrollment.featureIds || getAllBranchFeatureIds(enrollment.branch); + const reason = enrollment.isRollout + ? "rollout-updated" + : "experiment-updated"; + + for (const featureId of featureIds) { + this._emitFeatureUpdate(featureId, reason); + } + } + + _emitFeatureUpdate(featureId, reason) { + this.emit(`featureUpdate:${featureId}`, reason); + } + + _onFeatureUpdate(featureId, callback) { + if (this._isReady) { + const hasExperiment = this.hasExperimentForFeature(featureId); + if (hasExperiment || this.hasRolloutForFeature(featureId)) { + callback( + `featureUpdate:${featureId}`, + hasExperiment ? "experiment-updated" : "rollout-updated" + ); + } + } + + this.on(`featureUpdate:${featureId}`, callback); + } + + _offFeatureUpdate(featureId, callback) { + this.off(`featureUpdate:${featureId}`, callback); + } + + /** + * Persists early startup experiments or rollouts + * @param {Enrollment} enrollment Experiment or rollout + */ + _updateSyncStore(enrollment) { + let features = featuresCompat(enrollment.branch); + for (let feature of features) { + if ( + lazy.FeatureManifest[feature.featureId]?.isEarlyStartup || + feature.isEarlyStartup + ) { + if (!enrollment.active) { + // Remove experiments on un-enroll, no need to check if it exists + if (enrollment.isRollout) { + lazy.syncDataStore.deleteDefault(feature.featureId); + } else { + lazy.syncDataStore.delete(feature.featureId); + } + } else { + let updateEnrollmentSyncStore = enrollment.isRollout + ? lazy.syncDataStore.setDefault.bind(lazy.syncDataStore) + : lazy.syncDataStore.set.bind(lazy.syncDataStore); + updateEnrollmentSyncStore(feature.featureId, { + ...enrollment, + branch: { + ...enrollment.branch, + feature, + // Only store the early startup feature + features: null, + }, + }); + } + } + } + } + + /** + * Add an enrollment and notify listeners + * @param {Enrollment} enrollment + */ + addEnrollment(enrollment) { + if (!enrollment || !enrollment.slug) { + throw new Error( + `Tried to add an experiment but it didn't have a .slug property.` + ); + } + + this.set(enrollment.slug, enrollment); + this._updateSyncStore(enrollment); + this._emitUpdates(enrollment); + } + + /** + * Merge new properties into the properties of an existing experiment + * @param {string} slug + * @param {Partial<Enrollment>} newProperties + */ + updateExperiment(slug, newProperties) { + const oldProperties = this.get(slug); + if (!oldProperties) { + throw new Error( + `Tried to update experiment ${slug} but it doesn't exist` + ); + } + const updatedExperiment = { ...oldProperties, ...newProperties }; + this.set(slug, updatedExperiment); + this._updateSyncStore(updatedExperiment); + this._emitUpdates(updatedExperiment); + } + + /** + * Test only helper for cleanup + * + * @param slugOrFeatureId Can be called with slug (which removes the SharedDataMap entry) or + * with featureId which removes the SyncDataStore entry for the feature + */ + _deleteForTests(slugOrFeatureId) { + super._deleteForTests(slugOrFeatureId); + lazy.syncDataStore.deleteDefault(slugOrFeatureId); + lazy.syncDataStore.delete(slugOrFeatureId); + } +} diff --git a/toolkit/components/nimbus/lib/NimbusFeatureManifest.inc.h b/toolkit/components/nimbus/lib/NimbusFeatureManifest.inc.h new file mode 100644 index 0000000000..4d3190635c --- /dev/null +++ b/toolkit/components/nimbus/lib/NimbusFeatureManifest.inc.h @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +Maybe<nsCString> GetNimbusFallbackPrefName(const nsACString& aFeatureId, + const nsACString& aVariable) { + nsAutoCString manifestKey; + manifestKey.Append(aFeatureId); + manifestKey.Append("_"); + manifestKey.Append(aVariable); + + for (const auto& pair : NIMBUS_FALLBACK_PREFS) { + if (pair.first.Equals(manifestKey.get())) { + return Some(pair.second); + } + } + return Nothing{}; +} diff --git a/toolkit/components/nimbus/lib/NimbusFeatures.cpp b/toolkit/components/nimbus/lib/NimbusFeatures.cpp new file mode 100644 index 0000000000..4b580149f5 --- /dev/null +++ b/toolkit/components/nimbus/lib/NimbusFeatures.cpp @@ -0,0 +1,207 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/browser/NimbusFeatures.h" +#include "mozilla/browser/NimbusFeatureManifest.h" +#include "mozilla/Telemetry.h" +#include "mozilla/dom/ScriptSettings.h" +#include "jsapi.h" +#include "js/JSON.h" +#include "nsJSUtils.h" + +namespace mozilla { + +static nsTHashSet<nsCString> sExposureFeatureSet; + +void NimbusFeatures::GetPrefName(const nsACString& branchPrefix, + const nsACString& aFeatureId, + const nsACString& aVariable, + nsACString& aPref) { + nsAutoCString featureAndVariable; + featureAndVariable.Append(aFeatureId); + if (!aVariable.IsEmpty()) { + featureAndVariable.Append("."); + featureAndVariable.Append(aVariable); + } + aPref.Truncate(); + aPref.Append(branchPrefix); + aPref.Append(featureAndVariable); +} + +/** + * Returns the variable value configured via experiment or rollout. + * If a fallback pref is defined in the FeatureManifest and it + * has a user value set this takes precedence over remote configurations. + */ +bool NimbusFeatures::GetBool(const nsACString& aFeatureId, + const nsACString& aVariable, bool aDefault) { + nsAutoCString experimentPref; + GetPrefName(kSyncDataPrefBranch, aFeatureId, aVariable, experimentPref); + if (Preferences::HasUserValue(experimentPref.get())) { + return Preferences::GetBool(experimentPref.get(), aDefault); + } + + nsAutoCString rolloutPref; + GetPrefName(kSyncRolloutsPrefBranch, aFeatureId, aVariable, rolloutPref); + if (Preferences::HasUserValue(rolloutPref.get())) { + return Preferences::GetBool(rolloutPref.get(), aDefault); + } + + auto prefName = GetNimbusFallbackPrefName(aFeatureId, aVariable); + if (prefName.isSome()) { + return Preferences::GetBool(prefName->get(), aDefault); + } + return aDefault; +} + +/** + * Returns the variable value configured via experiment or rollout. + * If a fallback pref is defined in the FeatureManifest and it + * has a user value set this takes precedence over remote configurations. + */ +int NimbusFeatures::GetInt(const nsACString& aFeatureId, + const nsACString& aVariable, int aDefault) { + nsAutoCString experimentPref; + GetPrefName(kSyncDataPrefBranch, aFeatureId, aVariable, experimentPref); + if (Preferences::HasUserValue(experimentPref.get())) { + return Preferences::GetInt(experimentPref.get(), aDefault); + } + + nsAutoCString rolloutPref; + GetPrefName(kSyncRolloutsPrefBranch, aFeatureId, aVariable, rolloutPref); + if (Preferences::HasUserValue(rolloutPref.get())) { + return Preferences::GetInt(rolloutPref.get(), aDefault); + } + + auto prefName = GetNimbusFallbackPrefName(aFeatureId, aVariable); + if (prefName.isSome()) { + return Preferences::GetInt(prefName->get(), aDefault); + } + return aDefault; +} + +nsresult NimbusFeatures::OnUpdate(const nsACString& aFeatureId, + const nsACString& aVariable, + PrefChangedFunc aUserCallback, + void* aUserData) { + nsAutoCString experimentPref; + nsAutoCString rolloutPref; + GetPrefName(kSyncDataPrefBranch, aFeatureId, aVariable, experimentPref); + GetPrefName(kSyncRolloutsPrefBranch, aFeatureId, aVariable, rolloutPref); + nsresult rv = + Preferences::RegisterCallback(aUserCallback, experimentPref, aUserData); + NS_ENSURE_SUCCESS(rv, rv); + rv = Preferences::RegisterCallback(aUserCallback, rolloutPref, aUserData); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult NimbusFeatures::OffUpdate(const nsACString& aFeatureId, + const nsACString& aVariable, + PrefChangedFunc aUserCallback, + void* aUserData) { + nsAutoCString experimentPref; + nsAutoCString rolloutPref; + GetPrefName(kSyncDataPrefBranch, aFeatureId, aVariable, experimentPref); + GetPrefName(kSyncRolloutsPrefBranch, aFeatureId, aVariable, rolloutPref); + nsresult rv = + Preferences::UnregisterCallback(aUserCallback, experimentPref, aUserData); + NS_ENSURE_SUCCESS(rv, rv); + rv = Preferences::UnregisterCallback(aUserCallback, rolloutPref, aUserData); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +/** + * Attempt to read Nimbus preference to determine experiment and branch slug. + * Nimbus will store a pref with experiment metadata in the following format: + * { + * slug: "experiment slug", + * branch: { slug: "branch slug" }, + * ... + * } + * The naming convention for preference names is: + * `nimbus.syncdatastore.<feature_id>` + * These values are used to send `exposure` telemetry pings. + */ +nsresult NimbusFeatures::GetExperimentSlug(const nsACString& aFeatureId, + nsACString& aExperimentSlug, + nsACString& aBranchSlug) { + nsAutoCString prefName; + nsAutoString prefValue; + + aExperimentSlug.Truncate(); + aBranchSlug.Truncate(); + + GetPrefName(kSyncDataPrefBranch, aFeatureId, EmptyCString(), prefName); + MOZ_TRY(Preferences::GetString(prefName.get(), prefValue)); + if (prefValue.IsEmpty()) { + return NS_ERROR_UNEXPECTED; + } + dom::AutoJSAPI jsapi; + if (!jsapi.Init(xpc::PrivilegedJunkScope())) { + return NS_ERROR_UNEXPECTED; + } + JSContext* cx = jsapi.cx(); + JS::Rooted<JS::Value> json(cx, JS::NullValue()); + if (JS_ParseJSON(cx, prefValue.BeginReading(), prefValue.Length(), &json) && + json.isObject()) { + JS::Rooted<JSObject*> experimentJSON(cx, json.toObjectOrNull()); + JS::Rooted<JS::Value> expSlugValue(cx); + if (!JS_GetProperty(cx, experimentJSON, "slug", &expSlugValue)) { + return NS_ERROR_UNEXPECTED; + } + AssignJSString(cx, aExperimentSlug, expSlugValue.toString()); + + JS::Rooted<JS::Value> branchJSON(cx); + if (!JS_GetProperty(cx, experimentJSON, "branch", &branchJSON) && + !branchJSON.isObject()) { + return NS_ERROR_UNEXPECTED; + } + JS::Rooted<JSObject*> branchObj(cx, branchJSON.toObjectOrNull()); + JS::Rooted<JS::Value> branchSlugValue(cx); + if (!JS_GetProperty(cx, branchObj, "slug", &branchSlugValue)) { + return NS_ERROR_UNEXPECTED; + } + AssignJSString(cx, aBranchSlug, branchSlugValue.toString()); + } + + return NS_OK; +} + +/** + * Sends an exposure event for aFeatureId when enrolled in an experiment. + * By default attempt to send once per function call. For some usecases it might + * be useful to send only once, in which case set the optional aOnce to `true`. + */ +nsresult NimbusFeatures::RecordExposureEvent(const nsACString& aFeatureId, + const bool aOnce) { + nsAutoCString featureName(aFeatureId); + if (!sExposureFeatureSet.EnsureInserted(featureName) && aOnce) { + // We already sent (or tried to send) an exposure ping for this featureId + return NS_ERROR_ABORT; + } + nsAutoCString slugName; + nsAutoCString branchName; + MOZ_TRY(GetExperimentSlug(aFeatureId, slugName, branchName)); + if (slugName.IsEmpty() || branchName.IsEmpty()) { + // Failed getting experiment metadata or not enrolled in an experiment for + // this featureId + return NS_ERROR_UNEXPECTED; + } + Telemetry::SetEventRecordingEnabled("normandy"_ns, true); + nsTArray<Telemetry::EventExtraEntry> extra(2); + extra.AppendElement(Telemetry::EventExtraEntry{"branchSlug"_ns, branchName}); + extra.AppendElement(Telemetry::EventExtraEntry{"featureId"_ns, featureName}); + Telemetry::RecordEvent(Telemetry::EventID::Normandy_Expose_NimbusExperiment, + Some(slugName), Some(std::move(extra))); + + return NS_OK; +} + +} // namespace mozilla diff --git a/toolkit/components/nimbus/lib/NimbusFeatures.h b/toolkit/components/nimbus/lib/NimbusFeatures.h new file mode 100644 index 0000000000..7fe9ac8b5c --- /dev/null +++ b/toolkit/components/nimbus/lib/NimbusFeatures.h @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_NimbusFeatures_h +#define mozilla_NimbusFeatures_h + +#include "mozilla/Preferences.h" +#include "nsTHashSet.h" + +namespace mozilla { + +class NimbusFeatures { + private: + // This branch is used to store experiment data + static constexpr auto kSyncDataPrefBranch = "nimbus.syncdatastore."_ns; + // This branch is used to store rollouts data + static constexpr auto kSyncRolloutsPrefBranch = + "nimbus.syncdefaultsstore."_ns; + static void GetPrefName(const nsACString& branchPrefix, + const nsACString& aFeatureId, + const nsACString& aVariable, nsACString& aPref); + + static nsresult GetExperimentSlug(const nsACString& aFeatureId, + nsACString& aExperimentSlug, + nsACString& aBranchSlug); + + public: + static bool GetBool(const nsACString& aFeatureId, const nsACString& aVariable, + bool aDefault); + + static int GetInt(const nsACString& aFeatureId, const nsACString& aVariable, + int aDefault); + + static nsresult OnUpdate(const nsACString& aFeatureId, + const nsACString& aVariable, + PrefChangedFunc aUserCallback, void* aUserData); + + static nsresult OffUpdate(const nsACString& aFeatureId, + const nsACString& aVariable, + PrefChangedFunc aUserCallback, void* aUserData); + + static nsresult RecordExposureEvent(const nsACString& aFeatureId, + const bool aOnce = false); +}; + +} // namespace mozilla + +#endif diff --git a/toolkit/components/nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs b/toolkit/components/nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs new file mode 100644 index 0000000000..3d16027bfb --- /dev/null +++ b/toolkit/components/nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs @@ -0,0 +1,716 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + _ExperimentFeature: "resource://nimbus/ExperimentAPI.sys.mjs", + CleanupManager: "resource://normandy/lib/CleanupManager.sys.mjs", + ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + TargetingContext: "resource://messaging-system/targeting/Targeting.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + ASRouterTargeting: "resource://activity-stream/lib/ASRouterTargeting.jsm", +}); + +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("RSLoader"); +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "timerManager", + "@mozilla.org/updates/timer-manager;1", + "nsIUpdateTimerManager" +); + +const COLLECTION_ID_PREF = "messaging-system.rsexperimentloader.collection_id"; +const COLLECTION_ID_FALLBACK = "nimbus-desktop-experiments"; +const ENABLED_PREF = "messaging-system.rsexperimentloader.enabled"; + +const TIMER_NAME = "rs-experiment-loader-timer"; +const TIMER_LAST_UPDATE_PREF = `app.update.lastUpdateTime.${TIMER_NAME}`; +// Use the same update interval as normandy +const RUN_INTERVAL_PREF = "app.normandy.run_interval_seconds"; +const NIMBUS_DEBUG_PREF = "nimbus.debug"; +const NIMBUS_VALIDATION_PREF = "nimbus.validation.enabled"; +const NIMBUS_APPID_PREF = "nimbus.appId"; + +const STUDIES_ENABLED_CHANGED = "nimbus:studies-enabled-changed"; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "COLLECTION_ID", + COLLECTION_ID_PREF, + COLLECTION_ID_FALLBACK +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "NIMBUS_DEBUG", + NIMBUS_DEBUG_PREF, + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "APP_ID", + NIMBUS_APPID_PREF, + "firefox-desktop" +); + +const SCHEMAS = { + get NimbusExperiment() { + return fetch("resource://nimbus/schemas/NimbusExperiment.schema.json", { + credentials: "omit", + }) + .then(rsp => rsp.json()) + .then(json => json.definitions.NimbusExperiment); + }, +}; + +export class _RemoteSettingsExperimentLoader { + constructor() { + // Has the timer been set? + this._initialized = false; + // Are we in the middle of updating recipes already? + this._updating = false; + + // Make it possible to override for testing + this.manager = lazy.ExperimentManager; + + XPCOMUtils.defineLazyGetter(this, "remoteSettingsClient", () => { + return lazy.RemoteSettings(lazy.COLLECTION_ID); + }); + + Services.obs.addObserver(this, STUDIES_ENABLED_CHANGED); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "enabled", + ENABLED_PREF, + false, + this.onEnabledPrefChange.bind(this) + ); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "intervalInSeconds", + RUN_INTERVAL_PREF, + 21600, + () => this.setTimer() + ); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "validationEnabled", + NIMBUS_VALIDATION_PREF, + true + ); + } + + get studiesEnabled() { + return this.manager.studiesEnabled; + } + + /** + * Initialize the loader, updating recipes from Remote Settings. + * + * @param {Object} options additional options. + * @param {bool} options.forceSync force Remote Settings to sync recipe collection + * before updating recipes; throw if sync fails. + * @return {Promise} which resolves after initialization and recipes + * are updated. + */ + async init(options = {}) { + const { forceSync = false } = options; + + if (this._initialized || !this.enabled || !this.studiesEnabled) { + return; + } + + this.setTimer(); + lazy.CleanupManager.addCleanupHandler(() => this.uninit()); + this._initialized = true; + + await this.updateRecipes(undefined, { forceSync }); + } + + uninit() { + if (!this._initialized) { + return; + } + lazy.timerManager.unregisterTimer(TIMER_NAME); + this._initialized = false; + this._updating = false; + } + + /** + * Get all recipes from remote settings + * @param {string} trigger What caused the update to occur? + * @param {Object} options additional options. + * @param {bool} options.forceSync force Remote Settings to sync recipe collection + * before updating recipes; throw if sync fails. + * @return {Promise} which resolves after recipes are updated. + */ + async updateRecipes(trigger, options = {}) { + if (this._updating || !this._initialized) { + return; + } + + const { forceSync = false } = options; + + // Since this method is async, the enabled pref could change between await + // points. We don't want to half validate experiments, so we cache this to + // keep it consistent throughout updating. + const validationEnabled = this.validationEnabled; + + this._updating = true; + + lazy.log.debug( + "Updating recipes" + (trigger ? ` with trigger ${trigger}` : "") + ); + + let recipes; + let loadingError = false; + + try { + recipes = await this.remoteSettingsClient.get({ + forceSync, + // Throw instead of returning an empty list. + emptyListFallback: false, + }); + lazy.log.debug(`Got ${recipes.length} recipes from Remote Settings`); + } catch (e) { + lazy.log.debug("Error getting recipes from remote settings."); + loadingError = true; + console.error(e); + } + + let recipeValidator; + + if (validationEnabled) { + recipeValidator = new lazy.JsonSchema.Validator( + await SCHEMAS.NimbusExperiment + ); + } + + const enrollmentsCtx = new EnrollmentsContext( + this.manager, + recipeValidator, + { validationEnabled, shouldCheckTargeting: true } + ); + + if (recipes && !loadingError) { + for (const recipe of recipes) { + if (await enrollmentsCtx.checkRecipe(recipe)) { + await this.manager.onRecipe(recipe, "rs-loader"); + } + } + + lazy.log.debug( + `${enrollmentsCtx.matches} recipes matched. Finalizing ExperimentManager.` + ); + this.manager.onFinalize("rs-loader", enrollmentsCtx.getResults()); + } + + if (trigger !== "timer") { + const lastUpdateTime = Math.round(Date.now() / 1000); + Services.prefs.setIntPref(TIMER_LAST_UPDATE_PREF, lastUpdateTime); + } + + Services.obs.notifyObservers(null, "nimbus:enrollments-updated"); + + this._updating = false; + } + + async optInToExperiment({ slug, branch: branchSlug, collection }) { + lazy.log.debug(`Attempting force enrollment with ${slug} / ${branchSlug}`); + + if (!lazy.NIMBUS_DEBUG) { + lazy.log.debug( + `Force enrollment only works when '${NIMBUS_DEBUG_PREF}' is enabled.` + ); + // More generic error if no debug preference is on. + throw new Error("Could not opt in."); + } + + if (!this.studiesEnabled) { + lazy.log.debug( + "Force enrollment does not work when studies are disabled." + ); + throw new Error("Could not opt in: studies are disabled."); + } + + let recipes; + try { + recipes = await lazy + .RemoteSettings(collection || lazy.COLLECTION_ID) + .get({ + // Throw instead of returning an empty list. + emptyListFallback: false, + }); + } catch (e) { + console.error(e); + throw new Error("Error getting recipes from remote settings."); + } + + const recipe = recipes.find(r => r.slug === slug); + + if (!recipe) { + throw new Error( + `Could not find experiment slug ${slug} in collection ${ + collection || lazy.COLLECTION_ID + }.` + ); + } + + const recipeValidator = new lazy.JsonSchema.Validator( + await SCHEMAS.NimbusExperiment + ); + const enrollmentsCtx = new EnrollmentsContext( + this.manager, + recipeValidator, + { + validationEnabled: this.validationEnabled, + shouldCheckTargeting: false, + } + ); + + if (!(await enrollmentsCtx.checkRecipe(recipe))) { + const results = enrollmentsCtx.getResults(); + + if (results.invalidRecipes.length) { + console.error(`Recipe ${recipe.slug} did not match recipe schema`); + } else if (results.invalidBranches.size) { + // There will only be one entry becuase we only validated a single recipe. + for (const branches of results.invalidBranches.values()) { + for (const branch of branches) { + console.error( + `Recipe ${recipe.slug} failed feature validation for branch ${branch}` + ); + } + } + } else if (results.invalidFeatures) { + for (const featureIds of results.invalidFeatures.values()) { + for (const featureId of featureIds) { + console.error( + `Recipe ${recipe.slug} references unknown feature ID ${featureId}` + ); + } + } + } + + throw new Error(`Recipe ${recipe.slug} failed validation`); + } + + let branch = recipe.branches.find(b => b.slug === branchSlug); + if (!branch) { + throw new Error(`Could not find branch slug ${branchSlug} in ${slug}.`); + } + + await lazy.ExperimentManager.forceEnroll(recipe, branch); + } + + /** + * Handles feature status based on feature pref and STUDIES_OPT_OUT_PREF. + * Changing any of them to false will turn off any recipe fetching and + * processing. + */ + onEnabledPrefChange() { + if (this._initialized && !(this.enabled && this.studiesEnabled)) { + this.uninit(); + } else if (!this._initialized && this.enabled && this.studiesEnabled) { + // If the feature pref is turned on then turn on recipe processing. + // If the opt in pref is turned on then turn on recipe processing only if + // the feature pref is also enabled. + this.init(); + } + } + + observe(aSubect, aTopic, aData) { + if (aTopic === STUDIES_ENABLED_CHANGED) { + this.onEnabledPrefChange(); + } + } + + /** + * Sets a timer to update recipes every this.intervalInSeconds + */ + setTimer() { + if (this.intervalInSeconds === 0) { + // Used in tests where we want to turn this mechanism off + return; + } + // The callbacks will be called soon after the timer is registered + lazy.timerManager.registerTimer( + TIMER_NAME, + () => this.updateRecipes("timer"), + this.intervalInSeconds + ); + lazy.log.debug("Registered update timer"); + } +} + +export class EnrollmentsContext { + constructor( + experimentManager, + recipeValidator, + { validationEnabled = true, shouldCheckTargeting = true } = {} + ) { + this.experimentManager = experimentManager; + this.recipeValidator = recipeValidator; + + this.validationEnabled = validationEnabled; + this.shouldCheckTargeting = shouldCheckTargeting; + this.matches = 0; + + this.recipeMismatches = []; + this.invalidRecipes = []; + this.invalidBranches = new Map(); + this.invalidFeatures = new Map(); + this.validatorCache = {}; + this.missingLocale = []; + this.missingL10nIds = new Map(); + + this.locale = Services.locale.appLocaleAsBCP47; + } + + getResults() { + return { + recipeMismatches: this.recipeMismatches, + invalidRecipes: this.invalidRecipes, + invalidBranches: this.invalidBranches, + invalidFeatures: this.invalidFeatures, + missingLocale: this.missingLocale, + missingL10nIds: this.missingL10nIds, + locale: this.locale, + validationEnabled: this.validationEnabled, + }; + } + + async checkRecipe(recipe) { + if (recipe.appId !== "firefox-desktop") { + // Skip over recipes not intended for desktop. Experimenter publishes + // recipes into a collection per application (desktop goes to + // `nimbus-desktop-experiments`) but all preview experiments share the + // same collection (`nimbus-preview`). + // + // This is *not* the same as `lazy.APP_ID` which is used to + // distinguish between desktop Firefox and the desktop background + // updater. + return false; + } + + const validateFeatureSchemas = + this.validationEnabled && !recipe.featureValidationOptOut; + + if (this.validationEnabled) { + let validation = this.recipeValidator.validate(recipe); + if (!validation.valid) { + console.error( + `Could not validate experiment recipe ${recipe.id}: ${JSON.stringify( + validation.errors, + null, + 2 + )}` + ); + if (recipe.slug) { + this.invalidRecipes.push(recipe.slug); + } + return false; + } + } + + const featureIds = + recipe.featureIds ?? + recipe.branches + .flatMap(branch => branch.features ?? [branch.feature]) + .map(featureDef => featureDef.featureId); + + let haveAllFeatures = true; + + for (const featureId of featureIds) { + const feature = lazy.NimbusFeatures[featureId]; + + // If validation is enabled, we want to catch this later in + // _validateBranches to collect the correct stats for telemetry. + if (!feature) { + continue; + } + + if (!feature.applications.includes(lazy.APP_ID)) { + lazy.log.debug( + `${recipe.id} uses feature ${featureId} which is not enabled for this application (${lazy.APP_ID}) -- skipping` + ); + haveAllFeatures = false; + break; + } + } + + if (!haveAllFeatures) { + return false; + } + + if (this.shouldCheckTargeting) { + const match = await this.checkTargeting(recipe); + + if (match) { + const type = recipe.isRollout ? "rollout" : "experiment"; + lazy.log.debug(`[${type}] ${recipe.id} matched targeting`); + } else { + lazy.log.debug(`${recipe.id} did not match due to targeting`); + this.recipeMismatches.push(recipe.slug); + return false; + } + } + + this.matches++; + + if ( + typeof recipe.localizations === "object" && + recipe.localizations !== null + ) { + if ( + typeof recipe.localizations[this.locale] !== "object" || + recipe.localizations[this.locale] === null + ) { + this.missingLocale.push(recipe.slug); + lazy.log.debug( + `${recipe.id} is localized but missing locale ${this.locale}` + ); + return false; + } + } + + const result = await this._validateBranches(recipe, validateFeatureSchemas); + if (!result.valid) { + if (result.invalidBranchSlugs.length) { + this.invalidBranches.set(recipe.slug, result.invalidBranchSlugs); + } + if (result.invalidFeatureIds.length) { + this.invalidFeatures.set(recipe.slug, result.invalidFeatureIds); + } + if (result.missingL10nIds.length) { + this.missingL10nIds.set(recipe.slug, result.missingL10nIds); + } + lazy.log.debug(`${recipe.id} did not validate`); + return false; + } + + return true; + } + + async evaluateJexl(jexlString, customContext) { + if (customContext && !customContext.experiment) { + throw new Error( + "Expected an .experiment property in second param of this function" + ); + } + + if (!customContext.source) { + throw new Error( + "Expected a .source property that identifies which targeting expression is being evaluated." + ); + } + + const context = lazy.TargetingContext.combineContexts( + customContext, + this.experimentManager.createTargetingContext(), + lazy.ASRouterTargeting.Environment + ); + + lazy.log.debug("Testing targeting expression:", jexlString); + const targetingContext = new lazy.TargetingContext(context, { + source: customContext.source, + }); + + let result = null; + try { + result = await targetingContext.evalWithDefault(jexlString); + } catch (e) { + lazy.log.debug("Targeting failed because of an error", e); + console.error(e); + } + return result; + } + + /** + * Checks targeting of a recipe if it is defined + * @param {Recipe} recipe + * @param {{[key: string]: any}} customContext A custom filter context + * @returns {Promise<boolean>} Should we process the recipe? + */ + async checkTargeting(recipe) { + if (!recipe.targeting) { + lazy.log.debug("No targeting for recipe, so it matches automatically"); + return true; + } + + const result = await this.evaluateJexl(recipe.targeting, { + experiment: recipe, + source: recipe.slug, + }); + + return Boolean(result); + } + + /** + * Validate the branches of an experiment. + * + * @param {object} recipe The recipe object. + * @param {boolean} validateSchema Whether to validate the feature values + * using JSON schemas. + * + * @returns {object} The lists of invalid branch slugs and invalid feature + * IDs. + */ + async _validateBranches({ id, branches, localizations }, validateSchema) { + const invalidBranchSlugs = []; + const invalidFeatureIds = new Set(); + const missingL10nIds = new Set(); + + if (validateSchema || typeof localizations !== "undefined") { + for (const [branchIdx, branch] of branches.entries()) { + const features = branch.features ?? [branch.feature]; + for (const feature of features) { + const { featureId, value } = feature; + if (!lazy.NimbusFeatures[featureId]) { + console.error( + `Experiment ${id} has unknown featureId: ${featureId}` + ); + + invalidFeatureIds.add(featureId); + continue; + } + + let substitutedValue = value; + + if (localizations) { + // We already know that we have a localization table for this locale + // because we checked in `checkRecipe`. + try { + substitutedValue = + lazy._ExperimentFeature.substituteLocalizations( + value, + localizations[Services.locale.appLocaleAsBCP47], + missingL10nIds + ); + } catch (e) { + if (e?.reason === "l10n-missing-entry") { + // Skip validation because it *will* fail. + continue; + } + throw e; + } + } + + if (validateSchema) { + let validator; + if (this.validatorCache[featureId]) { + validator = this.validatorCache[featureId]; + } else if (lazy.NimbusFeatures[featureId].manifest.schema?.uri) { + const uri = lazy.NimbusFeatures[featureId].manifest.schema.uri; + try { + const schema = await fetch(uri, { + credentials: "omit", + }).then(rsp => rsp.json()); + + validator = this.validatorCache[featureId] = + new lazy.JsonSchema.Validator(schema); + } catch (e) { + throw new Error( + `Could not fetch schema for feature ${featureId} at "${uri}": ${e}` + ); + } + } else { + const schema = this._generateVariablesOnlySchema( + lazy.NimbusFeatures[featureId] + ); + validator = this.validatorCache[featureId] = + new lazy.JsonSchema.Validator(schema); + } + + const result = validator.validate(substitutedValue); + if (!result.valid) { + console.error( + `Experiment ${id} branch ${branchIdx} feature ${featureId} does not validate: ${JSON.stringify( + result.errors, + undefined, + 2 + )}` + ); + invalidBranchSlugs.push(branch.slug); + } + } + } + } + } + + return { + invalidBranchSlugs, + invalidFeatureIds: Array.from(invalidFeatureIds), + missingL10nIds: Array.from(missingL10nIds), + valid: + invalidBranchSlugs.length === 0 && + invalidFeatureIds.size === 0 && + missingL10nIds.size === 0, + }; + } + + _generateVariablesOnlySchema({ featureId, manifest }) { + // See-also: https://github.com/mozilla/experimenter/blob/main/app/experimenter/features/__init__.py#L21-L64 + const schema = { + $schema: "https://json-schema.org/draft/2019-09/schema", + title: featureId, + description: manifest.description, + type: "object", + properties: {}, + additionalProperties: true, + }; + + for (const [varName, desc] of Object.entries(manifest.variables)) { + const prop = {}; + switch (desc.type) { + case "boolean": + case "string": + prop.type = desc.type; + break; + + case "int": + prop.type = "integer"; + break; + + case "json": + // NB: Don't set a type of json fields, since they can be of any type. + break; + + default: + // NB: Experimenter doesn't outright reject invalid types either. + console.error( + `Feature ID ${featureId} has variable ${varName} with invalid FML type: ${prop.type}` + ); + break; + } + + if (prop.type === "string" && !!desc.enum) { + prop.enum = [...desc.enum]; + } + + schema.properties[varName] = prop; + } + + return schema; + } +} + +export const RemoteSettingsExperimentLoader = + new _RemoteSettingsExperimentLoader(); diff --git a/toolkit/components/nimbus/lib/SharedDataMap.sys.mjs b/toolkit/components/nimbus/lib/SharedDataMap.sys.mjs new file mode 100644 index 0000000000..f7d38f32ea --- /dev/null +++ b/toolkit/components/nimbus/lib/SharedDataMap.sys.mjs @@ -0,0 +1,179 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { EventEmitter } from "resource://gre/modules/EventEmitter.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", + PromiseUtils: "resource://gre/modules/PromiseUtils.sys.mjs", +}); + +const IS_MAIN_PROCESS = + Services.appinfo.processType === Services.appinfo.PROCESS_TYPE_DEFAULT; + +export class SharedDataMap extends EventEmitter { + constructor(sharedDataKey, options = { isParent: IS_MAIN_PROCESS }) { + super(); + + this._sharedDataKey = sharedDataKey; + this._isParent = options.isParent; + this._isReady = false; + this._readyDeferred = lazy.PromiseUtils.defer(); + this._data = null; + + if (this.isParent) { + // Lazy-load JSON file that backs Storage instances. + XPCOMUtils.defineLazyGetter(this, "_store", () => { + let path = options.path; + let store = null; + if (!path) { + try { + const profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile).path; + path = PathUtils.join(profileDir, `${sharedDataKey}.json`); + } catch (e) { + console.error(e); + } + } + try { + store = new lazy.JSONFile({ path }); + } catch (e) { + console.error(e); + } + return store; + }); + } else { + this._syncFromParent(); + Services.cpmm.sharedData.addEventListener("change", this); + } + } + + async init() { + if (!this._isReady && this.isParent) { + try { + await this._store.load(); + this._data = this._store.data; + this._syncToChildren({ flush: true }); + this._checkIfReady(); + } catch (e) { + console.error(e); + } + } + } + + get sharedDataKey() { + return this._sharedDataKey; + } + + get isParent() { + return this._isParent; + } + + ready() { + return this._readyDeferred.promise; + } + + get(key) { + if (!this._data) { + return null; + } + + let entry = this._data[key]; + + return entry; + } + + set(key, value) { + if (!this.isParent) { + throw new Error( + "Setting values from within a content process is not allowed" + ); + } + this._store.data[key] = value; + this._store.saveSoon(); + this._syncToChildren(); + this._notifyUpdate(); + } + + /** + * Replace the stored data with an updated filtered dataset for cleanup + * purposes. We don't notify of update because we're only filtering out + * old unused entries. + * + * @param {string[]} keysToRemove - list of keys to remove from the persistent store + */ + _removeEntriesByKeys(keysToRemove) { + if (!keysToRemove.length) { + return; + } + for (let key of keysToRemove) { + try { + delete this._store.data[key]; + } catch (e) { + // It's ok if this fails + } + } + this._store.saveSoon(); + } + + // Only used in tests + _deleteForTests(key) { + if (!this.isParent) { + throw new Error( + "Setting values from within a content process is not allowed" + ); + } + if (this.has(key)) { + delete this._store.data[key]; + this._store.saveSoon(); + this._syncToChildren(); + this._notifyUpdate(); + } + } + + has(key) { + return Boolean(this.get(key)); + } + + /** + * Notify store listeners of updates + * Called both from Main and Content process + */ + _notifyUpdate(process = "parent") { + for (let key of Object.keys(this._data || {})) { + this.emit(`${process}-store-update:${key}`, this._data[key]); + } + } + + _syncToChildren({ flush = false } = {}) { + Services.ppmm.sharedData.set(this.sharedDataKey, { + ...this._data, + }); + if (flush) { + Services.ppmm.sharedData.flush(); + } + } + + _syncFromParent() { + this._data = Services.cpmm.sharedData.get(this.sharedDataKey); + this._checkIfReady(); + this._notifyUpdate("child"); + } + + _checkIfReady() { + if (!this._isReady && this._data) { + this._isReady = true; + this._readyDeferred.resolve(); + } + } + + handleEvent(event) { + if (event.type === "change") { + if (event.changedKeys.includes(this.sharedDataKey)) { + this._syncFromParent(); + } + } + } +} diff --git a/toolkit/components/nimbus/metrics.yaml b/toolkit/components/nimbus/metrics.yaml new file mode 100644 index 0000000000..93b39fd6d4 --- /dev/null +++ b/toolkit/components/nimbus/metrics.yaml @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Adding a new metric? We have docs for that! +# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 +$tags: + - "Firefox :: Nimbus Desktop Client" + +nimbus_events: + enrollment: + type: event + description: > + Recorded when a user has met the conditions and is first bucketed into an + experiment (i.e. targeting matched and they were randomized into a bucket + and branch of the experiment). Expected a maximum of once per experiment + per user. + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + branch: + type: string + description: The branch slug/identifier that was randomly chosen + enrollment_id: + type: string + description: A unique identifier generated at enrollment time + experiment_type: + type: string + description: Indicates whether this is an experiemnt or rollout + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_sensitivity: + - technical + notification_emails: + - tlong@mozilla.com + - nimbus-team@mozilla.com + expires: never + send_in_pings: + - background-update + - events + enroll_failed: + type: event + description: > + Recorded when an enrollment fails, including the reason for the failure. + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + reason: + type: string + description: The reason for the enrollment failure + branch: + type: string + description: If reason == "invalid-branch", this is the invalid branch. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_sensitivity: + - technical + notification_emails: + - tlong@mozilla.com + - nimbus-team@mozilla.com + expires: never + send_in_pings: + - background-update + - events + unenrollment: + type: event + description: > + Recorded when either telemetry is disabled, or the experiment has run + for its designed duration (i.e. it is no longer present in the Nimbus + Remote Settings collection) + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + branch: + type: string + description: The branch slug/identifier that was randomly chosen + enrollment_id: + type: string + description: A unique identifier generated at enrollment time + reason: + type: string + description: The reason for the unenrollment + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_sensitivity: + - technical + notification_emails: + - tlong@mozilla.com + - nimbus-team@mozilla.com + expires: never + send_in_pings: + - background-update + - events + unenroll_failed: + type: event + description: > + Recorded when an unenrollment fails, including the reason for the failure. + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + reason: + type: string + description: The reason for the unenrollment failure + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_sensitivity: + - technical + notification_emails: + - tlong@mozilla.com + - nimbus-team@mozilla.com + expires: never + send_in_pings: + - background-update + - events + exposure: + type: event + description: > + Recorded when a user actually observes an experimental treatment, or + would have observed an experimental treatment if they had been in a + branch that would have shown one. + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + branch: + type: string + description: The branch slug/identifier that was randomly chosen + feature_id: + type: string + description: A unique identifier for the feature that was exposed + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1773563 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + data_sensitivity: + - technical + notification_emails: + - tlong@mozilla.com + - nimbus-team@mozilla.com + expires: never + send_in_pings: + - background-update + - events + validation_failed: + type: event + description: > + This records when validation of a recipe fails. + extra_keys: + experiment: + type: string + description: The slug/unique identifier of the experiment + reason: + type: string + description: > + Why validation failed (one of "invalid-recipe", "invalid-branch", + "invalid-reason", "missing-locale", or "missing-l10n-entry"). + branch: + type: string + description: > + If reason == invalid-branch, the branch that failed validation. + feature: + type: string + description: If reason == invalid-feature, the invalid feature ID. + locale: + type: string + description: > + If reason == missing-locale, the locale that was missing from the + localization table. + If reason == missing-l10n-entry, the locale that was missing the + localization entries. + l10n_ids: + type: string + description: > + If reason == missing-l10n-entry, a comma-sparated list of missing + localization entries. + + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1762652 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1821092 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1762652 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1781953 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1821092 + data_sensitivity: + - technical + notification_emails: + - barret@mozilla.com + expires: never + send_in_pings: + - background-update + - events diff --git a/toolkit/components/nimbus/moz.build b/toolkit/components/nimbus/moz.build new file mode 100644 index 0000000000..c0e31518dc --- /dev/null +++ b/toolkit/components/nimbus/moz.build @@ -0,0 +1,51 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Nimbus Desktop Client") + +EXPORTS.mozilla.browser += ["!lib/NimbusFeatureManifest.h", "lib/NimbusFeatures.h"] + +UNIFIED_SOURCES += [ + "lib/NimbusFeatures.cpp", +] + +# We expose NimbusFeatures to Android to make it easier to +# import everywhere but the tests will fail to run +if CONFIG["OS_TARGET"] != "Android": + TEST_DIRS += [ + "test/gtest", + ] + +BROWSER_CHROME_MANIFESTS += [ + "test/browser/browser.ini", +] + +XPCSHELL_TESTS_MANIFESTS += ["test/unit/xpcshell.ini"] + +SPHINX_TREES["docs"] = "docs" + +TESTING_JS_MODULES += [ + "test/NimbusTestUtils.sys.mjs", +] + +GeneratedFile( + "FeatureManifest.sys.mjs", + script="generate/generate_feature_manifest.py", + entry_point="generate_feature_manifest", + inputs=["./FeatureManifest.yaml"], +) + +GeneratedFile( + "lib/NimbusFeatureManifest.h", + script="generate/generate_feature_manifest.py", + entry_point="generate_platform_feature_manifest", + inputs=["./FeatureManifest.yaml"], +) + +FINAL_LIBRARY = "xul" + +JAR_MANIFESTS += ["jar.mn"] diff --git a/toolkit/components/nimbus/schemas/ExperimentFeatureManifest.schema.json b/toolkit/components/nimbus/schemas/ExperimentFeatureManifest.schema.json new file mode 100644 index 0000000000..664d0204a3 --- /dev/null +++ b/toolkit/components/nimbus/schemas/ExperimentFeatureManifest.schema.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "additionalProperties": false, + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "owner": { + "type": "string", + "description": "The owner of the feature." + }, + "applications": { + "description": "The applications that can enroll in experiments for this feature. Defaults to firefox-desktop if not present.", + "type": "array", + "items": { + "type": "string", + "enum": ["firefox-desktop", "firefox-desktop-background-task"] + }, + "minItems": 1 + }, + "hasExposure": { + "type": "boolean", + "description": "If the feature sends an exposure event." + }, + "exposureDescription": { + "type": "string", + "description": "A description of the implementation details of the exposure event, if one is sent." + }, + "isEarlyStartup": { + "type": "boolean", + "description": "If the feature values should be cached in prefs for fast early startup." + }, + "schema": { + "type": "object", + "description": "For features with large number of variables we instead point to a JSONSchema file instead of specifying them in the variables field", + "properties": { + "uri": { + "type": "string", + "description": "A resource:// URI that can be loaded at runtime from within Firefox.", + "format": "uri" + }, + "path": { + "type": "string", + "description": "The path to the schema file relative to the repository root" + } + }, + "required": ["uri", "path"] + }, + "variables": { + "additionalProperties": false, + "type": "object", + "patternProperties": { + "[a-zA-Z0-9_]+": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["json", "boolean", "int", "string"] + }, + "fallbackPref": { + "type": "string", + "description": "A pref that provides the default value for a feature when none is present" + }, + "setPref": { + "type": "string", + "description": "A pref that should be set to the value of this variable when enrolling in experiments." + }, + "enum": { + "description": "Validate feature value using a list of possible options (for string only values)." + }, + "description": { + "type": "string", + "description": "Explain how this value is being used" + } + }, + "required": ["type", "description"], + "additionalProperties": false, + "dependentSchemas": { + "fallbackPref": { + "description": "setPref is mutually exclusive with fallbackPref", + "properties": { + "setPref": { + "const": null + } + } + }, + "setPref": { + "description": "fallbackPref is mutually exclusive with setPref", + "properties": { + "fallbackPref": { + "const": null + } + } + } + } + } + } + } + }, + "required": ["description", "hasExposure"], + "if": { + "properties": { + "hasExposure": { + "const": true + } + } + }, + "then": { + "required": ["exposureDescription"] + } +} diff --git a/toolkit/components/nimbus/schemas/ExperimentFeatureRemote.schema.json b/toolkit/components/nimbus/schemas/ExperimentFeatureRemote.schema.json new file mode 100644 index 0000000000..a43d7d06af --- /dev/null +++ b/toolkit/components/nimbus/schemas/ExperimentFeatureRemote.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Feature", + "definitions": { + "RemoteFeatureConfigurations": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Feature name for which the defaults are provided." + }, + "configurations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Configuration identifier that will be included in Telemetry." + }, + "variables": { + "type": "object", + "description": "Key value pairs that should match the feature manifest definition.", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": ["enabled"] + }, + "targeting": { + "type": "string", + "description": "Target the configuration only to specific clients." + }, + "bucketConfig": { + "type": "object", + "properties": { + "randomizationUnit": { + "type": "string", + "description": "A unique, stable identifier for the user used as an input to bucket hashing" + }, + "namespace": { + "type": "string", + "description": "Additional inputs to the hashing function" + }, + "start": { + "type": "number", + "description": "Index of start of the range of buckets" + }, + "count": { + "type": "number", + "description": "Number of buckets to check" + }, + "total": { + "type": "number", + "description": "Total number of buckets", + "default": 10000 + } + }, + "required": [ + "randomizationUnit", + "namespace", + "start", + "count", + "total" + ], + "additionalProperties": false, + "description": "Bucketing configuration" + }, + "description": { + "type": "string", + "description": "Explanation for configuration and targeting" + } + }, + "required": ["variables", "targeting", "bucketConfig", "slug"], + "additionalProperties": false + } + } + }, + "required": ["id", "configurations"], + "additionalProperties": false + }, + "RemoteFeatureConfiguration": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Configuration identifier that will be included in Telemetry." + }, + "isEarlyStartup": { + "type": "boolean", + "description": "If the feature values should be cached in prefs for fast early startup." + }, + "variables": { + "type": "object", + "description": "Key value pairs that should match the feature manifest definition.", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": ["enabled"] + }, + "targeting": { + "type": "string", + "description": "Target the configuration only to specific clients." + }, + "bucketConfig": { + "type": "object", + "properties": { + "randomizationUnit": { + "type": "string", + "description": "A unique, stable identifier for the user used as an input to bucket hashing" + }, + "namespace": { + "type": "string", + "description": "Additional inputs to the hashing function" + }, + "start": { + "type": "number", + "description": "Index of start of the range of buckets" + }, + "count": { + "type": "number", + "description": "Number of buckets to check" + }, + "total": { + "type": "number", + "description": "Total number of buckets", + "default": 10000 + } + }, + "required": [ + "randomizationUnit", + "namespace", + "start", + "count", + "total" + ], + "additionalProperties": false, + "description": "Bucketing configuration" + }, + "description": { + "type": "string", + "description": "Explanation for configuration and targeting" + } + }, + "required": ["variables", "targeting", "slug"], + "additionalProperties": false + } + } +} diff --git a/toolkit/components/nimbus/schemas/LICENSE b/toolkit/components/nimbus/schemas/LICENSE new file mode 100644 index 0000000000..a612ad9813 --- /dev/null +++ b/toolkit/components/nimbus/schemas/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/toolkit/components/nimbus/schemas/NimbusEnrollment.schema.json b/toolkit/components/nimbus/schemas/NimbusEnrollment.schema.json new file mode 100644 index 0000000000..7f505102a0 --- /dev/null +++ b/toolkit/components/nimbus/schemas/NimbusEnrollment.schema.json @@ -0,0 +1,224 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/NimbusEnrollment", + "definitions": { + "NimbusEnrollment": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Unique identifier for the experiment" + }, + "userFacingName": { + "type": "string", + "description": "Public name of the experiment displayed on \"about:studies\"" + }, + "userFacingDescription": { + "type": "string", + "description": "Short public description of the experiment displayed on on \"about:studies\"" + }, + "isRollout": { + "type": "boolean", + "description": "When this property is set to true, treat this experiment as a rollout. Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments. See also: https://mozilla-hub.atlassian.net/browse/SDK-405" + }, + "featureIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of featureIds the experiment contains configurations for." + }, + "branch": { + "anyOf": [ + { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "feature": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": ["featureId", "value"], + "description": "A single feature configuration" + } + }, + "required": ["slug", "feature"] + }, + { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "feature": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "const": "unused-feature-id-for-legacy-support" + }, + "enabled": { + "type": "boolean", + "const": false + }, + "value": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["featureId", "enabled", "value"], + "description": "The feature key must be provided with valid values to prevent crashes if the DTO is encountered by Desktop clients earlier than version 95." + }, + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": ["featureId", "value"] + }, + "description": "An array of feature configurations" + } + }, + "required": ["slug", "feature", "features"] + }, + { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": ["featureId", "value"] + }, + "description": "An array of feature configurations" + } + }, + "required": ["slug", "features"] + } + ], + "description": "Branch configuration for the experiment" + }, + "experimentType": { + "type": "string", + "description": "What kind of experiment this enrollment corresponds to." + }, + "enrollmentId": { + "type": "string", + "description": "A unique identifier for the enrollment." + }, + "active": { + "type": "boolean", + "description": "Whether or not the enrollment is active." + }, + "lastSeen": { + "type": "string", + "format": "date-time", + "description": "The last time the experiment was seen." + }, + "force": { + "type": "boolean", + "description": "Whether or not this was a force enrollment." + }, + "prefs": { + "type": "array", + "description": "Information about prefs set by this enrollment.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the pref that was set." + }, + "featureId": { + "type": "string", + "description": "The ID of the feature that owns the variable that set this pref." + }, + "variable": { + "type": "string", + "description": "The variable that set this pref." + }, + "branch": { + "type": "string", + "enum": ["user", "default"], + "description": "The branch the pref was set on." + }, + "originalValue": { + "description": "The original value before the experiment." + } + }, + "additionalProperties": false + } + }, + "localizations": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + { + "type": "null" + } + ], + "description": "Per-locale localization substitutions.\n\nThe top level key is the locale (e.g., \"en-US\" or \"fr\"). Each entry is a mapping of string IDs to their localized equivalents.\n\nOnly supported on desktop." + }, + "unenrollReason": { + "type": "string", + "description": "The reason for unenrollment. Only present when the enrollment is inactive." + } + }, + "required": [ + "slug", + "userFacingName", + "userFacingDescription", + "branch", + "enrollmentId", + "active", + "lastSeen" + ], + "description": "An enrollment in a Nimbus Experiment saved to disk" + } + } +} diff --git a/toolkit/components/nimbus/schemas/NimbusExperiment.schema.json b/toolkit/components/nimbus/schemas/NimbusExperiment.schema.json new file mode 100644 index 0000000000..9fac063369 --- /dev/null +++ b/toolkit/components/nimbus/schemas/NimbusExperiment.schema.json @@ -0,0 +1,356 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/NimbusExperiment", + "definitions": { + "NimbusExperiment": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "string", + "description": "Version of the NimbusExperiment schema this experiment refers to" + }, + "slug": { + "type": "string", + "description": "Unique identifier for the experiment" + }, + "id": { + "type": "string", + "description": "Unique identifier for the experiment. This is a duplicate of slug, but is a required field for all Remote Settings records." + }, + "appName": { + "type": "string", + "description": "A slug identifying the targeted product for this experiment. It should be a lowercase_with_underscores name that is short and unambiguous and it should match the app_name found in https://probeinfo.telemetry.mozilla.org/glean/repositories. Examples are \"fenix\" or \"firefox_desktop\"." + }, + "appId": { + "type": "string", + "description": "The platform identifier for the targeted app. The app's identifier exactly as it appears in the relevant app store listing (for relevant platforms) or in the app's Glean initialization call (for other platforms). Examples are \"org.mozilla.firefox_beta\" or \"firefox-desktop\"." + }, + "channel": { + "type": "string", + "description": "A specific channel of an application such as \"nightly\", \"beta\", or \"release\"" + }, + "userFacingName": { + "type": "string", + "description": "Public name of the experiment displayed on \"about:studies\"" + }, + "userFacingDescription": { + "type": "string", + "description": "Short public description of the experiment displayed on on \"about:studies\"" + }, + "isEnrollmentPaused": { + "type": "boolean", + "description": "When this property is set to true, the the SDK should not enroll new users into the experiment that have not already been enrolled." + }, + "isRollout": { + "type": "boolean", + "description": "When this property is set to true, treat this experiment as a rollout. Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments. See also: https://mozilla-hub.atlassian.net/browse/SDK-405" + }, + "bucketConfig": { + "type": "object", + "properties": { + "randomizationUnit": { + "type": "string", + "description": "A unique, stable identifier for the user used as an input to bucket hashing" + }, + "namespace": { + "type": "string", + "description": "Additional inputs to the hashing function" + }, + "start": { + "type": "integer", + "description": "Index of start of the range of buckets" + }, + "count": { + "type": "integer", + "description": "Number of buckets to check" + }, + "total": { + "type": "integer", + "description": "Total number of buckets. You can assume this will always be 10000.", + "default": 10000 + } + }, + "required": [ + "randomizationUnit", + "namespace", + "start", + "count", + "total" + ], + "description": "Bucketing configuration" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the outcome" + }, + "priority": { + "type": "string", + "description": "e.g. \"primary\" or \"secondary\"" + } + }, + "required": [ + "slug", + "priority" + ] + }, + "description": "A list of outcomes relevant to the experiment analysis." + }, + "featureIds": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of featureIds the experiment contains configurations for." + }, + "branches": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "ratio": { + "type": "integer", + "description": "Relative ratio of population for the branch (e.g. if branch A=1 and branch B=3, branch A would get 25% of the population)", + "default": 1 + }, + "feature": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": [ + "featureId", + "value" + ], + "description": "A single feature configuration" + } + }, + "required": [ + "slug", + "ratio", + "feature" + ] + } + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "ratio": { + "type": "integer", + "description": "Relative ratio of population for the branch (e.g. if branch A=1 and branch B=3, branch A would get 25% of the population)", + "default": 1 + }, + "feature": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "const": "unused-feature-id-for-legacy-support" + }, + "enabled": { + "type": "boolean", + "const": false + }, + "value": { + "type": "object", + "additionalProperties": {} + } + }, + "required": [ + "featureId", + "enabled", + "value" + ], + "description": "The feature key must be provided with valid values to prevent crashes if the DTO is encountered by Desktop clients earlier than version 95." + }, + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": [ + "featureId", + "value" + ] + }, + "description": "An array of feature configurations" + } + }, + "required": [ + "slug", + "ratio", + "feature", + "features" + ] + } + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "slug": { + "type": "string", + "description": "Identifier for the branch" + }, + "ratio": { + "type": "integer", + "description": "Relative ratio of population for the branch (e.g. if branch A=1 and branch B=3, branch A would get 25% of the population)", + "default": 1 + }, + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "featureId": { + "type": "string", + "description": "The identifier for the feature flag" + }, + "value": { + "type": "object", + "additionalProperties": {}, + "description": "Optional extra params for the feature (this should be validated against a schema)" + } + }, + "required": [ + "featureId", + "value" + ] + }, + "description": "An array of feature configurations" + } + }, + "required": [ + "slug", + "ratio", + "features" + ] + } + } + ], + "description": "Branch configuration for the experiment" + }, + "targeting": { + "type": [ + "string", + "null" + ], + "description": "JEXL expression used to filter experiments based on locale, geo, etc." + }, + "startDate": { + "type": [ + "string", + "null" + ], + "description": "Actual publish date of the experiment Note that this value is expected to be null in Remote Settings.", + "format": "date" + }, + "enrollmentEndDate": { + "type": [ + "string", + "null" + ], + "description": "Actual enrollment end date of the experiment. Note that this value is expected to be null in Remote Settings.", + "format": "date" + }, + "endDate": { + "type": [ + "string", + "null" + ], + "description": "Actual end date of the experiment. Note that this value is expected to be null in Remote Settings.", + "format": "date" + }, + "proposedDuration": { + "type": "integer", + "description": "Duration of the experiment from the start date in days. Note that this property is only used during the analysis phase (not by the SDK)" + }, + "proposedEnrollment": { + "type": "integer", + "description": "This represents the number of days that we expect to enroll new users. Note that this property is only used during the analysis phase (not by the SDK)" + }, + "referenceBranch": { + "type": [ + "string", + "null" + ], + "description": "The slug of the reference branch (that is, which branch we consider \"control\")" + }, + "featureValidationOptOut": { + "type": "boolean", + "description": "Opt out of feature schema validation. Only supported on desktop." + }, + "localizations": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + { + "type": "null" + } + ], + "description": "Per-locale localization substitutions.\n\nThe top level key is the locale (e.g., \"en-US\" or \"fr\"). Each entry is a mapping of string IDs to their localized equivalents.\n\nOnly supported on desktop." + } + }, + "required": [ + "schemaVersion", + "slug", + "id", + "appName", + "appId", + "channel", + "userFacingName", + "userFacingDescription", + "isEnrollmentPaused", + "bucketConfig", + "branches", + "startDate", + "endDate", + "proposedEnrollment", + "referenceBranch" + ], + "description": "The experiment definition accessible to: 1. The Nimbus SDK via Remote Settings 2. Jetstream via the Experimenter API" + } + } +} diff --git a/toolkit/components/nimbus/schemas/moz.yaml b/toolkit/components/nimbus/schemas/moz.yaml new file mode 100644 index 0000000000..fb9dff6feb --- /dev/null +++ b/toolkit/components/nimbus/schemas/moz.yaml @@ -0,0 +1,31 @@ +schema: 1 + +bugzilla: + product: "Toolkit" + component: "Nimbus Desktop Client" + +origin: + name: "nimbus-shared" + description: "Shared data and schemas for Project Nimbus" + url: "https://github.com/mozilla/nimbus-shared" + license: "MPL-2.0" + release: "version 2.1.0" + revision: "v2.1.0" + +vendoring: + url: "https://github.com/mozilla/nimbus-shared" + source-hosting: "github" + tracking: "tag" + skip-vendoring-steps: + - "fetch" + - "update-moz-build" + + keep: + - "*.schema.json" + - "vendor.sh" + + update-actions: + - action: "run-script" + script: "vendor.sh" + cwd: "{yaml_dir}" + args: ["{revision}"] diff --git a/toolkit/components/nimbus/schemas/vendor.sh b/toolkit/components/nimbus/schemas/vendor.sh new file mode 100644 index 0000000000..de0f5b5ca6 --- /dev/null +++ b/toolkit/components/nimbus/schemas/vendor.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + + # Path to mach relative to toolkit/components/nimbus/schemas/ +MACH=$(realpath "../../../../mach") + +if [[ $(uname -a) == *MSYS* ]]; then + MACH="python ${MACH}" +fi + +NPM="${MACH} npm" + +# Strip the leading v from the tag to get the version number. +TAG="$1" +VERSION="${TAG:1}" +NAMESPACE="@mozilla/" +PACKAGE="nimbus-shared" +URL="https://registry.npmjs.org/${NAMESPACE}${PACKAGE}/-/${PACKAGE}-${VERSION}.tgz" + +mkdir -p tmp +cd tmp + +curl --proto '=https' --tlsv1.2 -sSf "${URL}" | tar -xzf - --strip-components 1 + +cp "schemas/experiments/NimbusExperiment.json" "../NimbusExperiment.schema.json" +cd .. + +# Ensure the generated file ends with a newline +sed -i -e '$a\' NimbusExperiment.schema.json + +rm -rf tmp diff --git a/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs new file mode 100644 index 0000000000..7be889ae06 --- /dev/null +++ b/toolkit/components/nimbus/test/NimbusTestUtils.sys.mjs @@ -0,0 +1,493 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { ExperimentStore } from "resource://nimbus/lib/ExperimentStore.sys.mjs"; +import { FileTestUtils } from "resource://testing-common/FileTestUtils.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + FeatureManifest: "resource://nimbus/FeatureManifest.sys.mjs", + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + NormandyUtils: "resource://normandy/lib/NormandyUtils.sys.mjs", + _ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + _RemoteSettingsExperimentLoader: + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + NetUtil: "resource://gre/modules/NetUtil.jsm", +}); + +function fetchSchemaSync(uri) { + // Yes, this is doing a sync load, but this is only done *once* and we cache + // the result after *and* it is test-only. + const channel = lazy.NetUtil.newChannel({ + uri, + loadUsingSystemPrincipal: true, + }); + const stream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance( + Ci.nsIScriptableInputStream + ); + + stream.init(channel.open()); + + const available = stream.available(); + const json = stream.read(available); + stream.close(); + + return JSON.parse(json); +} + +XPCOMUtils.defineLazyGetter(lazy, "enrollmentSchema", () => { + return fetchSchemaSync( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); +}); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; + +const PATH = FileTestUtils.getTempFile("shared-data-map").path; + +async function fetchSchema(url) { + const response = await fetch(url); + const schema = await response.json(); + if (!schema) { + throw new Error(`Failed to load ${url}`); + } + return schema; +} + +export const ExperimentTestUtils = { + _validateSchema(schema, value, errorMsg) { + const result = lazy.JsonSchema.validate(value, schema, { + shortCircuit: false, + }); + if (result.errors.length) { + throw new Error( + `${errorMsg}: ${JSON.stringify(result.errors, undefined, 2)}` + ); + } + return value; + }, + + _validateFeatureValueEnum({ branch }) { + let { features } = branch; + for (let feature of features) { + // If we're not using a real feature skip this check + if (!lazy.FeatureManifest[feature.featureId]) { + return true; + } + let { variables } = lazy.FeatureManifest[feature.featureId]; + for (let varName of Object.keys(variables)) { + let varValue = feature.value[varName]; + if ( + varValue && + variables[varName].enum && + !variables[varName].enum.includes(varValue) + ) { + throw new Error( + `${varName} should have one of the following values: ${JSON.stringify( + variables[varName].enum + )} but has value '${varValue}'` + ); + } + } + } + return true; + }, + + /** + * Checks if an experiment is valid acording to existing schema + */ + async validateExperiment(experiment) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusExperiment.schema.json" + ); + + // Ensure that the `featureIds` field is properly set + const { branches } = experiment; + branches.forEach(branch => { + branch.features.map(({ featureId }) => { + if (!experiment.featureIds.includes(featureId)) { + throw new Error( + `Branch(${branch.slug}) contains feature(${featureId}) but that's not declared in recipe(${experiment.slug}).featureIds` + ); + } + }); + }); + + return this._validateSchema( + schema, + experiment, + `Experiment ${experiment.slug} not valid` + ); + }, + validateEnrollment(enrollment) { + // We still have single feature experiment recipes for backwards + // compatibility testing but we don't do schema validation + if (!enrollment.branch.features && enrollment.branch.feature) { + return true; + } + + return ( + this._validateFeatureValueEnum(enrollment) && + this._validateSchema( + lazy.enrollmentSchema, + enrollment, + `Enrollment ${enrollment.slug} is not valid` + ) + ); + }, + async validateRollouts(rollout) { + const schema = await fetchSchema( + "resource://nimbus/schemas/NimbusEnrollment.schema.json" + ); + + return this._validateSchema( + schema, + rollout, + `Rollout configuration ${rollout.slug} is not valid` + ); + }, + /** + * Add features for tests. + * + * These features will only be visible to the JS Nimbus client. The native + * Nimbus client will have no access. + * + * @params features A list of |_NimbusFeature|s. + * + * @returns A cleanup function to remove the features once the test has completed. + */ + addTestFeatures(...features) { + for (const feature of features) { + if (Object.hasOwn(lazy.NimbusFeatures, feature.featureId)) { + throw new Error( + `Cannot add feature ${feature.featureId} -- a feature with this ID already exists!` + ); + } + lazy.NimbusFeatures[feature.featureId] = feature; + } + return () => { + for (const { featureId } of features) { + delete lazy.NimbusFeatures[featureId]; + } + }; + }, +}; + +export const ExperimentFakes = { + manager(store) { + let sandbox = lazy.sinon.createSandbox(); + let manager = new lazy._ExperimentManager({ store: store || this.store() }); + // We want calls to `store.addEnrollment` to implicitly validate the + // enrollment before saving to store + let origAddExperiment = manager.store.addEnrollment.bind(manager.store); + sandbox.stub(manager.store, "addEnrollment").callsFake(async enrollment => { + await ExperimentTestUtils.validateEnrollment(enrollment); + return origAddExperiment(enrollment); + }); + + return manager; + }, + store() { + return new ExperimentStore("FakeStore", { + path: PATH, + isParent: true, + }); + }, + waitForExperimentUpdate(ExperimentAPI, slug) { + return new Promise(resolve => + ExperimentAPI._store.once(`update:${slug}`, resolve) + ); + }, + async enrollWithRollout( + featureConfig, + { manager = lazy.ExperimentAPI._manager, source } = {} + ) { + await manager.store.init(); + const rollout = this.rollout(`${featureConfig.featureId}-rollout`, { + branch: { + slug: `${featureConfig.featureId}-rollout-branch`, + features: [featureConfig], + }, + }); + if (source) { + rollout.source = source; + } + await ExperimentTestUtils.validateRollouts(rollout); + // After storing the remote configuration to store and updating the feature + // we want to flush so that NimbusFeature usage in content process also + // receives the update + await manager.store.addEnrollment(rollout); + manager.store._syncToChildren({ flush: true }); + + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, enrollment) => { + if (enrollment.slug === rollout.slug && !enrollment.active) { + manager.store._deleteForTests(rollout.slug); + resolve(); + } + }) + ); + + return () => { + let promise = unenrollCompleted(rollout.slug); + manager.unenroll(rollout.slug, "cleanup"); + return promise; + }; + }, + /** + * Enroll in an experiment branch with the given feature configuration. + * + * NB: It is unnecessary to await the enrollmentPromise. + * See bug 1773583 and bug 1829412. + */ + async enrollWithFeatureConfig( + featureConfig, + { manager = lazy.ExperimentAPI._manager, isRollout = false } = {} + ) { + await manager.store.ready(); + // Use id passed in featureConfig value to compute experimentId + // This help filter telemetry events (such as expose) in race conditions when telemetry + // from multiple experiments with same featureId co-exist in snapshot + let experimentId = `${featureConfig.featureId}${ + featureConfig?.value?.id ? "-" + featureConfig?.value?.id : "" + }-experiment-${Math.random()}`; + + let recipe = this.recipe(experimentId, { + bucketConfig: { + namespace: "mstest-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + branches: [ + { + slug: "control", + ratio: 1, + features: [featureConfig], + }, + ], + isRollout, + }); + let { enrollmentPromise, doExperimentCleanup } = this.enrollmentHelper( + recipe, + { manager } + ); + + await enrollmentPromise; + + return doExperimentCleanup; + }, + /** + * Enroll in the given recipe. + * + * NB: It is unnecessary to await the enrollmentPromise. + * See bug 1773583 and bug 1829412. + */ + enrollmentHelper( + recipe, + { manager = lazy.ExperimentAPI._manager, source = "enrollmentHelper" } = {} + ) { + if (!recipe?.slug) { + throw new Error("Enrollment helper expects a recipe"); + } + + let enrollmentPromise = new Promise(resolve => + manager.store.on(`update:${recipe.slug}`, (event, experiment) => { + if (experiment.active) { + manager.store._syncToChildren({ flush: true }); + resolve(experiment); + } + }) + ); + let unenrollCompleted = slug => + new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(recipe.slug); + resolve(); + } + }) + ); + let doExperimentCleanup = async () => { + const experiment = manager.store.get(recipe.slug); + let promise = unenrollCompleted(experiment.slug); + manager.unenroll(experiment.slug, "cleanup"); + await promise; + }; + + if (!manager.store._isReady) { + throw new Error("Manager store not ready, call `manager.onStartup`"); + } + manager.enroll(recipe, source); + + return { enrollmentPromise, doExperimentCleanup }; + }, + async cleanupAll(slugs, { manager = lazy.ExperimentAPI._manager } = {}) { + function unenrollCompleted(slug) { + return new Promise(resolve => + manager.store.on(`update:${slug}`, (event, experiment) => { + if (!experiment.active) { + // Removes recipe from file storage which + // (normally the users archive of past experiments) + manager.store._deleteForTests(slug); + resolve(); + } + }) + ); + } + + for (const slug of slugs) { + let promise = unenrollCompleted(slug); + manager.unenroll(slug, "cleanup"); + await promise; + } + + if (manager.store.getAllActiveExperiments().length) { + throw new Error("Cleanup failed"); + } + }, + // Experiment store caches in prefs Enrollments for fast sync access + cleanupStorePrefCache() { + try { + Services.prefs.deleteBranch(SYNC_DATA_PREF_BRANCH); + Services.prefs.deleteBranch(SYNC_DEFAULTS_PREF_BRANCH); + } catch (e) { + // Expected if nothing is cached + } + }, + childStore() { + return new ExperimentStore("FakeStore", { isParent: false }); + }, + rsLoader() { + const loader = new lazy._RemoteSettingsExperimentLoader(); + // Replace RS client with a fake + Object.defineProperty(loader, "remoteSettingsClient", { + value: { get: () => Promise.resolve([]) }, + }); + // Replace xman with a fake + loader.manager = this.manager(); + + return loader; + }, + experiment(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "NimbusTestUtils", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: props?.branch?.features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, + rollout(slug, props = {}) { + return { + slug, + active: true, + enrollmentId: lazy.NormandyUtils.generateUuid(), + isRollout: true, + branch: { + slug: "treatment", + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + ...props, + }, + source: "NimbusTestUtils", + isEnrollmentPaused: true, + experimentType: "rollout", + userFacingName: "NimbusTestUtils", + userFacingDescription: "NimbusTestUtils", + lastSeen: new Date().toJSON(), + featureIds: (props?.branch?.features || props?.features)?.map( + f => f.featureId + ) || ["testFeature"], + ...props, + }; + }, + recipe(slug = lazy.NormandyUtils.generateUuid(), props = {}) { + return { + // This field is required for populating remote settings + id: lazy.NormandyUtils.generateUuid(), + schemaVersion: "1.7.0", + appName: "firefox_desktop", + appId: "firefox-desktop", + channel: "nightly", + slug, + isEnrollmentPaused: false, + probeSets: [], + startDate: null, + endDate: null, + proposedEnrollment: 7, + referenceBranch: "control", + application: "firefox-desktop", + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { testInt: 123, enabled: true }, + }, + ], + }, + ], + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 100, + total: 1000, + }, + userFacingName: "Nimbus recipe", + userFacingDescription: "NimbusTestUtils recipe", + featureIds: props?.branches?.[0].features?.map(f => f.featureId) || [ + "testFeature", + ], + ...props, + }; + }, +}; diff --git a/toolkit/components/nimbus/test/browser/browser.ini b/toolkit/components/nimbus/test/browser/browser.ini new file mode 100644 index 0000000000..a2845089ea --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser.ini @@ -0,0 +1,25 @@ +[DEFAULT] +support-files = + head.js +prefs = + # This turns off the update interval for fetching recipes from Remote Settings + app.normandy.run_interval_seconds=0 +skip-if = + toolkit == "android" + appname == "thunderbird" + os == "win" && os_version == "6.1" # Skip on Azure - frequent failure + +[browser_experiment_single_feature_enrollment.js] +[browser_prefs.js] +[browser_remotesettingsexperimentloader_remote_defaults.js] +[browser_remotesettingsexperimentloader_force_enrollment.js] +[browser_experimentstore_load.js] +[browser_experimentstore_load_single_feature.js] +skip-if = + os == "win" && os_version == "6.1" # Skip on Azure - frequent failure +[browser_remotesettings_experiment_enroll.js] +[browser_experiment_evaluate_jexl.js] +[browser_remotesettingsexperimentloader_init.js] +[browser_nimbus_telemetry.js] +tags = remote-settings + diff --git a/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js b/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js new file mode 100644 index 0000000000..50d83330a9 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experiment_evaluate_jexl.js @@ -0,0 +1,104 @@ +"use strict"; + +const { EnrollmentsContext, RemoteSettingsExperimentLoader } = + ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" + ); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +add_setup(async function setup() { + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["app.shield.optoutstudies.enabled", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + }); + + CONTEXT = new EnrollmentsContext(RemoteSettingsExperimentLoader.manager); +}); + +let CONTEXT; + +const FAKE_CONTEXT = { + experiment: ExperimentFakes.recipe("fake-test-experiment"), + source: "browser_experiment_evaluate_jexl", +}; + +add_task(async function test_throws_if_no_experiment_in_context() { + await Assert.rejects( + CONTEXT.evaluateJexl("true", { + customThing: 1, + source: "test_throws_if_no_experiment_in_context", + }), + /Expected an .experiment/, + "should throw if experiment is not passed to the custom context" + ); +}); + +add_task(async function test_evaluate_jexl() { + Assert.deepEqual( + await CONTEXT.evaluateJexl(`["hello"]`, FAKE_CONTEXT), + ["hello"], + "should return the evaluated result of a jexl expression" + ); +}); + +add_task(async function test_evaluate_custom_context() { + const result = await CONTEXT.evaluateJexl("experiment.slug", FAKE_CONTEXT); + Assert.equal( + result, + "fake-test-experiment", + "should have the custom .experiment context" + ); +}); + +add_task(async function test_evaluate_active_experiments_isFirstStartup() { + const result = await CONTEXT.evaluateJexl("isFirstStartup", FAKE_CONTEXT); + Assert.equal( + typeof result, + "boolean", + "should have a .isFirstStartup property from ExperimentManager " + ); +}); + +add_task(async function test_evaluate_active_experiments_activeExperiments() { + // Add an experiment to active experiments + const slug = "foo" + Math.random(); + // Init the store before we use it + await ExperimentManager.onStartup(); + + let recipe = ExperimentFakes.recipe(slug); + recipe.branches[0].slug = "mochitest-active-foo"; + delete recipe.branches[1]; + + let { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(recipe); + + await enrollmentPromise; + + Assert.equal( + await CONTEXT.evaluateJexl(`"${slug}" in activeExperiments`, FAKE_CONTEXT), + true, + "should find an active experiment" + ); + + Assert.equal( + await CONTEXT.evaluateJexl( + `"does-not-exist-fake" in activeExperiments`, + FAKE_CONTEXT + ), + false, + "should not find an experiment that doesn't exist" + ); + + await doExperimentCleanup(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js b/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js new file mode 100644 index 0000000000..b9a016d778 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experiment_single_feature_enrollment.js @@ -0,0 +1,128 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); + +const SINGLE_FEATURE_RECIPE = { + appId: "firefox-desktop", + appName: "firefox_desktop", + arguments: {}, + branches: [ + { + feature: { + featureId: "urlbar", + isEarlyStartup: true, + value: { + enabled: true, + quickSuggestEnabled: false, + quickSuggestNonSponsoredIndex: -1, + quickSuggestShouldShowOnboardingDialog: true, + quickSuggestShowOnboardingDialogAfterNRestarts: 2, + quickSuggestSponsoredIndex: -1, + }, + }, + ratio: 1, + slug: "control", + }, + { + feature: { + featureId: "urlbar", + isEarlyStartup: true, + value: { + enabled: true, + quickSuggestEnabled: true, + quickSuggestNonSponsoredIndex: -1, + quickSuggestShouldShowOnboardingDialog: false, + quickSuggestShowOnboardingDialogAfterNRestarts: 2, + quickSuggestSponsoredIndex: -1, + }, + }, + ratio: 1, + slug: "treatment", + }, + ], + bucketConfig: { + count: 10000, + namespace: "urlbar-9", + randomizationUnit: "normandy_id", + start: 0, + total: 10000, + }, + channel: "release", + endDate: null, + featureIds: ["urlbar"], + id: "firefox-suggest-history-vs-offline", + isEnrollmentPaused: false, + outcomes: [], + probeSets: [], + proposedDuration: 28, + proposedEnrollment: 7, + referenceBranch: "control", + schemaVersion: "1.5.0", + slug: "firefox-suggest-history-vs-offline", + startDate: "2021-07-21", + targeting: "true", + userFacingDescription: "Smarter suggestions in the AwesomeBar", + userFacingName: "Firefox Suggest - History vs Offline", +}; + +const SYNC_DATA_PREF_BRANCH = "nimbus.syncdatastore."; + +add_task(async function test_TODO() { + let { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(SINGLE_FEATURE_RECIPE); + let sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await enrollmentPromise; + + Assert.ok( + ExperimentAPI.getExperiment({ featureId: "urlbar" }), + "Should enroll in single feature experiment" + ); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}urlbar`), + "Should store early startup feature for sync access" + ); + Assert.equal( + Services.prefs.getIntPref( + `${SYNC_DATA_PREF_BRANCH}urlbar.quickSuggestSponsoredIndex` + ), + -1, + "Should store early startup variable for sync access" + ); + + Assert.equal( + NimbusFeatures.urlbar.getVariable( + "quickSuggestShowOnboardingDialogAfterNRestarts" + ), + 2, + "Should return value" + ); + + NimbusFeatures.urlbar.recordExposureEvent(); + + Assert.ok(stub.calledOnce, "Should be called once by urlbar"); + Assert.equal( + stub.firstCall.args[0].experimentSlug, + "firefox-suggest-history-vs-offline", + "Should have expected slug" + ); + Assert.equal( + stub.firstCall.args[0].featureId, + "urlbar", + "Should have expected featureId" + ); + + await doExperimentCleanup(); + sandbox.restore(); + NimbusFeatures.urlbar._didSendExposureEvent = false; +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js b/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js new file mode 100644 index 0000000000..a6f526e764 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experimentstore_load.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentStore } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentStore.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentFeatures } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +ChromeUtils.defineESModuleGetters(this, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", +}); + +function getPath() { + const profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile).path; + // NOTE: If this test is failing because you have updated this path in `ExperimentStore`, + // users will lose their old experiment data. You should do something to migrate that data. + return PathUtils.join(profileDir, "ExperimentStoreData.json"); +} + +// Ensure that data persisted to disk is succesfully loaded by the store. +// We write data to the expected location in the user profile and +// instantiate an ExperimentStore that should then see the value. +add_task(async function test_loadFromFile() { + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data.test = { + slug: "test", + active: true, + lastSeen: Date.now(), + }; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + await store.init(); + await store.ready(); + + Assert.equal( + previousSession.path, + store._store.path, + "Should have the same path" + ); + + Assert.ok( + store.get("test"), + "This should pass if the correct store path loaded successfully" + ); +}); + +add_task(async function test_load_from_disk_event() { + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + lastSeen: Date.now(), + }); + const stub = sinon.stub(); + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data.foo = experiment; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + store._onFeatureUpdate("green", stub); + + await store.init(); + await store.ready(); + + Assert.equal( + previousSession.path, + store._store.path, + "Should have the same path as previousSession." + ); + + await TestUtils.waitForCondition(() => stub.called, "Stub was called"); + + Assert.ok(stub.firstCall.args[1], "feature-experiment-loaded"); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js b/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js new file mode 100644 index 0000000000..7e9a19e21d --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_experimentstore_load_single_feature.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentStore } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentStore.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +ChromeUtils.defineESModuleGetters(this, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", +}); + +const SINGLE_FEATURE_RECIPE = { + ...ExperimentFakes.experiment(), + branch: { + feature: { + featureId: "urlbar", + value: { + valueThatWillDefinitelyShowUp: 42, + quickSuggestNonSponsoredIndex: 2021, + }, + }, + ratio: 1, + slug: "control", + }, + featureIds: ["urlbar"], + slug: "browser_experimentstore_load_single_feature", + userFacingDescription: "Smarter suggestions in the AwesomeBar", + userFacingName: "Firefox Suggest - History vs Offline", +}; + +function getPath() { + const profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile).path; + // NOTE: If this test is failing because you have updated this path in `ExperimentStore`, + // users will lose their old experiment data. You should do something to migrate that data. + return PathUtils.join(profileDir, "ExperimentStoreData.json"); +} + +add_task(async function test_load_from_disk_event() { + Services.prefs.setStringPref("messaging-system.log", "all"); + const stub = sinon.stub(); + const previousSession = new JSONFile({ path: getPath() }); + await previousSession.load(); + previousSession.data[SINGLE_FEATURE_RECIPE.slug] = SINGLE_FEATURE_RECIPE; + previousSession.saveSoon(); + await previousSession.finalize(); + + // Create a store and expect to load data from previous session + const store = new ExperimentStore(); + + let apiStoreStub = sinon.stub(ExperimentAPI, "_store").get(() => store); + + store._onFeatureUpdate("urlbar", stub); + + await store.init(); + await store.ready(); + + await TestUtils.waitForCondition(() => stub.called, "Stub was called"); + Assert.ok( + store.get(SINGLE_FEATURE_RECIPE.slug)?.slug, + "Experiment is loaded from disk" + ); + Assert.ok(stub.firstCall.args[1], "feature-experiment-loaded"); + Assert.equal( + NimbusFeatures.urlbar.getAllVariables().valueThatWillDefinitelyShowUp, + SINGLE_FEATURE_RECIPE.branch.feature.value.valueThatWillDefinitelyShowUp, + "Should match getAllVariables" + ); + Assert.equal( + NimbusFeatures.urlbar.getVariable("quickSuggestNonSponsoredIndex"), + SINGLE_FEATURE_RECIPE.branch.feature.value.quickSuggestNonSponsoredIndex, + "Should match getVariable" + ); + + registerCleanupFunction(async () => { + // Remove the experiment from disk + const fileStore = new JSONFile({ path: getPath() }); + await fileStore.load(); + fileStore.data = {}; + fileStore.saveSoon(); + await fileStore.finalize(); + apiStoreStub.restore(); + }); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js b/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js new file mode 100644 index 0000000000..c5bae5eff2 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_nimbus_telemetry.js @@ -0,0 +1,157 @@ +"use strict"; + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const TELEMETRY_CATEGORY = "normandy"; +const TELEMETRY_OBJECT = "nimbus_experiment"; +// Included with active experiment information +const EXPERIMENT_TYPE = "nimbus"; +const EVENT_FILTER = { category: TELEMETRY_CATEGORY }; + +add_setup(async function () { + let sandbox = sinon.createSandbox(); + // stub the `observe` method to make sure the Experiment Manager + // pref listener doesn't trigger and cause side effects + sandbox.stub(ExperimentManager, "observe"); + await SpecialPowers.pushPrefEnv({ + set: [["app.shield.optoutstudies.enabled", true]], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + sandbox.restore(); + }); +}); + +add_task(async function test_experiment_enroll_unenroll_Telemetry() { + Services.telemetry.clearEvents(); + const cleanup = await ExperimentFakes.enrollWithFeatureConfig({ + featureId: "test-feature", + value: { enabled: false }, + }); + let experiment = ExperimentAPI.getExperiment({ + featureId: "test-feature", + }); + + Assert.ok(experiment.branch, "Should be enrolled in the experiment"); + TelemetryTestUtils.assertEvents( + [ + { + method: "enroll", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + experimentType: EXPERIMENT_TYPE, + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "unenroll", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + reason: "cleanup", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + }, + ], + EVENT_FILTER + ); +}); + +add_task(async function test_experiment_expose_Telemetry() { + const featureManifest = { + description: "Test feature", + exposureDescription: "Used in tests", + }; + const cleanup = await ExperimentFakes.enrollWithFeatureConfig({ + featureId: "test-feature", + value: { enabled: false }, + }); + + let experiment = ExperimentAPI.getExperiment({ + featureId: "test-feature", + }); + + const { featureId } = experiment.branch.features[0]; + const feature = new ExperimentFeature(featureId, featureManifest); + + Services.telemetry.clearEvents(); + feature.recordExposureEvent(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "expose", + object: TELEMETRY_OBJECT, + value: experiment.slug, + extra: { + branchSlug: experiment.branch.slug, + featureId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); +}); + +add_task(async function test_rollout_expose_Telemetry() { + const featureManifest = { + description: "Test feature", + exposureDescription: "Used in tests", + }; + const cleanup = await ExperimentFakes.enrollWithRollout({ + featureId: "test-feature", + value: { enabled: false }, + }); + + let rollout = ExperimentAPI.getRolloutMetaData({ + featureId: "test-feature", + }); + + Assert.ok(rollout.slug, "Found enrolled experiment"); + + const feature = new ExperimentFeature("test-feature", featureManifest); + + Services.telemetry.clearEvents(); + feature.recordExposureEvent(); + + TelemetryTestUtils.assertEvents( + [ + { + method: "expose", + object: TELEMETRY_OBJECT, + value: rollout.slug, + extra: { + branchSlug: rollout.branch.slug, + featureId: feature.featureId, + }, + }, + ], + EVENT_FILTER + ); + + await cleanup(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_prefs.js b/toolkit/components/nimbus/test/browser/browser_prefs.js new file mode 100644 index 0000000000..6c38d16428 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_prefs.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); + +const EXPERIMENT_VALUE = "experiment-value"; +const ROLLOUT_VALUE = "rollout-value"; +const ROLLOUT = "rollout"; +const EXPERIMENT = "experiment"; + +const VALUES = { + [ROLLOUT]: ROLLOUT_VALUE, + [EXPERIMENT]: EXPERIMENT_VALUE, +}; + +add_task(async function test_prefs_priority() { + const pref = "nimbus.testing.testSetString"; + const featureId = "testFeature"; + + async function doTest({ settingEnrollments, expectedValue }) { + info( + `Enrolling in a rollout and experiment where the ${settingEnrollments.join( + " and " + )} set the same pref variable.` + ); + const enrollmentCleanup = []; + + for (const enrollmentKind of [ROLLOUT, EXPERIMENT]) { + const config = { + featureId, + value: {}, + }; + + if (settingEnrollments.includes(enrollmentKind)) { + config.value.testSetString = VALUES[enrollmentKind]; + } + + enrollmentCleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(config, { + isRollout: enrollmentKind === ROLLOUT, + }) + ); + } + + is( + NimbusFeatures[featureId].getVariable("testSetString"), + expectedValue, + "Expected the variable to match the expected value" + ); + + is( + Services.prefs.getStringPref(pref), + expectedValue, + "Expected the pref to match the expected value" + ); + + for (const cleanup of enrollmentCleanup) { + await cleanup(); + } + + Services.prefs.deleteBranch(pref); + } + + for (const settingEnrollments of [ + [ROLLOUT], + [EXPERIMENT], + [ROLLOUT, EXPERIMENT], + ]) { + const expectedValue = settingEnrollments.includes(EXPERIMENT) + ? EXPERIMENT_VALUE + : ROLLOUT_VALUE; + + await doTest({ settingEnrollments, expectedValue }); + } +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js b/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js new file mode 100644 index 0000000000..f4b1d27b4c --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettings_experiment_enroll.js @@ -0,0 +1,115 @@ +"use strict"; + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" +); +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +let rsClient; + +add_setup(async function () { + rsClient = RemoteSettings("nimbus-desktop-experiments"); + await rsClient.db.importChanges({}, Date.now(), [], { clear: true }); + + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["datareporting.healthreport.uploadEnabled", true], + ["app.shield.optoutstudies.enabled", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + await rsClient.db.clear(); + }); +}); + +add_task(async function test_experimentEnrollment() { + // Need to randomize the slug so subsequent test runs don't skip enrollment + // due to a conflicting slug + const recipe = ExperimentFakes.recipe("foo" + Math.random(), { + bucketConfig: { + start: 0, + // Make sure the experiment enrolls + count: 10000, + total: 10000, + namespace: "mochitest", + randomizationUnit: "normandy_id", + }, + }); + await rsClient.db.importChanges({}, Date.now(), [recipe], { + clear: true, + }); + + let waitForExperimentEnrollment = ExperimentFakes.waitForExperimentUpdate( + ExperimentAPI, + recipe.slug + ); + RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + + await waitForExperimentEnrollment; + + let experiment = ExperimentAPI.getExperiment({ + slug: recipe.slug, + }); + + Assert.ok(experiment.active, "Should be enrolled in the experiment"); + + let waitForExperimentUnenrollment = ExperimentFakes.waitForExperimentUpdate( + ExperimentAPI, + recipe.slug + ); + ExperimentManager.unenroll(recipe.slug, "mochitest-cleanup"); + + await waitForExperimentUnenrollment; + + experiment = ExperimentAPI.getExperiment({ + slug: recipe.slug, + }); + + Assert.ok(!experiment.active, "Experiment is no longer active"); + ExperimentAPI._store._deleteForTests(recipe.slug); +}); + +add_task(async function test_experimentEnrollment_startup() { + // Studies pref can turn the feature off but if the feature pref is off + // then it stays off. + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.rsexperimentloader.enabled", false], + ["app.shield.optoutstudies.enabled", false], + ], + }); + + Assert.ok(!RemoteSettingsExperimentLoader.enabled, "Should be disabled"); + + await SpecialPowers.pushPrefEnv({ + set: [["app.shield.optoutstudies.enabled", true]], + }); + + Assert.ok( + !RemoteSettingsExperimentLoader.enabled, + "Should still be disabled (feature pref is off)" + ); + + await SpecialPowers.pushPrefEnv({ + set: [["messaging-system.rsexperimentloader.enabled", true]], + }); + + Assert.ok( + RemoteSettingsExperimentLoader.enabled, + "Should finally be enabled" + ); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js new file mode 100644 index 0000000000..86031e600b --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_force_enrollment.js @@ -0,0 +1,250 @@ +//creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); + +async function setup(recipes) { + const client = RemoteSettings("nimbus-desktop-experiments"); + await client.db.importChanges({}, Date.now(), recipes, { + clear: true, + }); + + await BrowserTestUtils.waitForCondition( + async () => (await client.get()).length, + "RS is ready" + ); + + return { + client, + cleanup: () => client.db.clear(), + }; +} + +add_setup(async function () { + await SpecialPowers.pushPrefEnv({ + set: [ + ["messaging-system.log", "all"], + ["datareporting.healthreport.uploadEnabled", true], + ["app.shield.optoutstudies.enabled", true], + ["nimbus.debug", true], + ], + }); + + registerCleanupFunction(async () => { + await SpecialPowers.popPrefEnv(); + }); +}); + +add_task(async function test_fetch_recipe_and_branch_no_debug() { + const sandbox = sinon.createSandbox(); + Services.prefs.setBoolPref("nimbus.debug", false); + let stub = sandbox.stub(ExperimentManager, "forceEnroll"); + let recipes = [ExperimentFakes.recipe("slug123")]; + + const { cleanup } = await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "control", + }), + /Could not opt in/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + Services.prefs.setBoolPref("nimbus.debug", true); + + await RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "control", + }); + + Assert.ok(stub.called, "forceEnroll is called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_fetch_recipe_and_branch_badslug() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll"); + let recipes = [ExperimentFakes.recipe("slug123")]; + + const { cleanup } = await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "other_slug", + branch: "control", + }), + /Could not find experiment slug other_slug/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_fetch_recipe_and_branch_badbranch() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll"); + let recipes = [ExperimentFakes.recipe("slug123")]; + + const { cleanup } = await setup(recipes); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug123", + branch: "other_branch", + }), + /Could not find branch slug other_branch in slug123/, + "should throw an error" + ); + + Assert.ok(stub.notCalled, "forceEnroll is not called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_fetch_recipe_and_branch() { + const sandbox = sinon.createSandbox(); + let stub = sandbox.stub(ExperimentManager, "forceEnroll"); + let recipes = [ExperimentFakes.recipe("slug_fetch_recipe")]; + + const { cleanup } = await setup(recipes); + await RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "slug_fetch_recipe", + branch: "control", + }); + + Assert.ok(stub.called, "Called forceEnroll"); + Assert.deepEqual(stub.firstCall.args[0], recipes[0], "Called with recipe"); + Assert.deepEqual( + stub.firstCall.args[1], + recipes[0].branches[0], + "Called with branch" + ); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_invalid_recipe() { + const sandbox = sinon.createSandbox(); + const stub = sandbox.stub(ExperimentManager, "forceEnroll"); + const recipe = ExperimentFakes.recipe("invalid-recipe"); + delete recipe.branches; + + const { cleanup } = await setup([recipe]); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "invalid-recipe", + branch: "control", + }), + /failed validation/ + ); + + Assert.ok(stub.notCalled, "forceEnroll not called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_invalid_branch_variablesOnly() { + const sandbox = sinon.createSandbox(); + const stub = sandbox.stub(ExperimentManager, "forceEnroll"); + const recipe = ExperimentFakes.recipe("invalid-value"); + recipe.featureIds = ["testFeature"]; + recipe.branches = [recipe.branches[0]]; + recipe.branches[0].features[0].featureId = "testFeature"; + recipe.branches[0].features[0].value = { + enabled: "foo", + testInt: true, + testSetString: 123, + }; + + const { cleanup } = await setup([recipe]); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "invalid-value", + branch: "control", + }), + /failed validation/ + ); + + Assert.ok(stub.notCalled, "forceEnroll not called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_invalid_branch_schema() { + const sandbox = sinon.createSandbox(); + const stub = sandbox.stub(ExperimentManager, "forceEnroll"); + + const recipe = ExperimentFakes.recipe("invalid-value"); + recipe.featureIds = ["legacyHeartbeat"]; + recipe.branches = [recipe.branches[0]]; + recipe.branches[0].features[0].featureId = "legacyHeartbeat"; + recipe.branches[0].features[0].value = { + foo: "bar", + }; + + const { cleanup } = await setup([recipe]); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "invalid-value", + branch: "control", + }), + /failed validation/ + ); + + Assert.ok(stub.notCalled, "forceEnroll not called"); + + sandbox.restore(); + await cleanup(); +}); + +add_task(async function test_invalid_branch_featureId() { + const sandbox = sinon.createSandbox(); + const stub = sandbox.stub(ExperimentManager, "forceEnroll"); + const recipe = ExperimentFakes.recipe("invalid-value"); + recipe.featureIds = ["UNKNOWN"]; + recipe.branches = [recipe.branches[0]]; + recipe.branches[0].features[0].featureId = "UNKNOWN"; + + const { cleanup } = await setup([recipe]); + + await Assert.rejects( + RemoteSettingsExperimentLoader.optInToExperiment({ + slug: "invalid-value", + branch: "control", + }), + /failed validation/ + ); + + Assert.ok(stub.notCalled, "forceEnroll not called"); + + sandbox.restore(); + await cleanup(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js new file mode 100644 index 0000000000..f80fb7dfa5 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js @@ -0,0 +1,84 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); + +function getRecipe(slug) { + return ExperimentFakes.recipe(slug, { + bucketConfig: { + start: 0, + // Make sure the experiment enrolls + count: 10000, + total: 10000, + namespace: "mochitest", + randomizationUnit: "normandy_id", + }, + targeting: "!(experiment.slug in activeExperiments)", + }); +} + +add_task(async function test_double_feature_enrollment() { + let sandbox = sinon.createSandbox(); + let sendFailureTelemetryStub = sandbox.stub( + ExperimentManager, + "sendFailureTelemetry" + ); + await ExperimentAPI.ready(); + + Assert.ok( + ExperimentManager.store.getAllActiveExperiments().length === 0, + "Clean state" + ); + + let recipe1 = getRecipe("foo" + Math.random()); + let recipe2 = getRecipe("foo" + Math.random()); + + let enrollPromise1 = ExperimentFakes.waitForExperimentUpdate( + ExperimentAPI, + recipe1.slug + ); + + ExperimentManager.enroll(recipe1, "test_double_feature_enrollment"); + await enrollPromise1; + ExperimentManager.enroll(recipe2, "test_double_feature_enrollment"); + + Assert.equal( + ExperimentManager.store.getAllActiveExperiments().length, + 1, + "1 active experiment" + ); + + await BrowserTestUtils.waitForCondition( + () => sendFailureTelemetryStub.callCount === 1, + "Expected to fail one of the recipes" + ); + + Assert.equal( + sendFailureTelemetryStub.firstCall.args[0], + "enrollFailed", + "Check expected event" + ); + Assert.ok( + sendFailureTelemetryStub.firstCall.args[1] === recipe1.slug || + sendFailureTelemetryStub.firstCall.args[1] === recipe2.slug, + "Failed one of the two recipes" + ); + Assert.equal( + sendFailureTelemetryStub.firstCall.args[2], + "feature-conflict", + "Check expected reason" + ); + + await ExperimentFakes.cleanupAll([recipe1.slug]); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js new file mode 100644 index 0000000000..a500fa2c81 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_remote_defaults.js @@ -0,0 +1,584 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); +const { + _ExperimentFeature: ExperimentFeature, + + ExperimentAPI, +} = ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { ExperimentTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { RemoteSettingsExperimentLoader } = ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" +); + +const FOO_FAKE_FEATURE_MANIFEST = { + isEarlyStartup: true, + variables: { + remoteValue: { + type: "int", + }, + enabled: { + type: "boolean", + }, + }, +}; + +const BAR_FAKE_FEATURE_MANIFEST = { + isEarlyStartup: true, + variables: { + remoteValue: { + type: "int", + }, + enabled: { + type: "boolean", + }, + }, +}; + +const ENSURE_ENROLLMENT = { + targeting: "true", + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, +}; + +const REMOTE_CONFIGURATION_FOO = ExperimentFakes.recipe("foo-rollout", { + isRollout: true, + branches: [ + { + slug: "foo-rollout-branch", + ratio: 1, + features: [ + { + featureId: "foo", + isEarlyStartup: true, + value: { remoteValue: 42, enabled: true }, + }, + ], + }, + ], + ...ENSURE_ENROLLMENT, +}); +const REMOTE_CONFIGURATION_BAR = ExperimentFakes.recipe("bar-rollout", { + isRollout: true, + branches: [ + { + slug: "bar-rollout-branch", + ratio: 1, + features: [ + { + featureId: "bar", + isEarlyStartup: true, + value: { remoteValue: 3, enabled: true }, + }, + ], + }, + ], + ...ENSURE_ENROLLMENT, +}); + +const SYNC_DEFAULTS_PREF_BRANCH = "nimbus.syncdefaultsstore."; + +add_setup(function () { + const client = RemoteSettings("nimbus-desktop-experiments"); + sinon.stub(client, "get").resolves([]); + + registerCleanupFunction(() => client.get.restore()); +}); + +async function setup(configuration) { + const client = RemoteSettings("nimbus-desktop-experiments"); + client.get.resolves( + configuration ?? [REMOTE_CONFIGURATION_FOO, REMOTE_CONFIGURATION_BAR] + ); + + // Simulate a state where no experiment exists. + const cleanup = () => client.get.resolves([]); + return { client, cleanup }; +} + +add_task(async function test_remote_fetch_and_ready() { + const fooInstance = new ExperimentFeature("foo", FOO_FAKE_FEATURE_MANIFEST); + const barInstance = new ExperimentFeature("bar", BAR_FAKE_FEATURE_MANIFEST); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + fooInstance, + barInstance + ); + + const sandbox = sinon.createSandbox(); + const setExperimentActiveStub = sandbox.stub( + TelemetryEnvironment, + "setExperimentActive" + ); + const setExperimentInactiveStub = sandbox.stub( + TelemetryEnvironment, + "setExperimentInactive" + ); + + Assert.equal( + fooInstance.getVariable("remoteValue"), + undefined, + "This prop does not exist before we sync" + ); + + // Create to promises that get resolved when the features update + // with the remote setting rollouts + let fooUpdate = new Promise(resolve => fooInstance.onUpdate(resolve)); + let barUpdate = new Promise(resolve => barInstance.onUpdate(resolve)); + + await ExperimentAPI.ready(); + + let { cleanup } = await setup(); + + // Fake being initialized so we can update recipes + // we don't need to start any timers + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes( + "browser_rsel_remote_defaults" + ); + + // We need to await here because remote configurations are processed + // async to evaluate targeting + await Promise.all([fooUpdate, barUpdate]); + + Assert.equal( + fooInstance.getVariable("remoteValue"), + REMOTE_CONFIGURATION_FOO.branches[0].features[0].value.remoteValue, + "`foo` feature is set by remote defaults" + ); + Assert.equal( + barInstance.getVariable("remoteValue"), + REMOTE_CONFIGURATION_BAR.branches[0].features[0].value.remoteValue, + "`bar` feature is set by remote defaults" + ); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`), + "Pref cache is set" + ); + + // Check if we sent active experiment data for defaults + Assert.equal( + setExperimentActiveStub.callCount, + 2, + "setExperimentActive called once per feature" + ); + + Assert.ok( + setExperimentActiveStub.calledWith( + REMOTE_CONFIGURATION_FOO.slug, + REMOTE_CONFIGURATION_FOO.branches[0].slug, + { + type: "nimbus-rollout", + enrollmentId: sinon.match.string, + } + ), + "should call setExperimentActive with `foo` feature" + ); + Assert.ok( + setExperimentActiveStub.calledWith( + REMOTE_CONFIGURATION_BAR.slug, + REMOTE_CONFIGURATION_BAR.branches[0].slug, + { + type: "nimbus-rollout", + enrollmentId: sinon.match.string, + } + ), + "should call setExperimentActive with `bar` feature" + ); + + // Test Glean experiment API interaction + Assert.equal( + Services.fog.testGetExperimentData(REMOTE_CONFIGURATION_FOO.slug).branch, + REMOTE_CONFIGURATION_FOO.branches[0].slug, + "Glean.setExperimentActive called with `foo` feature" + ); + Assert.equal( + Services.fog.testGetExperimentData(REMOTE_CONFIGURATION_BAR.slug).branch, + REMOTE_CONFIGURATION_BAR.branches[0].slug, + "Glean.setExperimentActive called with `bar` feature" + ); + + Assert.equal(fooInstance.getVariable("remoteValue"), 42, "Has rollout value"); + Assert.equal(barInstance.getVariable("remoteValue"), 3, "Has rollout value"); + + // Clear RS db and load again. No configurations so should clear the cache. + await cleanup(); + await RemoteSettingsExperimentLoader.updateRecipes( + "browser_rsel_remote_defaults" + ); + + Assert.ok( + !fooInstance.getVariable("remoteValue"), + "foo-rollout should be removed" + ); + Assert.ok( + !barInstance.getVariable("remoteValue"), + "bar-rollout should be removed" + ); + + // Check if we sent active experiment data for defaults + Assert.equal( + setExperimentInactiveStub.callCount, + 2, + "setExperimentInactive called once per feature" + ); + + Assert.ok( + setExperimentInactiveStub.calledWith(REMOTE_CONFIGURATION_FOO.slug), + "should call setExperimentInactive with `foo` feature" + ); + Assert.ok( + setExperimentInactiveStub.calledWith(REMOTE_CONFIGURATION_BAR.slug), + "should call setExperimentInactive with `bar` feature" + ); + + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Should clear the pref" + ); + Assert.ok(!barInstance.getVariable("remoteValue"), "Should be missing"); + + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + ExperimentAPI._store._deleteForTests(REMOTE_CONFIGURATION_FOO.slug); + ExperimentAPI._store._deleteForTests(REMOTE_CONFIGURATION_BAR.slug); + sandbox.restore(); + + cleanupTestFeatures(); + await cleanup(); +}); + +add_task(async function test_remote_fetch_on_updateRecipes() { + let sandbox = sinon.createSandbox(); + let updateRecipesStub = sandbox.stub( + RemoteSettingsExperimentLoader, + "updateRecipes" + ); + // Work around the pref change callback that would trigger `setTimer` + sandbox.replaceGetter( + RemoteSettingsExperimentLoader, + "intervalInSeconds", + () => 1 + ); + + // This will un-register the timer + RemoteSettingsExperimentLoader._initialized = true; + RemoteSettingsExperimentLoader.uninit(); + Services.prefs.clearUserPref( + "app.update.lastUpdateTime.rs-experiment-loader-timer" + ); + + RemoteSettingsExperimentLoader.setTimer(); + + await BrowserTestUtils.waitForCondition( + () => updateRecipesStub.called, + "Wait for timer to call" + ); + + Assert.ok(updateRecipesStub.calledOnce, "Timer calls function"); + Assert.equal(updateRecipesStub.firstCall.args[0], "timer", "Called by timer"); + sandbox.restore(); + // This will un-register the timer + RemoteSettingsExperimentLoader._initialized = true; + RemoteSettingsExperimentLoader.uninit(); + Services.prefs.clearUserPref( + "app.update.lastUpdateTime.rs-experiment-loader-timer" + ); +}); + +add_task(async function test_finalizeRemoteConfigs_cleanup() { + const featureFoo = new ExperimentFeature("foo", { + description: "mochitests", + variables: { + foo: { type: "boolean" }, + }, + }); + const featureBar = new ExperimentFeature("bar", { + description: "mochitests", + variables: { + bar: { type: "boolean" }, + }, + }); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + featureFoo, + featureBar + ); + + let fooCleanup = await ExperimentFakes.enrollWithRollout( + { + featureId: "foo", + isEarlyStartup: true, + value: { foo: true }, + }, + { + source: "rs-loader", + } + ); + await ExperimentFakes.enrollWithRollout( + { + featureId: "bar", + isEarlyStartup: true, + value: { bar: true }, + }, + { + source: "rs-loader", + } + ); + let stubFoo = sinon.stub(); + let stubBar = sinon.stub(); + featureFoo.onUpdate(stubFoo); + featureBar.onUpdate(stubBar); + let cleanupPromise = new Promise(resolve => featureBar.onUpdate(resolve)); + + // stubFoo and stubBar will be called because the store is ready. We are not interested in these calls. + // Reset call history and check calls stats after cleanup. + Assert.ok( + stubFoo.called, + "feature foo update triggered becuase store is ready" + ); + Assert.ok( + stubBar.called, + "feature bar update triggered because store is ready" + ); + stubFoo.resetHistory(); + stubBar.resetHistory(); + + Services.prefs.setStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}foo`, + JSON.stringify({ foo: true, branch: { feature: { featureId: "foo" } } }) + ); + Services.prefs.setStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}bar`, + JSON.stringify({ bar: true, branch: { feature: { featureId: "bar" } } }) + ); + + const remoteConfiguration = { + ...REMOTE_CONFIGURATION_FOO, + branches: [ + { + ...REMOTE_CONFIGURATION_FOO.branches[0], + features: [ + { + ...REMOTE_CONFIGURATION_FOO.branches[0].features[0], + value: { + foo: true, + }, + }, + ], + }, + ], + }; + + const { cleanup } = await setup([remoteConfiguration]); + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes(); + await cleanupPromise; + + Assert.ok( + stubFoo.notCalled, + "Not called, not enrolling in rollout feature already exists" + ); + Assert.ok(stubBar.called, "Called because no recipe is seen, cleanup"); + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}foo`), + "Pref is not cleared" + ); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Pref was cleared" + ); + + await fooCleanup(); + // This will also remove the inactive recipe from the store + // the previous update (from recipe not seen code path) + // only sets the recipe as inactive + ExperimentAPI._store._deleteForTests("bar-rollout"); + ExperimentAPI._store._deleteForTests("foo-rollout"); + + cleanupTestFeatures(); + cleanup(); +}); + +// If the remote config data returned from the store is not modified +// this test should not throw +add_task(async function remote_defaults_no_mutation() { + let sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getRolloutForFeature").returns( + Cu.cloneInto( + { + featureIds: ["foo"], + branch: { + features: [{ featureId: "foo", value: { remoteStub: true } }], + }, + }, + {}, + { deepFreeze: true } + ) + ); + + let fooInstance = new ExperimentFeature("foo", FOO_FAKE_FEATURE_MANIFEST); + let config = fooInstance.getAllVariables(); + + Assert.ok(config.remoteStub, "Got back the expected value"); + + sandbox.restore(); +}); + +add_task(async function remote_defaults_active_remote_defaults() { + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + let barFeature = new ExperimentFeature("bar", { + description: "mochitest", + variables: { enabled: { type: "boolean" } }, + }); + let fooFeature = new ExperimentFeature("foo", { + description: "mochitest", + variables: { enabled: { type: "boolean" } }, + }); + + const cleanupTestFeatures = ExperimentTestUtils.addTestFeatures( + barFeature, + fooFeature + ); + + let rollout1 = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "bar-rollout-branch", + ratio: 1, + features: [ + { + featureId: "bar", + value: { enabled: true }, + }, + ], + }, + ], + isRollout: true, + ...ENSURE_ENROLLMENT, + targeting: "true", + }); + + let rollout2 = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "foo-rollout-branch", + ratio: 1, + features: [ + { + featureId: "foo", + value: { enabled: true }, + }, + ], + }, + ], + isRollout: true, + ...ENSURE_ENROLLMENT, + targeting: "'bar' in activeRollouts", + }); + + // Order is important, rollout2 won't match at first + const { cleanup } = await setup([rollout2, rollout1]); + let updatePromise = new Promise(resolve => barFeature.onUpdate(resolve)); + RemoteSettingsExperimentLoader._initialized = true; + await RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + + await updatePromise; + + Assert.ok(barFeature.getVariable("enabled"), "Enabled on first sync"); + Assert.ok(!fooFeature.getVariable("enabled"), "Targeting doesn't match"); + + let featureUpdate = new Promise(resolve => fooFeature.onUpdate(resolve)); + await RemoteSettingsExperimentLoader.updateRecipes("mochitest"); + await featureUpdate; + + Assert.ok(fooFeature.getVariable("enabled"), "Targeting should match"); + ExperimentAPI._store._deleteForTests("foo"); + ExperimentAPI._store._deleteForTests("bar"); + + cleanup(); + cleanupTestFeatures(); +}); + +add_task(async function remote_defaults_variables_storage() { + let barFeature = new ExperimentFeature("bar", { + description: "mochitest", + variables: { + enabled: { + type: "boolean", + }, + storage: { + type: "int", + }, + object: { + type: "json", + }, + string: { + type: "string", + }, + bool: { + type: "boolean", + }, + }, + }); + let rolloutValue = { + storage: 42, + object: { foo: "foo" }, + string: "string", + bool: true, + enabled: true, + }; + + let doCleanup = await ExperimentFakes.enrollWithRollout({ + featureId: "bar", + isEarlyStartup: true, + value: rolloutValue, + }); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar`, ""), + "Experiment stored in prefs" + ); + Assert.ok( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, 0), + "Stores variable in separate pref" + ); + Assert.equal( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, 0), + 42, + "Stores variable in correct type" + ); + Assert.deepEqual( + barFeature.getAllVariables(), + rolloutValue, + "Test types are returned correctly" + ); + + await doCleanup(); + + Assert.equal( + Services.prefs.getIntPref(`${SYNC_DEFAULTS_PREF_BRANCH}bar.storage`, -1), + -1, + "Variable pref is cleared" + ); + Assert.ok(!barFeature.getVariable("string"), "Variable is no longer defined"); + ExperimentAPI._store._deleteForTests("bar"); + ExperimentAPI._store._deleteForTests("bar-rollout"); +}); diff --git a/toolkit/components/nimbus/test/browser/head.js b/toolkit/components/nimbus/test/browser/head.js new file mode 100644 index 0000000000..f8a91df3d5 --- /dev/null +++ b/toolkit/components/nimbus/test/browser/head.js @@ -0,0 +1,38 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Globals +const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); + +ChromeUtils.defineESModuleGetters(this, { + ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + ExperimentTestUtils: "resource://testing-common/NimbusTestUtils.sys.mjs", + ExperimentFakes: "resource://testing-common/NimbusTestUtils.sys.mjs", +}); + +add_setup(function () { + let sandbox = sinon.createSandbox(); + + /* We stub the functions that operate with enrollments and remote rollouts + * so that any access to store something is implicitly validated against + * the schema and no records have missing (or extra) properties while in tests + */ + + let origAddExperiment = ExperimentManager.store.addEnrollment.bind( + ExperimentManager.store + ); + sandbox + .stub(ExperimentManager.store, "addEnrollment") + .callsFake(enrollment => { + ExperimentTestUtils.validateEnrollment(enrollment); + return origAddExperiment(enrollment); + }); + + registerCleanupFunction(() => { + sandbox.restore(); + }); +}); diff --git a/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp b/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp new file mode 100644 index 0000000000..0880ad3ae7 --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/NimbusFeatures_GetTest.cpp @@ -0,0 +1,187 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Preferences.h" +#include "mozilla/browser/NimbusFeatures.h" + +using namespace mozilla; + +static bool gPrefUpdate = false; + +TEST(NimbusFeaturesGet, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 42, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 42); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.foo.enabled", true, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, false)); + + ASSERT_EQ(Preferences::ClearUser("nimbus.syncdatastore.foo.value"), NS_OK); +} + +TEST(NimbusFeaturesGetRollout, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.rollout.value", 7, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("rollout"_ns, "value"_ns, 0), 7); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdefaultsstore.rollout.enabled", + true, PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("rollout"_ns, "enabled"_ns, false)); +} + +TEST(NimbusFeaturesExperimentPriorityOverRollouts, Errors) +{ + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.feature.value", 12, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.feature.value", 22, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("feature"_ns, "value"_ns, 0), 12); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.feature.enabled", true, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdefaultsstore.feature.enabled", + false, PrefValueKind::User), + NS_OK); + ASSERT_TRUE(NimbusFeatures::GetBool("feature"_ns, "enabled"_ns, false)); +} + +TEST(NimbusFeaturesDataSourcePrecedence, Errors) +{ + const auto FALLBACK_VALUE = 1; + const auto EXPERIMENT_VALUE = 2; + const auto ROLLOUT_VALUE = 3; + + ASSERT_EQ(Preferences::SetInt("nimbus.testing.testInt", FALLBACK_VALUE, + PrefValueKind::User), + NS_OK); + + // If there is no experiment or rollout, the fallback value should be + // returned. + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + FALLBACK_VALUE); + + // Enroll in an experiment. + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.testFeature.testInt", + EXPERIMENT_VALUE, PrefValueKind::User), + NS_OK); + + // Enroll in a rollout. + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.testFeature.testInt", + ROLLOUT_VALUE, PrefValueKind::User), + NS_OK); + + // Experiment value should take precedence. + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + EXPERIMENT_VALUE); + + // After experiments it should default to rollouts. + Preferences::ClearUser("nimbus.syncdatastore.testFeature.testInt"); + ASSERT_EQ(NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, 0), + ROLLOUT_VALUE); + + // Cleanup + Preferences::ClearUser("nimbus.syncdefaultsstore.testFeature.testInt"); + Preferences::ClearUser("nimbus.testing.testInt"); +} + +static void FooValueUpdated(const char* aPref, void* aUserData) { + ASSERT_STREQ(aPref, "nimbus.syncdatastore.foo.value"); + ASSERT_EQ(aUserData, reinterpret_cast<void*>(13)); + + ASSERT_FALSE(gPrefUpdate); + gPrefUpdate = true; + + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 24); +} + +static void BarRolloutValueUpdated(const char* aPref, void* aUserData) { + ASSERT_STREQ(aPref, "nimbus.syncdefaultsstore.bar.value"); + + ASSERT_FALSE(gPrefUpdate); + gPrefUpdate = true; +} + +TEST(NimbusFeaturesGetFallback, Errors) +{ + // No experiment is set and we expect to return fallback pref values + + // As defined by fallbackPref browser.aboutwelcome.enabled + // in FeatureManifest.yaml + Preferences::SetBool("browser.aboutwelcome.enabled", true, + PrefValueKind::Default); + ASSERT_EQ(NimbusFeatures::GetBool("aboutwelcome"_ns, "enabled"_ns, false), + true); + Preferences::SetBool("browser.aboutwelcome.enabled", false, + PrefValueKind::User); + ASSERT_EQ(NimbusFeatures::GetBool("aboutwelcome"_ns, "enabled"_ns, true), + false); + Preferences::ClearUser("browser.aboutwelcome.enabled"); + + const auto FALLBACK_VALUE = 5; + const auto DEFAULT_VALUE = 42; + + Preferences::SetInt("nimbus.testing.testInt", FALLBACK_VALUE, + PrefValueKind::Default); + ASSERT_EQ( + NimbusFeatures::GetInt("testFeature"_ns, "testInt"_ns, DEFAULT_VALUE), + FALLBACK_VALUE); + + Preferences::ClearUser("nimbus.testing.testInt"); +} + +TEST(NimbusFeaturesUpdate, Errors) +{ + // Verify updating foo.value calls FooValueUpdated. + ASSERT_EQ(NimbusFeatures::OnUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ( + NimbusFeatures::OnUpdate("bar"_ns, "value"_ns, BarRolloutValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 24, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(gPrefUpdate); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 24); + + // Verify updating foo.enabled doesn't call FooValueUpdated. + ASSERT_TRUE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, false)); + ASSERT_EQ(Preferences::SetBool("nimbus.syncdatastore.foo.enabled", false, + PrefValueKind::User), + NS_OK); + ASSERT_FALSE(NimbusFeatures::GetBool("foo"_ns, "enabled"_ns, true)); + gPrefUpdate = false; + + ASSERT_EQ(Preferences::SetInt("nimbus.syncdefaultsstore.bar.value", 25, + PrefValueKind::User), + NS_OK); + ASSERT_TRUE(gPrefUpdate); + gPrefUpdate = false; + + // Verify OffUpdate requires a matching user data pointer to unregister. + ASSERT_EQ(NimbusFeatures::OffUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(14)), + NS_ERROR_FAILURE); + + // Verify updating foo.value no longer calls FooValueUpdated after it has + // been unregistered. + ASSERT_EQ(NimbusFeatures::OffUpdate("foo"_ns, "value"_ns, FooValueUpdated, + reinterpret_cast<void*>(13)), + NS_OK); + ASSERT_EQ(Preferences::SetInt("nimbus.syncdatastore.foo.value", 25, + PrefValueKind::User), + NS_OK); + ASSERT_EQ(NimbusFeatures::GetInt("foo"_ns, "value"_ns, 0), 25); +} diff --git a/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp b/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp new file mode 100644 index 0000000000..7b0d1e0e7b --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/NimbusFeatures_RecordExposure.cpp @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Preferences.h" +#include "mozilla/browser/NimbusFeatures.h" +#include "js/Array.h" +#include "js/PropertyAndElement.h" +#include "js/TypeDecls.h" +#include "TelemetryFixture.h" +#include "TelemetryTestHelpers.h" + +using namespace mozilla; +using namespace TelemetryTestHelpers; + +class NimbusTelemetryFixture : public TelemetryTestFixture {}; + +TEST_F(NimbusTelemetryFixture, NimbusFeaturesTelemetry) { + constexpr auto prefName = "nimbus.syncdatastore.foo"_ns; + constexpr auto prefValue = + R"({"slug":"experiment-slug","branch":{"slug":"branch-slug"}})"; + AutoJSContextWithGlobal cx(mCleanGlobal); + Unused << mTelemetry->ClearEvents(); + + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns), NS_ERROR_UNEXPECTED) + << "Should fail because not enrolled in experiment"; + // Set the experiment info for `foo` + Preferences::SetCString(prefName.get(), prefValue); + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns), NS_OK) + << "Should work for the 2nd call"; + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("foo"_ns, true), NS_ERROR_ABORT) + << "Should abort because we've sent exposure and aOnce is true"; + ASSERT_EQ(NimbusFeatures::RecordExposureEvent("bar"_ns), NS_ERROR_UNEXPECTED) + << "Should fail because we don't have an experiment for bar"; + + JS::Rooted<JS::Value> eventsSnapshot(cx.GetJSContext()); + GetEventSnapshot(cx.GetJSContext(), &eventsSnapshot); + ASSERT_TRUE(EventPresent(cx.GetJSContext(), eventsSnapshot, "normandy"_ns, + "expose"_ns, "nimbus_experiment"_ns)); +} diff --git a/toolkit/components/nimbus/test/gtest/moz.build b/toolkit/components/nimbus/test/gtest/moz.build new file mode 100644 index 0000000000..41befbcd46 --- /dev/null +++ b/toolkit/components/nimbus/test/gtest/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "NimbusFeatures_GetTest.cpp", + "NimbusFeatures_RecordExposure.cpp", +] + +LOCAL_INCLUDES += [ + "/toolkit/components/telemetry/tests/gtest", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/toolkit/components/nimbus/test/unit/head.js b/toolkit/components/nimbus/test/unit/head.js new file mode 100644 index 0000000000..f5d6124ea5 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/head.js @@ -0,0 +1,86 @@ +"use strict"; +// Globals + +const { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +ChromeUtils.defineModuleGetter( + this, + "ObjectUtils", + "resource://gre/modules/ObjectUtils.jsm" +); +ChromeUtils.defineESModuleGetters(this, { + ExperimentFakes: "resource://testing-common/NimbusTestUtils.sys.mjs", + ExperimentTestUtils: "resource://testing-common/NimbusTestUtils.sys.mjs", +}); + +// Sinon does not support Set or Map in spy.calledWith() +function onFinalizeCalled(spyOrCallArgs, ...expectedArgs) { + function mapToObject(map) { + return Object.assign( + {}, + ...Array.from(map.entries()).map(([k, v]) => ({ [k]: v })) + ); + } + + function toPlainObjects(args) { + return [ + args[0], + { + ...args[1], + invalidBranches: mapToObject(args[1].invalidBranches), + invalidFeatures: mapToObject(args[1].invalidFeatures), + missingLocale: Array.from(args[1].missingLocale), + missingL10nIds: mapToObject(args[1].missingL10nIds), + }, + ]; + } + + const plainExpected = toPlainObjects(expectedArgs); + + if (Array.isArray(spyOrCallArgs)) { + return ObjectUtils.deepEqual(toPlainObjects(spyOrCallArgs), plainExpected); + } + + for (const args of spyOrCallArgs.args) { + if (ObjectUtils.deepEqual(toPlainObjects(args), plainExpected)) { + return true; + } + } + + return false; +} + +/** + * Assert the store has no active experiments or rollouts. + */ +async function assertEmptyStore(store, { cleanup = false } = {}) { + Assert.deepEqual( + store + .getAll() + .filter(e => e.active) + .map(e => e.slug), + [], + "Store should have no active enrollments" + ); + + Assert.deepEqual( + store + .getAll() + .filter(e => e.inactive) + .map(e => e.slug), + [], + "Store should have no inactive enrollments" + ); + + if (cleanup) { + // We need to call finalize first to ensure that any pending saves from + // JSONFile.saveSoon overwrite files on disk. + store._store.saveSoon(); + await store._store.finalize(); + await IOUtils.remove(store._store.path); + } +} diff --git a/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json b/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json new file mode 100644 index 0000000000..e7b8927248 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/reference_aboutwelcome_experiment_content.json @@ -0,0 +1,186 @@ +{ + "id": "msw-late-setdefault", + "template": "multistage", + "screens": [ + { + "id": "AW_GET_STARTED", + "order": 0, + "content": { + "zap": true, + "title": { + "string_id": "onboarding-multistage-welcome-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-welcome-subtitle" + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-welcome-primary-button-label" + }, + "action": { + "navigate": true + } + }, + "secondary_button": { + "text": { + "string_id": "onboarding-multistage-welcome-secondary-button-text" + }, + "label": { + "string_id": "onboarding-multistage-welcome-secondary-button-label" + }, + "position": "top", + "action": { + "type": "SHOW_FIREFOX_ACCOUNTS", + "addFlowParams": true, + "data": { + "entrypoint": "activity-stream-firstrun" + } + } + } + } + }, + { + "id": "AW_IMPORT_SETTINGS", + "order": 1, + "content": { + "zap": true, + "disclaimer": { + "string_id": "onboarding-import-sites-disclaimer" + }, + "title": { + "string_id": "onboarding-multistage-import-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-import-subtitle" + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-import-primary-button-label" + }, + "action": { + "type": "SHOW_MIGRATION_WIZARD", + "navigate": true + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-import-secondary-button-label" + }, + "action": { + "navigate": true + } + } + } + }, + { + "id": "AW_CHOOSE_THEME", + "order": 2, + "content": { + "zap": true, + "title": { + "string_id": "onboarding-multistage-theme-header" + }, + "subtitle": { + "string_id": "onboarding-multistage-theme-subtitle" + }, + "tiles": { + "type": "theme", + "action": { + "theme": "<event>" + }, + "data": [ + { + "theme": "automatic", + "label": { + "string_id": "onboarding-multistage-theme-label-automatic" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-automatic-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-automatic-2" + } + }, + { + "theme": "light", + "label": { + "string_id": "onboarding-multistage-theme-label-light" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-light-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-light" + } + }, + { + "theme": "dark", + "label": { + "string_id": "onboarding-multistage-theme-label-dark" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-dark-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-dark" + } + }, + { + "theme": "alpenglow", + "label": { + "string_id": "onboarding-multistage-theme-label-alpenglow" + }, + "tooltip": { + "string_id": "onboarding-multistage-theme-tooltip-alpenglow-2" + }, + "description": { + "string_id": "onboarding-multistage-theme-description-alpenglow" + } + } + ] + }, + "primary_button": { + "label": { + "string_id": "onboarding-multistage-theme-primary-button-label" + }, + "action": { + "navigate": true + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-theme-secondary-button-label" + }, + "action": { + "theme": "automatic", + "navigate": true + } + } + } + }, + { + "id": "AW_SET_DEFAULT", + "order": 3, + "content": { + "zap": true, + "title": "Make Firefox your default browser", + "subtitle": "Speed, safety, and privacy every time you browse.", + "primary_button": { + "label": "Make Default", + "action": { + "navigate": true, + "type": "SET_DEFAULT_BROWSER" + } + }, + "secondary_button": { + "label": { + "string_id": "onboarding-multistage-import-secondary-button-label" + }, + "action": { + "navigate": true + } + } + } + } + ] +} diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js new file mode 100644 index 0000000000..722011f5b8 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI.js @@ -0,0 +1,588 @@ +"use strict"; + +const { ExperimentAPI } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); +const COLLECTION_ID_PREF = "messaging-system.rsexperimentloader.collection_id"; + +/** + * #getExperiment + */ +add_task(async function test_getExperiment_fromChild_slug() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => ExperimentFakes.childStore()); + + await manager.store.addEnrollment(expected); + + // Wait to sync to child + await TestUtils.waitForCondition( + () => ExperimentAPI.getExperiment({ slug: "foo" }), + "Wait for child to sync" + ); + + Assert.equal( + ExperimentAPI.getExperiment({ slug: "foo" }).slug, + expected.slug, + "should return an experiment by slug" + ); + + Assert.deepEqual( + ExperimentAPI.getExperiment({ slug: "foo" }).branch, + expected.branch, + "should return the right branch by slug" + ); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_fromParent_slug() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + Assert.equal( + ExperimentAPI.getExperiment({ slug: "foo" }).slug, + expected.slug, + "should return an experiment by slug" + ); + + sandbox.restore(); +}); + +add_task(async function test_getExperimentMetaData() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo"); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + let metadata = ExperimentAPI.getExperimentMetaData({ slug: expected.slug }); + + Assert.equal( + Object.keys(metadata.branch).length, + 1, + "Should only expose one property" + ); + Assert.equal( + metadata.branch.slug, + expected.branch.slug, + "Should have the slug prop" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this method"); + + sandbox.restore(); +}); + +add_task(async function test_getRolloutMetaData() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.rollout("foo"); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.onStartup(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + await ExperimentAPI.ready(); + + await manager.store.addEnrollment(expected); + + let metadata = ExperimentAPI.getExperimentMetaData({ slug: expected.slug }); + + Assert.equal( + Object.keys(metadata.branch).length, + 1, + "Should only expose one property" + ); + Assert.equal( + metadata.branch.slug, + expected.branch.slug, + "Should have the slug prop" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this method"); + + sandbox.restore(); +}); + +add_task(function test_getExperimentMetaData_safe() { + const sandbox = sinon.createSandbox(); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + sandbox.stub(ExperimentAPI._store, "get").throws(); + sandbox.stub(ExperimentAPI._store, "getExperimentForFeature").throws(); + + try { + let metadata = ExperimentAPI.getExperimentMetaData({ slug: "foo" }); + Assert.equal(metadata, null, "Should not throw"); + } catch (e) { + Assert.ok(false, "Error should be caught in ExperimentAPI"); + } + + Assert.ok(ExperimentAPI._store.get.calledOnce, "Sanity check"); + + try { + let metadata = ExperimentAPI.getExperimentMetaData({ featureId: "foo" }); + Assert.equal(metadata, null, "Should not throw"); + } catch (e) { + Assert.ok(false, "Error should be caught in ExperimentAPI"); + } + + Assert.ok( + ExperimentAPI._store.getExperimentForFeature.calledOnce, + "Sanity check" + ); + + Assert.ok(exposureStub.notCalled, "Not called for this feature"); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_feature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + features: [{ featureId: "cfr", value: null }], + feature: { + featureId: "unused-feature-id-for-legacy-support", + enabled: false, + value: {}, + }, + }, + }); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => ExperimentFakes.childStore()); + let exposureStub = sandbox.stub(ExperimentAPI, "recordExposureEvent"); + + await manager.store.addEnrollment(expected); + + // Wait to sync to child + await TestUtils.waitForCondition( + () => ExperimentAPI.getExperiment({ featureId: "cfr" }), + "Wait for child to sync" + ); + + Assert.equal( + ExperimentAPI.getExperiment({ featureId: "cfr" }).slug, + expected.slug, + "should return an experiment by featureId" + ); + + Assert.deepEqual( + ExperimentAPI.getExperiment({ featureId: "cfr" }).branch, + expected.branch, + "should return the right branch by featureId" + ); + + Assert.ok(exposureStub.notCalled, "Not called by default"); + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_safe() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getExperimentForFeature").throws(); + + try { + Assert.equal( + ExperimentAPI.getExperiment({ featureId: "foo" }), + null, + "It should not fail even when it throws." + ); + } catch (e) { + Assert.ok(false, "Error should be caught by ExperimentAPI"); + } + + sandbox.restore(); +}); + +add_task(async function test_getExperiment_featureAccess() { + const sandbox = sinon.createSandbox(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + value: { title: "hi" }, + features: [{ featureId: "cfr", value: { message: "content" } }], + }, + }); + const stub = sandbox + .stub(ExperimentAPI._store, "getExperimentForFeature") + .returns(expected); + + let { branch } = ExperimentAPI.getExperiment({ featureId: "cfr" }); + + Assert.equal(branch.slug, "treatment"); + let feature = branch.cfr; + Assert.ok(feature, "Should allow to access by featureId"); + Assert.equal(feature.value.message, "content"); + + stub.restore(); +}); + +add_task(async function test_getExperiment_featureAccess_backwardsCompat() { + const sandbox = sinon.createSandbox(); + const expected = ExperimentFakes.experiment("foo", { + branch: { + slug: "treatment", + feature: { featureId: "cfr", value: { message: "content" } }, + }, + }); + const stub = sandbox + .stub(ExperimentAPI._store, "getExperimentForFeature") + .returns(expected); + + let { branch } = ExperimentAPI.getExperiment({ featureId: "cfr" }); + + Assert.equal(branch.slug, "treatment"); + let feature = branch.cfr; + Assert.ok(feature, "Should allow to access by featureId"); + Assert.equal(feature.value.message, "content"); + + stub.restore(); +}); + +/** + * #getRecipe + */ +add_task(async function test_getRecipe() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + const collectionName = Services.prefs.getStringPref(COLLECTION_ID_PREF); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const recipe = await ExperimentAPI.getRecipe("foo"); + Assert.deepEqual( + recipe, + RECIPE, + "should return an experiment recipe if found" + ); + Assert.equal( + ExperimentAPI._remoteSettingsClient.collectionName, + collectionName, + "Loaded the expected collection" + ); + + sandbox.restore(); +}); + +add_task(async function test_getRecipe_Failure() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").throws(); + + const recipe = await ExperimentAPI.getRecipe("foo"); + Assert.equal(recipe, undefined, "should return undefined if RS throws"); + + sandbox.restore(); +}); + +/** + * #getAllBranches + */ +add_task(async function test_getAllBranches() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + + sandbox.restore(); +}); + +// API used by Messaging System +add_task(async function test_getAllBranches_featureIdAccessor() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + branches.forEach(branch => { + Assert.equal( + branch.testFeature.featureId, + "testFeature", + "Should use the experimentBranchAccessor proxy getter" + ); + }); + + sandbox.restore(); +}); + +// For schema version before 1.6.2 branch.feature was accessed +// instead of branch.features +add_task(async function test_getAllBranches_backwardsCompat() { + const sandbox = sinon.createSandbox(); + const RECIPE = ExperimentFakes.recipe("foo"); + delete RECIPE.branches[0].features; + delete RECIPE.branches[1].features; + let feature = { + featureId: "backwardsCompat", + value: { + enabled: true, + }, + }; + RECIPE.branches[0].feature = feature; + RECIPE.branches[1].feature = feature; + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").resolves([RECIPE]); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.deepEqual( + branches, + RECIPE.branches, + "should return all branches if found a recipe" + ); + branches.forEach(branch => { + Assert.equal( + branch.backwardsCompat.featureId, + "backwardsCompat", + "Should use the experimentBranchAccessor proxy getter" + ); + }); + + sandbox.restore(); +}); + +add_task(async function test_getAllBranches_Failure() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._remoteSettingsClient, "get").throws(); + + const branches = await ExperimentAPI.getAllBranches("foo"); + Assert.equal(branches, undefined, "should return undefined if RS throws"); + + sandbox.restore(); +}); + +/** + * Store events + */ +add_task(async function test_addEnrollment_eventEmit_add() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + store.on("update:foo", slugStub); + store.on("featureUpdate:purple", featureStub); + + await store.addEnrollment(experiment); + + Assert.equal( + slugStub.callCount, + 1, + "should call 'update' callback for slug when experiment is added" + ); + Assert.equal(slugStub.firstCall.args[1].slug, experiment.slug); + Assert.equal( + featureStub.callCount, + 1, + "should call 'featureUpdate' callback for featureId when an experiment is added" + ); + Assert.equal(featureStub.firstCall.args[0], "featureUpdate:purple"); + Assert.equal(featureStub.firstCall.args[1], "experiment-updated"); + + store.off("update:foo", slugStub); + store.off("featureUpdate:purple", featureStub); + sandbox.restore(); +}); + +add_task(async function test_updateExperiment_eventEmit_add_and_update() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + await store.addEnrollment(experiment); + + store.on("update:foo", slugStub); + store._onFeatureUpdate("purple", featureStub); + + store.updateExperiment(experiment.slug, experiment); + + await TestUtils.waitForCondition( + () => featureStub.callCount == 2, + "Wait for `on` method to notify callback about the `add` event." + ); + // Called twice, once when attaching the event listener (because there is an + // existing experiment with that name) and 2nd time for the update event + Assert.equal(slugStub.firstCall.args[1].slug, experiment.slug); + Assert.equal(featureStub.callCount, 2, "Called twice for feature"); + Assert.equal(featureStub.firstCall.args[0], "featureUpdate:purple"); + Assert.equal(featureStub.firstCall.args[1], "experiment-updated"); + + store.off("update:foo", slugStub); + store._offFeatureUpdate("featureUpdate:purple", featureStub); +}); + +add_task(async function test_updateExperiment_eventEmit_off() { + const sandbox = sinon.createSandbox(); + const slugStub = sandbox.stub(); + const featureStub = sandbox.stub(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple", value: null }], + }, + }); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + + await store.init(); + await ExperimentAPI.ready(); + + store.on("update:foo", slugStub); + store.on("featureUpdate:purple", featureStub); + + await store.addEnrollment(experiment); + + store.off("update:foo", slugStub); + store.off("featureUpdate:purple", featureStub); + + store.updateExperiment(experiment.slug, experiment); + + Assert.equal(slugStub.callCount, 1, "Called only once before `off`"); + Assert.equal(featureStub.callCount, 1, "Called only once before `off`"); + + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green", value: null }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + + Assert.deepEqual( + ExperimentAPI.getActiveBranch({ featureId: "green" }), + experiment.branch, + "Should return feature of active experiment" + ); + + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_safe() { + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI._store, "getAllActiveExperiments").throws(); + + try { + Assert.equal( + ExperimentAPI.getActiveBranch({ featureId: "green" }), + null, + "Should not throw" + ); + } catch (e) { + Assert.ok(false, "Should catch error in ExperimentAPI"); + } + + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_storeFailure() { + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + // Adding stub later because `addEnrollment` emits update events + const stub = sandbox.stub(store, "emit"); + // Call getActiveBranch to trigger an activation event + sandbox.stub(store, "getAllActiveExperiments").throws(); + try { + ExperimentAPI.getActiveBranch({ featureId: "green" }); + } catch (e) { + /* This is expected */ + } + + Assert.equal(stub.callCount, 0, "Not called if store somehow fails"); + sandbox.restore(); +}); + +add_task(async function test_getActiveBranch_noActivationEvent() { + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + sandbox.stub(ExperimentAPI, "_store").get(() => store); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "green" }], + }, + }); + + await store.init(); + await store.addEnrollment(experiment); + // Adding stub later because `addEnrollment` emits update events + const stub = sandbox.stub(store, "emit"); + // Call getActiveBranch to trigger an activation event + ExperimentAPI.getActiveBranch({ featureId: "green" }); + + Assert.equal(stub.callCount, 0, "Not called: sendExposureEvent is false"); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js new file mode 100644 index 0000000000..e4ce12caaa --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature.js @@ -0,0 +1,324 @@ +"use strict"; + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +function setDefaultBranch(pref, value) { + let branch = Services.prefs.getDefaultBranch(""); + branch.setStringPref(pref, value); +} + +const TEST_FALLBACK_PREF = "testprefbranch.config"; +const FAKE_FEATURE_MANIFEST = { + description: "Test feature", + exposureDescription: "Used in tests", + variables: { + enabled: { + type: "boolean", + fallbackPref: "testprefbranch.enabled", + }, + config: { + type: "json", + fallbackPref: TEST_FALLBACK_PREF, + }, + remoteValue: { + type: "boolean", + }, + test: { + type: "boolean", + }, + title: { + type: "string", + }, + }, +}; + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +add_task(async function test_ExperimentFeature_test_helper_ready() { + const { manager } = await setupForExperimentFeature(); + await manager.store.ready(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + + await ExperimentFakes.enrollWithRollout( + { + featureId: "foo", + value: { remoteValue: "mochitest", enabled: true }, + }, + { + manager, + } + ); + + Assert.equal( + featureInstance.getVariable("remoteValue"), + "mochitest", + "set by remote config" + ); +}); + +add_task(async function test_record_exposure_event() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + const getExperimentSpy = sandbox.spy(ExperimentAPI, "getExperimentMetaData"); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + featureInstance.recordExposureEvent(); + + Assert.ok( + exposureSpy.notCalled, + "should not emit an exposure event when no experiment is active" + ); + + // Check that there aren't any Glean exposure events yet + var exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + Assert.equal( + undefined, + exposureEvents, + "no Glean exposure events before exposure" + ); + + await manager.store.addEnrollment( + ExperimentFakes.experiment("blah", { + branch: { + slug: "treatment", + features: [ + { + featureId: "foo", + value: { enabled: false }, + }, + ], + }, + }) + ); + + featureInstance.recordExposureEvent(); + + Assert.ok( + exposureSpy.calledOnce, + "should emit an exposure event when there is an experiment" + ); + Assert.equal(getExperimentSpy.callCount, 2, "Should be called every time"); + + // Check that the Glean exposure event was recorded. + exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect only one event + Assert.equal(1, exposureEvents.length); + // And that one event matches the expected + Assert.equal( + "blah", + exposureEvents[0].extra.experiment, + "Glean.nimbusEvents.exposure recorded with correct experiment slug" + ); + Assert.equal( + "treatment", + exposureEvents[0].extra.branch, + "Glean.nimbusEvents.exposure recorded with correct branch slug" + ); + Assert.equal( + "foo", + exposureEvents[0].extra.feature_id, + "Glean.nimbusEvents.exposure recorded with correct feature id" + ); + + sandbox.restore(); +}); + +add_task(async function test_record_exposure_event_once() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.store.addEnrollment( + ExperimentFakes.experiment("blah", { + branch: { + slug: "treatment", + features: [ + { + featureId: "foo", + value: { enabled: false }, + }, + ], + }, + }) + ); + + featureInstance.recordExposureEvent({ once: true }); + featureInstance.recordExposureEvent({ once: true }); + featureInstance.recordExposureEvent({ once: true }); + + Assert.ok( + exposureSpy.calledOnce, + "Should emit a single exposure event when the once param is true." + ); + + // Check that the Glean exposure event was recorded. + let exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect only one event + Assert.equal(1, exposureEvents.length); + + sandbox.restore(); +}); + +add_task(async function test_allow_multiple_exposure_events() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const exposureSpy = sandbox.spy(ExperimentAPI, "recordExposureEvent"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + let doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "foo", + value: { enabled: false }, + }, + { manager } + ); + + featureInstance.recordExposureEvent(); + featureInstance.recordExposureEvent(); + featureInstance.recordExposureEvent(); + + Assert.ok(exposureSpy.called, "Should emit exposure event"); + Assert.equal( + exposureSpy.callCount, + 3, + "Should emit an exposure event for each function call" + ); + + // Check that the Glean exposure event was recorded. + let exposureEvents = Glean.nimbusEvents.exposure.testGetValue(); + // We expect 3 events + Assert.equal(3, exposureEvents.length); + + sandbox.restore(); + await doExperimentCleanup(); +}); + +add_task(async function test_onUpdate_before_store_ready() { + let sandbox = sinon.createSandbox(); + const feature = new ExperimentFeature("foo", FAKE_FEATURE_MANIFEST); + const stub = sandbox.stub(); + const manager = ExperimentFakes.manager(); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([ + ExperimentFakes.experiment("foo-experiment", { + branch: { + slug: "control", + features: [ + { + featureId: "foo", + value: null, + }, + ], + }, + }), + ]); + + // We register for updates before the store finished loading experiments + // from disk + feature.onUpdate(stub); + + await manager.onStartup(); + + Assert.ok( + stub.calledOnce, + "Called on startup after loading experiments from disk" + ); + Assert.equal( + stub.firstCall.args[0], + `featureUpdate:${feature.featureId}`, + "Called for correct feature" + ); + + Assert.equal( + stub.firstCall.args[1], + "feature-experiment-loaded", + "Called for the expected reason" + ); +}); + +add_task(async function test_ExperimentFeature_test_ready_late() { + const { manager, sandbox } = await setupForExperimentFeature(); + const stub = sandbox.stub(); + + const featureInstance = new ExperimentFeature( + "test-feature", + FAKE_FEATURE_MANIFEST + ); + + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "slug", + features: [ + { + featureId: featureInstance.featureId, + value: { + title: "hello", + enabled: true, + }, + }, + ], + }, + }); + + sandbox.stub(manager.store, "getAllActiveRollouts").returns([rollout]); + + await manager.onStartup(); + + featureInstance.onUpdate(stub); + + await featureInstance.ready(); + + Assert.ok(stub.calledOnce, "Callback called"); + Assert.equal(stub.firstCall.args[0], "featureUpdate:test-feature"); + Assert.equal(stub.firstCall.args[1], "rollout-updated"); + + setDefaultBranch(TEST_FALLBACK_PREF, JSON.stringify({ foo: true })); + + Assert.deepEqual( + featureInstance.getVariable("config"), + { foo: true }, + "Feature is ready even when initialized after store update" + ); + Assert.equal( + featureInstance.getVariable("title"), + "hello", + "Returns the NimbusTestUtils rollout default value" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js new file mode 100644 index 0000000000..fd9e09c03d --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getAllVariables.js @@ -0,0 +1,249 @@ +"use strict"; + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +const { cleanupStorePrefCache } = ExperimentFakes; + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +const FEATURE_ID = "aboutwelcome"; +const TEST_FALLBACK_PREF = "browser.aboutwelcome.screens"; +const FAKE_FEATURE_MANIFEST = { + variables: { + screens: { + type: "json", + fallbackPref: TEST_FALLBACK_PREF, + }, + source: { + type: "string", + }, + }, +}; + +add_task( + async function test_ExperimentFeature_getAllVariables_prefsOverDefaults() { + const { sandbox } = await setupForExperimentFeature(); + + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables().screens?.length, + undefined, + "pref is not set" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[]"); + + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the user pref value over the defaults" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + sandbox.restore(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_experimentOverPref() { + const { sandbox, manager } = await setupForExperimentFeature(); + const recipe = ExperimentFakes.experiment("awexperiment", { + branch: { + slug: "treatment", + features: [ + { + featureId: "aboutwelcome", + value: { screens: ["test-value"] }, + }, + ], + }, + }); + + await manager.store.addEnrollment(recipe); + + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.ok( + !!featureInstance.getAllVariables().screens, + "should return the AW experiment value" + ); + + Assert.equal( + featureInstance.getAllVariables().screens[0], + "test-value", + "should return the AW experiment value" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[]"); + Assert.equal( + featureInstance.getAllVariables().screens[0], + "test-value", + "should return the AW experiment value" + ); + + await ExperimentFakes.cleanupAll([recipe.slug], { manager }); + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the user pref value" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + sandbox.restore(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_experimentOverRemote() { + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + const recipe = ExperimentFakes.experiment("aw-experiment", { + branch: { + slug: "treatment", + features: [ + { + featureId: FEATURE_ID, + value: { screens: ["test-value"] }, + }, + ], + }, + }); + const rollout = ExperimentFakes.rollout("aw-rollout", { + branch: { + slug: "treatment", + features: [ + { featureId: FEATURE_ID, value: { screens: [], source: "rollout" } }, + ], + }, + }); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + const rolloutPromise = new Promise(resolve => + featureInstance.onUpdate((feature, reason) => { + if (reason === "rollout-updated") { + resolve(); + } + }) + ); + const experimentPromise = new Promise(resolve => + featureInstance.onUpdate((feature, reason) => { + if (reason === "experiment-updated") { + resolve(); + } + }) + ); + manager.store.addEnrollment(recipe); + manager.store.addEnrollment(rollout); + await rolloutPromise; + await experimentPromise; + + let allVariables = featureInstance.getAllVariables(); + + Assert.equal(allVariables.screens.length, 1, "Returns experiment value"); + Assert.ok(!allVariables.source, "Does not include rollout value"); + + await ExperimentFakes.cleanupAll([recipe.slug], { manager }); + cleanupStorePrefCache(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_rolloutOverPrefDefaults() { + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + const rollout = ExperimentFakes.rollout("foo-aw", { + branch: { + slug: "getAllVariables", + features: [{ featureId: FEATURE_ID, value: { screens: [] } }], + }, + }); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables().screens?.length, + undefined, + "Pref is not set" + ); + + const updatePromise = new Promise(resolve => + featureInstance.onUpdate(resolve) + ); + // Load remote defaults + manager.store.addEnrollment(rollout); + + // Wait for feature to load the rollout + await updatePromise; + + Assert.deepEqual( + featureInstance.getAllVariables().screens?.length, + 0, + "Should return the rollout value over the defaults" + ); + + Services.prefs.setStringPref(TEST_FALLBACK_PREF, "[1,2,3]"); + + Assert.deepEqual( + featureInstance.getAllVariables().screens.length, + 0, + "should return the rollout value over the user pref" + ); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + cleanupStorePrefCache(); + } +); + +add_task( + async function test_ExperimentFeature_getAllVariables_defaultValuesParam() { + const { manager } = await setupForExperimentFeature(); + const featureInstance = new ExperimentFeature( + FEATURE_ID, + FAKE_FEATURE_MANIFEST + ); + // We're using the store in this test we need to wait for it to load + await manager.store.ready(); + + Services.prefs.clearUserPref(TEST_FALLBACK_PREF); + + Assert.equal( + featureInstance.getAllVariables({ defaultValues: { screens: null } }) + .screens, + null, + "should return defaultValues param over default pref settings" + ); + } +); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js new file mode 100644 index 0000000000..4866b23a1a --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_ExperimentFeature_getVariable.js @@ -0,0 +1,196 @@ +"use strict"; + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +const FEATURE_ID = "testfeature1"; +// Note: this gets deleted at the end of tests +const TEST_PREF_BRANCH = "testfeature1."; +const TEST_VARIABLES = { + enabled: { + type: "boolean", + fallbackPref: `${TEST_PREF_BRANCH}enabled`, + }, + name: { + type: "string", + fallbackPref: `${TEST_PREF_BRANCH}name`, + }, + count: { + type: "int", + fallbackPref: `${TEST_PREF_BRANCH}count`, + }, + items: { + type: "json", + fallbackPref: `${TEST_PREF_BRANCH}items`, + }, +}; + +function createInstanceWithVariables(variables) { + return new ExperimentFeature(FEATURE_ID, { + variables, + }); +} + +add_task(async function test_ExperimentFeature_getFallbackPrefName() { + const instance = createInstanceWithVariables(TEST_VARIABLES); + + Assert.equal( + instance.getFallbackPrefName("enabled"), + "testfeature1.enabled", + "should return the fallback preference name" + ); +}); + +add_task(async function test_ExperimentFeature_getVariable_notRegistered() { + const instance = createInstanceWithVariables(TEST_VARIABLES); + + if (Cu.isInAutomation || AppConstants.NIGHTLY_BUILD) { + Assert.throws( + () => { + instance.getVariable("non_existant_variable"); + }, + /Nimbus: Warning - variable "non_existant_variable" is not defined in FeatureManifest\.yaml/, + "should throw in automation for variables not defined in the manifest" + ); + } else { + info("Won't throw when running in Beta and release candidates"); + } +}); + +add_task(async function test_ExperimentFeature_getVariable_noFallbackPref() { + const instance = createInstanceWithVariables({ + foo: { type: "json" }, + }); + + Assert.equal( + instance.getVariable("foo"), + undefined, + "should return undefined if no values are set and no fallback pref is defined" + ); +}); + +add_task(async function test_ExperimentFeature_getVariable_precedence() { + const { sandbox, manager } = await setupForExperimentFeature(); + + const instance = createInstanceWithVariables(TEST_VARIABLES); + const prefName = TEST_VARIABLES.items.fallbackPref; + const rollout = ExperimentFakes.rollout(`${FEATURE_ID}-rollout`, { + branch: { + slug: "slug", + features: [ + { + featureId: FEATURE_ID, + value: { items: [4, 5, 6] }, + }, + ], + }, + }); + + Services.prefs.clearUserPref(prefName); + + Assert.equal( + instance.getVariable("items"), + undefined, + "should return undefined if the fallback pref is not set" + ); + + // Default pref values + Services.prefs.setStringPref(prefName, JSON.stringify([1, 2, 3])); + + Assert.deepEqual( + instance.getVariable("items"), + [1, 2, 3], + "should return the default pref value" + ); + + // Remote default values + await manager.store.addEnrollment(rollout); + + Assert.deepEqual( + instance.getVariable("items"), + [4, 5, 6], + "should return the remote default value over the default pref value" + ); + + // Experiment values + const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: FEATURE_ID, + value: { + items: [7, 8, 9], + }, + }, + { manager } + ); + + Assert.deepEqual( + instance.getVariable("items"), + [7, 8, 9], + "should return the experiment value over the remote value" + ); + + // Cleanup + Services.prefs.deleteBranch(TEST_PREF_BRANCH); + await doExperimentCleanup(); + sandbox.restore(); +}); + +add_task(async function test_ExperimentFeature_getVariable_partial_values() { + const { sandbox, manager } = await setupForExperimentFeature(); + const instance = createInstanceWithVariables(TEST_VARIABLES); + const rollout = ExperimentFakes.rollout(`${FEATURE_ID}-rollout`, { + branch: { + slug: "slug", + features: [ + { + featureId: FEATURE_ID, + value: { name: "abc" }, + }, + ], + }, + }); + + // Set up a pref value for .enabled, + // a remote value for .name, + // an experiment value for .items + Services.prefs.setBoolPref(TEST_VARIABLES.enabled.fallbackPref, true); + await manager.store.addEnrollment(rollout); + const doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: FEATURE_ID, + value: {}, + }, + { manager } + ); + + Assert.equal( + instance.getVariable("enabled"), + true, + "should skip missing variables from remote defaults" + ); + + Assert.equal( + instance.getVariable("name"), + "abc", + "should skip missing variables from experiments" + ); + + // Cleanup + Services.prefs.getDefaultBranch("").deleteBranch(TEST_PREF_BRANCH); + Services.prefs.deleteBranch(TEST_PREF_BRANCH); + await doExperimentCleanup(); + sandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js new file mode 100644 index 0000000000..9333a128f5 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentAPI_NimbusFeatures.js @@ -0,0 +1,289 @@ +"use strict"; + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); + +const { JsonSchema } = ChromeUtils.importESModule( + "resource://gre/modules/JsonSchema.sys.mjs" +); + +Cu.importGlobalProperties(["fetch"]); + +XPCOMUtils.defineLazyGetter(this, "fetchSchema", () => { + return fetch("resource://nimbus/schemas/NimbusEnrollment.schema.json", { + credentials: "omit", + }).then(rsp => rsp.json()); +}); + +const NON_MATCHING_ROLLOUT = Object.freeze( + ExperimentFakes.rollout("non-matching-rollout", { + branch: { + slug: "slug", + features: [ + { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + ], + }, + }) +); +const MATCHING_ROLLOUT = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + features: [ + { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + ], + }, + }) +); + +const AW_FAKE_MANIFEST = { + description: "Different manifest with a special test variable", + isEarlyStartup: true, + variables: { + remoteValue: { + type: "boolean", + description: "Test value", + }, + mochitest: { + type: "boolean", + }, + enabled: { + type: "boolean", + }, + }, +}; + +async function setupForExperimentFeature() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + await manager.onStartup(); + + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + return { sandbox, manager }; +} + +add_task(async function validSchema() { + const validator = new JsonSchema.Validator(await fetchSchema, { + shortCircuit: false, + }); + + { + const result = validator.validate(NON_MATCHING_ROLLOUT); + Assert.ok(result.valid, JSON.stringify(result.errors, undefined, 2)); + } + { + const result = validator.validate(MATCHING_ROLLOUT); + Assert.ok(result.valid, JSON.stringify(result.errors, undefined, 2)); + } +}); + +add_task(async function readyCallAfterStore_with_remote_value() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + + Assert.ok(feature.getVariable("enabled"), "Feature is true by default"); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(!feature.getVariable("enabled"), "Loads value from store"); + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function has_sync_value_before_ready() { + let { manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome", AW_FAKE_MANIFEST); + + Assert.equal( + feature.getVariable("remoteValue"), + undefined, + "Feature is true by default" + ); + + Services.prefs.setStringPref( + "nimbus.syncdefaultsstore.aboutwelcome", + JSON.stringify({ + ...MATCHING_ROLLOUT, + branch: { feature: MATCHING_ROLLOUT.branch.features[0] }, + }) + ); + + Services.prefs.setBoolPref( + "nimbus.syncdefaultsstore.aboutwelcome.remoteValue", + true + ); + + Assert.equal(feature.getVariable("remoteValue"), true, "Sync load from pref"); + + manager.store._deleteForTests("aboutwelcome"); +}); + +add_task(async function update_remote_defaults_onUpdate() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + let stub = sandbox.stub(); + + feature.onUpdate(stub); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(stub.called, "update event called"); + Assert.equal(stub.callCount, 1, "Called once for remote configs"); + Assert.equal(stub.firstCall.args[1], "rollout-updated", "Correct reason"); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function test_features_over_feature() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + const rollout_features_and_feature = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + feature: { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + features: [ + { + featureId: "aboutwelcome", + value: { enabled: true }, + }, + ], + }, + }) + ); + const rollout_just_feature = Object.freeze( + ExperimentFakes.rollout("matching-rollout", { + branch: { + slug: "slug", + feature: { + featureId: "aboutwelcome", + value: { enabled: false }, + }, + }, + }) + ); + + await manager.store.addEnrollment(rollout_features_and_feature); + Assert.ok( + feature.getVariable("enabled"), + "Should read from the features property over feature" + ); + + manager.store._deleteForTests("aboutwelcome"); + manager.store._deleteForTests("matching-rollout"); + + await manager.store.addEnrollment(rollout_just_feature); + Assert.ok( + !feature.getVariable("enabled"), + "Should read from the feature property when features doesn't exist" + ); + + manager.store._deleteForTests("aboutwelcome"); + manager.store._deleteForTests("matching-rollout"); + sandbox.restore(); +}); + +add_task(async function update_remote_defaults_readyPromise() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + let stub = sandbox.stub(); + + feature.onUpdate(stub); + + await manager.store.addEnrollment(MATCHING_ROLLOUT); + + Assert.ok(stub.calledOnce, "Update called after enrollment processed."); + Assert.ok( + stub.calledWith("featureUpdate:aboutwelcome", "rollout-updated"), + "Update called after enrollment processed." + ); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +add_task(async function update_remote_defaults_enabled() { + let { sandbox, manager } = await setupForExperimentFeature(); + let feature = new ExperimentFeature("aboutwelcome"); + + Assert.equal( + feature.getVariable("enabled"), + true, + "Feature is enabled by manifest.variables.enabled" + ); + + await manager.store.addEnrollment(NON_MATCHING_ROLLOUT); + + Assert.ok( + !feature.getVariable("enabled"), + "Feature is disabled by remote configuration" + ); + + manager.store._deleteForTests("aboutwelcome"); + sandbox.restore(); +}); + +// If the branch data returned from the store is not modified +// this test should not throw +add_task(async function test_getVariable_no_mutation() { + let { sandbox, manager } = await setupForExperimentFeature(); + sandbox.stub(manager.store, "getExperimentForFeature").returns( + Cu.cloneInto( + { + branch: { + features: [{ featureId: "aboutwelcome", value: { mochitest: true } }], + }, + }, + {}, + { deepFreeze: true } + ) + ); + let feature = new ExperimentFeature("aboutwelcome", AW_FAKE_MANIFEST); + + Assert.ok(feature.getVariable("mochitest"), "Got back the expected feature"); + + sandbox.restore(); +}); + +add_task(async function remote_isEarlyStartup_config() { + let { manager } = await setupForExperimentFeature(); + let rollout = ExperimentFakes.rollout("password-autocomplete", { + branch: { + slug: "remote-config-isEarlyStartup", + features: [ + { + featureId: "password-autocomplete", + enabled: true, + value: { remote: true }, + isEarlyStartup: true, + }, + ], + }, + }); + + await manager.onStartup(); + await manager.store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.prefHasUserValue( + "nimbus.syncdefaultsstore.password-autocomplete" + ), + "Configuration is marked early startup" + ); + + Services.prefs.clearUserPref( + "nimbus.syncdefaultsstore.password-autocomplete" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js new file mode 100644 index 0000000000..b8cdb3afc9 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_context.js @@ -0,0 +1,41 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +const { FirstStartup } = ChromeUtils.importESModule( + "resource://gre/modules/FirstStartup.sys.mjs" +); + +add_task(async function test_createTargetingContext() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const rollout = ExperimentFakes.rollout("bar"); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([recipe]); + sandbox.stub(manager.store, "getAllActiveRollouts").returns([rollout]); + + let context = manager.createTargetingContext(); + const activeSlugs = await context.activeExperiments; + const activeRollouts = await context.activeRollouts; + + Assert.ok(!context.isFirstStartup, "should not set the first startup flag"); + Assert.deepEqual( + activeSlugs, + ["foo"], + "should return slugs for all the active experiment" + ); + Assert.deepEqual( + activeRollouts, + ["bar"], + "should return slugs for all rollouts stored" + ); + + // Pretend to be in the first startup + FirstStartup._state = FirstStartup.IN_PROGRESS; + context = manager.createTargetingContext(); + + Assert.ok(context.isFirstStartup, "should set the first startup flag"); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js new file mode 100644 index 0000000000..ae84f6e7f6 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_enroll.js @@ -0,0 +1,1023 @@ +"use strict"; + +const { NormandyTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/NormandyTestUtils.sys.mjs" +); +const { Sampling } = ChromeUtils.importESModule( + "resource://gre/modules/components-utils/Sampling.sys.mjs" +); +const { ClientEnvironment } = ChromeUtils.importESModule( + "resource://normandy/lib/ClientEnvironment.sys.mjs" +); +const { cleanupStorePrefCache } = ExperimentFakes; + +const { ExperimentStore } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentStore.sys.mjs" +); +const { TelemetryEnvironment } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryEnvironment.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.importESModule( + "resource://normandy/lib/TelemetryEvents.sys.mjs" +); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; + +const globalSandbox = sinon.createSandbox(); +globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); +globalSandbox.spy(TelemetryEvents, "sendEvent"); +registerCleanupFunction(() => { + globalSandbox.restore(); +}); + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +/** + * The normal case: Enrollment of a new experiment + */ +add_task(async function test_add_to_store() { + const manager = ExperimentFakes.manager(); + const recipe = ExperimentFakes.recipe("foo"); + const enrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + + await manager.onStartup(); + + await manager.enroll(recipe, "test_add_to_store"); + await enrollPromise; + const experiment = manager.store.get("foo"); + + Assert.ok(experiment, "should add an experiment with slug foo"); + Assert.ok( + recipe.branches.includes(experiment.branch), + "should choose a branch from the recipe.branches" + ); + Assert.equal(experiment.active, true, "should set .active = true"); + Assert.ok( + NormandyTestUtils.isUuid(experiment.enrollmentId), + "should add a valid enrollmentId" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_add_rollout_to_store() { + const manager = ExperimentFakes.manager(); + const recipe = { + ...ExperimentFakes.recipe("rollout-slug"), + branches: [ExperimentFakes.rollout("rollout").branch], + isRollout: true, + active: true, + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + }; + const enrollPromise = new Promise(resolve => + manager.store.on("update:rollout-slug", resolve) + ); + + await manager.onStartup(); + + await manager.enroll(recipe, "test_add_rollout_to_store"); + await enrollPromise; + const experiment = manager.store.get("rollout-slug"); + + Assert.ok(experiment, `Should add an experiment with slug ${recipe.slug}`); + Assert.ok( + recipe.branches.includes(experiment.branch), + "should choose a branch from the recipe.branches" + ); + Assert.equal(experiment.isRollout, true, "should have .isRollout"); + + manager.unenroll("rollout-slug", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task( + async function test_setExperimentActive_sendEnrollmentTelemetry_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const enrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + sandbox.spy(manager, "setExperimentActive"); + sandbox.spy(manager, "sendEnrollmentTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Ensure there is no experiment active with the id in FOG + Assert.equal( + undefined, + Services.fog.testGetExperimentData("foo"), + "no active experiment exists before enrollment" + ); + + // Check that there aren't any Glean enrollment events yet + var enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + Assert.equal( + undefined, + enrollmentEvents, + "no Glean enrollment events before enrollment" + ); + + await manager.enroll( + ExperimentFakes.recipe("foo"), + "test_setExperimentActive_sendEnrollmentTelemetry_called" + ); + await enrollPromise; + const experiment = manager.store.get("foo"); + + Assert.equal( + manager.setExperimentActive.calledWith(experiment), + true, + "should call setExperimentActive after an enrollment" + ); + + Assert.equal( + manager.sendEnrollmentTelemetry.calledWith(experiment), + true, + "should call sendEnrollmentTelemetry after an enrollment" + ); + + // Test Glean experiment API interaction + Assert.notEqual( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "Glean.setExperimentActive called with `foo` feature" + ); + + // Check that the Glean enrollment event was recorded. + enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + // We expect only one event + Assert.equal(1, enrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + enrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.enrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + enrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.enrollment recorded with correct branch slug" + ); + Assert.equal( + experiment.experimentType, + enrollmentEvents[0].extra.experiment_type, + "Glean.nimbusEvents.enrollment recorded with correct experiment type" + ); + Assert.equal( + experiment.enrollmentId, + enrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.enrollment recorded with correct enrollment id" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); + } +); + +add_task(async function test_setRolloutActive_sendEnrollmentTelemetry_called() { + globalSandbox.reset(); + globalSandbox.spy(TelemetryEnvironment, "setExperimentActive"); + globalSandbox.spy(TelemetryEvents.sendEvent); + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const rolloutRecipe = { + ...ExperimentFakes.recipe("rollout"), + branches: [ExperimentFakes.rollout("rollout").branch], + isRollout: true, + }; + const enrollPromise = new Promise(resolve => + manager.store.on("update:rollout", resolve) + ); + sandbox.spy(manager, "setExperimentActive"); + sandbox.spy(manager, "sendEnrollmentTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData("rollout"), + "no rollout active before enrollment" + ); + + // Check that there aren't any Glean enrollment events yet + var enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + Assert.equal( + undefined, + enrollmentEvents, + "no Glean enrollment events before enrollment" + ); + + let result = await manager.enroll( + rolloutRecipe, + "test_setRolloutActive_sendEnrollmentTelemetry_called" + ); + + await enrollPromise; + + const enrollment = manager.store.get("rollout"); + + Assert.ok(!!result && !!enrollment, "Enrollment was successful"); + + Assert.equal( + TelemetryEnvironment.setExperimentActive.called, + true, + "should call setExperimentActive" + ); + Assert.ok( + manager.setExperimentActive.calledWith(enrollment), + "Should call setExperimentActive with the rollout" + ); + Assert.equal( + manager.setExperimentActive.firstCall.args[0].experimentType, + "rollout", + "Should have the correct experimentType" + ); + Assert.equal( + manager.sendEnrollmentTelemetry.calledWith(enrollment), + true, + "should call sendEnrollmentTelemetry after an enrollment" + ); + Assert.ok( + TelemetryEvents.sendEvent.calledOnce, + "Should send out enrollment telemetry" + ); + Assert.ok( + TelemetryEvents.sendEvent.calledWith( + "enroll", + sinon.match.string, + enrollment.slug, + { + experimentType: "rollout", + branch: enrollment.branch.slug, + enrollmentId: enrollment.enrollmentId, + } + ), + "Should send telemetry with expected values" + ); + + // Test Glean experiment API interaction + Assert.equal( + enrollment.branch.slug, + Services.fog.testGetExperimentData(enrollment.slug).branch, + "Glean.setExperimentActive called with expected values" + ); + + // Check that the Glean enrollment event was recorded. + enrollmentEvents = Glean.nimbusEvents.enrollment.testGetValue(); + // We expect only one event + Assert.equal(1, enrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + enrollment.slug, + enrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.enrollment recorded with correct experiment slug" + ); + Assert.equal( + enrollment.branch.slug, + enrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.enrollment recorded with correct branch slug" + ); + Assert.equal( + enrollment.experimentType, + enrollmentEvents[0].extra.experiment_type, + "Glean.nimbusEvents.enrollment recorded with correct experiment type" + ); + Assert.equal( + enrollment.enrollmentId, + enrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.enrollment recorded with correct enrollment id" + ); + + manager.unenroll("rollout", "test-cleanup"); + + await assertEmptyStore(manager.store); + + globalSandbox.restore(); +}); + +// /** +// * Failure cases: +// * - slug conflict +// * - group conflict +// */ + +add_task(async function test_failure_name_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // simulate adding a previouly enrolled experiment + await manager.store.addEnrollment(ExperimentFakes.experiment("foo")); + + await Assert.rejects( + manager.enroll(ExperimentFakes.recipe("foo"), "test_failure_name_conflict"), + /An experiment with the slug "foo" already exists/, + "should throw if a conflicting experiment exists" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + "foo", + "name-conflict" + ), + true, + "should send failure telemetry if a conflicting experiment exists" + ); + + // Check that the Glean enrollment event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + "foo", + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "name-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_failure_group_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // Two conflicting branches that both have the group "pink" + // These should not be allowed to exist simultaneously. + const existingBranch = { + slug: "treatment", + features: [{ featureId: "pink", value: {} }], + }; + const newBranch = { + slug: "treatment", + features: [{ featureId: "pink", value: {} }], + }; + + // simulate adding an experiment with a conflicting group "pink" + await manager.store.addEnrollment( + ExperimentFakes.experiment("foo", { + branch: existingBranch, + }) + ); + + // ensure .enroll chooses the special branch with the conflict + sandbox.stub(manager, "chooseBranch").returns(newBranch); + Assert.equal( + await manager.enroll( + ExperimentFakes.recipe("bar", { branches: [newBranch] }), + "test_failure_group_conflict" + ), + null, + "should not enroll if there is a feature conflict" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + "bar", + "feature-conflict" + ), + true, + "should send failure telemetry if a feature conflict exists" + ); + + // Check that the Glean enroll_failed event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that event matches the expected experiment and reason + Assert.equal( + "bar", + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "feature-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("foo", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_rollout_failure_group_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const rollout = ExperimentFakes.rollout("rollout-enrollment"); + const recipe = { + ...ExperimentFakes.recipe("rollout-recipe"), + branches: [rollout.branch], + isRollout: true, + }; + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + // simulate adding an experiment with a conflicting group "pink" + await manager.store.addEnrollment(rollout); + + Assert.equal( + await manager.enroll(recipe, "test_rollout_failure_group_conflict"), + null, + "should not enroll if there is a feature conflict" + ); + + Assert.equal( + manager.sendFailureTelemetry.calledWith( + "enrollFailed", + recipe.slug, + "feature-conflict" + ), + true, + "should send failure telemetry if a feature conflict exists" + ); + + // Check that the Glean enroll_failed event was recorded. + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + // We expect only one event + Assert.equal(1, failureEvents.length); + // And that event matches the expected experiment and reason + Assert.equal( + recipe.slug, + failureEvents[0].extra.experiment, + "Glean.nimbusEvents.enroll_failed recorded with correct experiment slug" + ); + Assert.equal( + "feature-conflict", + failureEvents[0].extra.reason, + "Glean.nimbusEvents.enroll_failed recorded with correct reason" + ); + + manager.unenroll("rollout-enrollment", "test-cleanup"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_rollout_experiment_no_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const experiment = ExperimentFakes.recipe("experiment"); + const rollout = ExperimentFakes.recipe("rollout", { isRollout: true }); + + sandbox.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean enroll_failed events yet + var failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + await ExperimentFakes.enrollmentHelper(experiment, { + manager, + }).enrollmentPromise; + await ExperimentFakes.enrollmentHelper(rollout, { + manager, + }).enrollmentPromise; + + Assert.ok( + manager.store.get(experiment.slug).active, + "Enrolled in the experiment for the feature" + ); + + Assert.ok( + manager.store.get(rollout.slug).active, + "Enrolled in the rollout for the feature" + ); + + Assert.ok( + manager.sendFailureTelemetry.notCalled, + "Should send failure telemetry if a feature conflict exists" + ); + + // Check that there aren't any Glean enroll_failed events + failureEvents = Glean.nimbusEvents.enrollFailed.testGetValue(); + Assert.equal( + undefined, + failureEvents, + "no Glean enroll_failed events before failure" + ); + + await ExperimentFakes.cleanupAll([experiment.slug, rollout.slug], { + manager, + }); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_sampling_check() { + const manager = ExperimentFakes.manager(); + let recipe = ExperimentFakes.recipe("foo", { bucketConfig: null }); + const sandbox = sinon.createSandbox(); + sandbox.stub(Sampling, "bucketSample").resolves(true); + sandbox.replaceGetter(ClientEnvironment, "userId", () => 42); + + Assert.ok( + !manager.isInBucketAllocation(recipe.bucketConfig), + "fails for no bucket config" + ); + + recipe = ExperimentFakes.recipe("foo2", { + bucketConfig: { randomizationUnit: "foo" }, + }); + + Assert.ok( + !manager.isInBucketAllocation(recipe.bucketConfig), + "fails for unknown randomizationUnit" + ); + + recipe = ExperimentFakes.recipe("foo3"); + + const result = await manager.isInBucketAllocation(recipe.bucketConfig); + + Assert.equal( + Sampling.bucketSample.callCount, + 1, + "it should call bucketSample" + ); + Assert.ok(result, "result should be true"); + const { args } = Sampling.bucketSample.firstCall; + Assert.equal(args[0][0], 42, "called with expected randomization id"); + Assert.equal( + args[0][1], + recipe.bucketConfig.namespace, + "called with expected namespace" + ); + Assert.equal( + args[1], + recipe.bucketConfig.start, + "called with expected start" + ); + Assert.equal( + args[2], + recipe.bucketConfig.count, + "called with expected count" + ); + Assert.equal( + args[3], + recipe.bucketConfig.total, + "called with expected total" + ); + + await assertEmptyStore(manager.store); + + sandbox.reset(); +}); + +add_task(async function enroll_in_reference_aw_experiment() { + cleanupStorePrefCache(); + + let dir = Services.dirsvc.get("CurWorkD", Ci.nsIFile).path; + let src = PathUtils.join( + dir, + "reference_aboutwelcome_experiment_content.json" + ); + const content = await IOUtils.readJSON(src); + // Create two dummy branches with the content from disk + const branches = ["treatment-a", "treatment-b"].map(slug => ({ + slug, + ratio: 1, + features: [ + { value: { ...content, enabled: true }, featureId: "aboutwelcome" }, + ], + })); + let recipe = ExperimentFakes.recipe("reference-aw", { branches }); + // Ensure we get enrolled + recipe.bucketConfig.count = recipe.bucketConfig.total; + + const manager = ExperimentFakes.manager(); + const enrollPromise = new Promise(resolve => + manager.store.on("update:reference-aw", resolve) + ); + await manager.onStartup(); + await manager.enroll(recipe, "enroll_in_reference_aw_experiment"); + await enrollPromise; + + Assert.ok(manager.store.get("reference-aw"), "Successful onboarding"); + let prefValue = Services.prefs.getStringPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome` + ); + Assert.ok( + prefValue, + "aboutwelcome experiment enrollment should be stored to prefs" + ); + // In case some regression causes us to store a significant amount of data + // in prefs. + Assert.ok(prefValue.length < 3498, "Make sure we don't bloat the prefs"); + + manager.unenroll(recipe.slug, "enroll_in_reference_aw_experiment:cleanup"); + manager.store._deleteForTests("aboutwelcome"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_forceEnroll_cleanup() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const fooEnrollPromise = new Promise(resolve => + manager.store.on("update:foo", resolve) + ); + const barEnrollPromise = new Promise(resolve => + manager.store.on("update:optin-bar", resolve) + ); + let unenrollStub = sandbox.spy(manager, "unenroll"); + let existingRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "treatment", + ratio: 1, + features: [{ featureId: "force-enrollment", value: {} }], + }, + ], + }); + let forcedRecipe = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "treatment", + ratio: 1, + features: [{ featureId: "force-enrollment", value: {} }], + }, + ], + }); + + await manager.onStartup(); + await manager.enroll(existingRecipe, "test_forceEnroll_cleanup"); + await fooEnrollPromise; + + let setExperimentActiveSpy = sandbox.spy(manager, "setExperimentActive"); + manager.forceEnroll(forcedRecipe, forcedRecipe.branches[0]); + await barEnrollPromise; + + Assert.ok(unenrollStub.called, "Unenrolled from existing experiment"); + Assert.equal( + unenrollStub.firstCall.args[0], + existingRecipe.slug, + "Called with existing recipe slug" + ); + Assert.ok(setExperimentActiveSpy.calledOnce, "Activated forced experiment"); + Assert.equal( + setExperimentActiveSpy.firstCall.args[0].slug, + `optin-${forcedRecipe.slug}`, + "Called with forced experiment slug" + ); + Assert.equal( + manager.store.getExperimentForFeature("force-enrollment").slug, + `optin-${forcedRecipe.slug}`, + "Enrolled in forced experiment" + ); + + manager.unenroll(`optin-${forcedRecipe.slug}`, "test-cleanup"); + + await assertEmptyStore(manager.store); + + sandbox.restore(); +}); + +add_task(async function test_rollout_unenroll_conflict() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + let unenrollStub = sandbox.stub(manager, "unenroll").returns(true); + let enrollStub = sandbox.stub(manager, "_enroll").returns(true); + let rollout = ExperimentFakes.rollout("rollout_conflict"); + + // We want to force a conflict + sandbox.stub(manager.store, "getRolloutForFeature").returns(rollout); + + manager.forceEnroll(rollout, rollout.branch); + + Assert.ok(unenrollStub.calledOnce, "Should unenroll the conflicting rollout"); + Assert.ok( + unenrollStub.calledWith(rollout.slug, "force-enrollment"), + "Should call with expected slug" + ); + Assert.ok(enrollStub.calledOnce, "Should call enroll as expected"); + + await assertEmptyStore(manager.store); + + sandbox.restore(); +}); + +add_task(async function test_forceEnroll() { + const experiment1 = ExperimentFakes.recipe("experiment-1"); + const experiment2 = ExperimentFakes.recipe("experiment-2"); + const rollout1 = ExperimentFakes.recipe("rollout-1", { isRollout: true }); + const rollout2 = ExperimentFakes.recipe("rollout-2", { isRollout: true }); + + const TEST_CASES = [ + { + enroll: [experiment1, rollout1], + expected: [experiment1, rollout1], + }, + { + enroll: [rollout1, experiment1], + expected: [experiment1, rollout1], + }, + { + enroll: [experiment1, experiment2], + expected: [experiment2], + }, + { + enroll: [rollout1, rollout2], + expected: [rollout2], + }, + { + enroll: [experiment1, rollout1, rollout2, experiment2], + expected: [experiment2, rollout2], + }, + ]; + + async function forceEnroll(manager, recipe) { + const enrollmentPromise = new Promise(resolve => { + manager.store.on(`update:optin-${recipe.slug}`, resolve); + }); + + manager.forceEnroll(recipe, recipe.branches[0]); + + return enrollmentPromise; + } + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([experiment1, experiment2, rollout1, rollout2]); + sinon.stub(loader, "setTimer"); + + await loader.init(); + await manager.onStartup(); + + for (const { enroll, expected } of TEST_CASES) { + for (const recipe of enroll) { + await forceEnroll(manager, recipe); + } + + const activeSlugs = manager.store + .getAll() + .filter(enrollment => enrollment.active) + .map(r => r.slug); + + Assert.equal( + activeSlugs.length, + expected.length, + `Should be enrolled in ${expected.length} experiments and rollouts` + ); + + for (const { slug, isRollout } of expected) { + Assert.ok( + activeSlugs.includes(`optin-${slug}`), + `Should be enrolled in ${ + isRollout ? "rollout" : "experiment" + } with slug optin-${slug}` + ); + } + + for (const { slug } of expected) { + manager.unenroll(`optin-${slug}`); + manager.store._deleteForTests(`optin-${slug}`); + } + } + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_featureIds_is_stored() { + Services.prefs.setStringPref("messaging-system.log", "all"); + const recipe = ExperimentFakes.recipe("featureIds"); + // Ensure we get enrolled + recipe.bucketConfig.count = recipe.bucketConfig.total; + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + const { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(recipe, { manager }); + + await enrollmentPromise; + + Assert.ok(manager.store.addEnrollment.calledOnce, "experiment is stored"); + let [enrollment] = manager.store.addEnrollment.firstCall.args; + Assert.ok("featureIds" in enrollment, "featureIds is stored"); + Assert.deepEqual( + enrollment.featureIds, + ["testFeature"], + "Has expected value" + ); + + await doExperimentCleanup(); + + await assertEmptyStore(manager.store); +}); + +add_task(async function experiment_and_rollout_enroll_and_cleanup() { + let store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + let rolloutCleanup = await ExperimentFakes.enrollWithRollout( + { + featureId: "aboutwelcome", + value: { enabled: true }, + }, + { + manager, + } + ); + + let experimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "aboutwelcome", + value: { enabled: true }, + }, + { manager } + ); + + Assert.ok( + Services.prefs.getBoolPref(`${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`) + ); + Assert.ok( + Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled` + ) + ); + + await experimentCleanup(); + + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + Assert.ok( + Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled` + ) + ); + + await rolloutCleanup(); + + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DATA_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + Assert.ok( + !Services.prefs.getBoolPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome.enabled`, + false + ) + ); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_reEnroll() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment"); + experiment.bucketConfig = { + ...experiment.bucketConfig, + start: 0, + count: 1000, + total: 1000, + }; + const rollout = ExperimentFakes.recipe("rollout", { isRollout: true }); + rollout.bucketConfig = { + ...rollout.bucketConfig, + start: 0, + count: 1000, + total: 1000, + }; + + await manager.enroll(experiment, "test"); + Assert.equal( + manager.store.getExperimentForFeature("testFeature")?.slug, + experiment.slug, + "Should enroll in experiment" + ); + + await manager.enroll(rollout, "test"); + Assert.equal( + manager.store.getRolloutForFeature("testFeature")?.slug, + rollout.slug, + "Should enroll in rollout" + ); + + manager.unenroll(experiment.slug); + Assert.ok( + !manager.store.getExperimentForFeature("testFeature"), + "Should unenroll from experiment" + ); + + manager.unenroll(rollout.slug); + Assert.ok( + !manager.store.getRolloutForFeature("testFeature"), + "Should unenroll from rollout" + ); + + await Assert.rejects( + manager.enroll(experiment, "test", { reenroll: true }), + /An experiment with the slug "experiment" already exists/, + "Should not re-enroll in experiment" + ); + + await manager.enroll(rollout, "test", { reenroll: true }); + Assert.equal( + manager.store.getRolloutForFeature("testFeature")?.slug, + rollout.slug, + "Should re-enroll in rollout" + ); + + manager.unenroll(rollout.slug); + await assertEmptyStore(store); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js new file mode 100644 index 0000000000..83f7eb70d9 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_generateTestIds.js @@ -0,0 +1,144 @@ +"use strict"; +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); + +const TEST_CONFIG = { + slug: "test-experiment", + branches: [ + { + slug: "control", + ratio: 1, + }, + { + slug: "branchA", + ratio: 1, + }, + { + slug: "branchB", + ratio: 1, + }, + ], + namespace: "test-namespace", + start: 0, + count: 2000, + total: 10000, +}; + +add_task(async function test_generateTestIds() { + let result = await ExperimentManager.generateTestIds(TEST_CONFIG); + + Assert.ok(result, "should return object"); + Assert.ok(result.notInExperiment, "should have a id for no experiment"); + Assert.ok(result.control, "should have id for control"); + Assert.ok(result.branchA, "should have id for branchA"); + Assert.ok(result.branchB, "should have id for branchB"); +}); + +add_task(async function test_generateTestIds_bucketConfig() { + const { slug, branches, namespace, start, count, total } = TEST_CONFIG; + const result = await ExperimentManager.generateTestIds({ + slug, + branches, + bucketConfig: { namespace, start, count, total }, + }); + + Assert.ok(result, "should return object"); + Assert.ok(result.notInExperiment, "should have a id for no experiment"); + Assert.ok(result.control, "should have id for control"); + Assert.ok(result.branchA, "should have id for branchA"); + Assert.ok(result.branchB, "should have id for branchB"); +}); + +add_task(async function test_generateTestIds_withoutNot() { + const result = await ExperimentManager.generateTestIds({ + ...TEST_CONFIG, + count: TEST_CONFIG.total, + }); + + Assert.ok(result, "should return object"); + Assert.equal( + result.notInExperiment, + undefined, + "should not have a id for no experiment" + ); + Assert.ok(result.control, "should have id for control"); + Assert.ok(result.branchA, "should have id for branchA"); + Assert.ok(result.branchB, "should have id for branchB"); +}); + +add_task(async function test_generateTestIds_input_errors() { + const { slug, branches, namespace, start, count, total } = TEST_CONFIG; + await Assert.rejects( + ExperimentManager.generateTestIds({ + branches, + namespace, + start, + count, + total, + }), + /slug, namespace not in expected format/, + "should throw because of missing slug" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ slug, branches, start, count, total }), + /slug, namespace not in expected format/, + "should throw because of missing namespace" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + count, + total, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing start" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + start, + total, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing count" + ); + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches, + namespace, + count, + start, + }), + /Must include start, count, and total as integers/, + "should throw beause of missing total" + ); + + // Intentionally misspelled slug + let invalidBranches = [ + { slug: "a", ratio: 1 }, + { slugG: "b", ratio: 1 }, + ]; + + await Assert.rejects( + ExperimentManager.generateTestIds({ + slug, + branches: invalidBranches, + namespace, + start, + count, + total, + }), + /branches parameter not in expected format/, + "should throw because of invalid format for branches" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js new file mode 100644 index 0000000000..d9b9a16932 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_lifecycle.js @@ -0,0 +1,464 @@ +"use strict"; + +const { Sampling } = ChromeUtils.importESModule( + "resource://gre/modules/components-utils/Sampling.sys.mjs" +); + +async function cleanupStore(store) { + Assert.deepEqual( + store.getAllActiveExperiments(), + [], + "There should be no experiments active." + ); + + Assert.deepEqual( + store.getAllActiveRollouts(), + [], + "There should be no rollouts active" + ); + + // We need to call finalize first to ensure that any pending saves from + // JSONFile.saveSoon overwrite files on disk. + await store._store.finalize(); + await IOUtils.remove(store._store.path); +} + +/** + * onStartup() + * - should set call setExperimentActive for each active experiment + */ +add_task(async function test_onStartup_setExperimentActive_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const experiments = []; + sandbox.stub(manager, "setExperimentActive"); + sandbox.stub(manager.store, "init").resolves(); + sandbox.stub(manager.store, "getAll").returns(experiments); + sandbox + .stub(manager.store, "get") + .callsFake(slug => experiments.find(expt => expt.slug === slug)); + sandbox.stub(manager.store, "set"); + + const active = ["foo", "bar"].map(ExperimentFakes.experiment); + + const inactive = ["baz", "qux"].map(slug => + ExperimentFakes.experiment(slug, { active: false }) + ); + + [...active, ...inactive].forEach(exp => experiments.push(exp)); + + await manager.onStartup(); + + active.forEach(exp => + Assert.equal( + manager.setExperimentActive.calledWith(exp), + true, + `should call setExperimentActive for active experiment: ${exp.slug}` + ) + ); + + inactive.forEach(exp => + Assert.equal( + manager.setExperimentActive.calledWith(exp), + false, + `should not call setExperimentActive for inactive experiment: ${exp.slug}` + ) + ); + + sandbox.restore(); + await cleanupStore(manager.store); +}); + +add_task(async function test_onStartup_setRolloutActive_called() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.stub(manager, "setExperimentActive"); + sandbox.stub(manager.store, "init").resolves(); + + const active = ["foo", "bar"].map(ExperimentFakes.rollout); + sandbox.stub(manager.store, "getAll").returns(active); + sandbox + .stub(manager.store, "get") + .callsFake(slug => active.find(e => e.slug === slug)); + sandbox.stub(manager.store, "set"); + + await manager.onStartup(); + + active.forEach(r => + Assert.equal( + manager.setExperimentActive.calledWith(r), + true, + `should call setExperimentActive for rollout: ${r.slug}` + ) + ); + + sandbox.restore(); + await cleanupStore(manager.store); +}); + +add_task(async function test_startup_unenroll() { + Services.prefs.setBoolPref("app.shield.optoutstudies.enabled", false); + const store = ExperimentFakes.store(); + const sandbox = sinon.createSandbox(); + let recipe = ExperimentFakes.experiment("startup_unenroll", { + experimentType: "unittest", + source: "test", + }); + // Test initializing ExperimentManager with an active + // recipe in the store. If the user has opted out it should + // unenroll. + await store.init(); + let enrollmentPromise = new Promise(resolve => + store.on(`update:${recipe.slug}`, resolve) + ); + store.addEnrollment(recipe); + await enrollmentPromise; + + const manager = ExperimentFakes.manager(store); + const unenrollSpy = sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + Assert.ok( + unenrollSpy.calledOnce, + "Unenrolled from active experiment if user opt out is true" + ); + Assert.ok( + unenrollSpy.calledWith("startup_unenroll", "studies-opt-out"), + "Called unenroll for expected recipe" + ); + + Services.prefs.clearUserPref("app.shield.optoutstudies.enabled"); + + await cleanupStore(manager.store); +}); + +/** + * onRecipe() + * - should add recipe slug to .session[source] + * - should call .enroll() if the recipe hasn't been seen before; + * - should call .update() if the Enrollment already exists in the store; + * - should skip enrollment if recipe.isEnrollmentPaused is true + */ +add_task(async function test_onRecipe_track_slug() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + const fooRecipe = ExperimentFakes.recipe("foo"); + fooRecipe.bucketConfig.start = 0; + fooRecipe.bucketConfig.count = 0; + + await manager.onStartup(); + // The first time a recipe has seen; + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.sessions.get("test").has("foo"), + true, + "should add slug to sessions[test]" + ); + + await cleanupStore(manager.store); +}); + +add_task(async function test_onRecipe_enroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + sandbox.stub(Sampling, "bucketSample").resolves(true); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + const fooRecipe = ExperimentFakes.recipe("foo"); + await manager.onStartup(); + + Assert.deepEqual( + manager.store.getAllActiveExperiments(), + [], + "There should be no active experiments" + ); + + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.enroll.calledWith(fooRecipe), + true, + "should call .enroll() the first time a recipe is seen" + ); + Assert.equal( + manager.store.has("foo"), + true, + "should add recipe to the store" + ); + + manager.unenroll(fooRecipe.slug, "test-cleanup"); + + await cleanupStore(manager.store); +}); + +add_task(async function test_onRecipe_update() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + + const fooRecipe = ExperimentFakes.recipe("foo"); + const experimentUpdate = new Promise(resolve => + manager.store.on(`update:${fooRecipe.slug}`, resolve) + ); + + await manager.onStartup(); + await manager.onRecipe(fooRecipe, "test"); + // onRecipe calls enroll which saves the experiment in the store + // but none of them wait on disk operations to finish + await experimentUpdate; + // Call again after recipe has already been enrolled + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(fooRecipe), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + + manager.unenroll(fooRecipe.slug, "test-cleanup"); + + await cleanupStore(manager.store); +}); + +add_task(async function test_onRecipe_rollout_update() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "unenroll"); + sandbox.spy(manager, "updateEnrollment"); + sandbox.stub(manager, "isInBucketAllocation").resolves(true); + + const fooRecipe = { + ...ExperimentFakes.recipe("foo"), + isRollout: true, + }; + // Rollouts should only have 1 branch + fooRecipe.branches = fooRecipe.branches.slice(0, 1); + + await manager.onStartup(); + await manager.onRecipe(fooRecipe, "test"); + // onRecipe calls enroll which saves the experiment in the store + // but none of them wait on disk operations to finish + // Call again after recipe has already been enrolled + await manager.onRecipe(fooRecipe, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(fooRecipe), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + Assert.ok( + manager.updateEnrollment.alwaysReturned(Promise.resolve(true)), + "updateEnrollment will confirm the enrolled branch still exists in the recipe and exit" + ); + Assert.ok( + manager.unenroll.notCalled, + "Should not call if the branches did not change" + ); + + // We call again but this time we change the branch slug + // Has to be a deep clone otherwise you're changing the + // value found in the experiment store + let recipeClone = Cu.cloneInto(fooRecipe, {}); + recipeClone.branches[0].slug = "control-v2"; + await manager.onRecipe(recipeClone, "test"); + + Assert.equal( + manager.updateEnrollment.calledWith(recipeClone), + true, + "should call .updateEnrollment() if the recipe has already been enrolled" + ); + Assert.ok( + manager.unenroll.called, + "updateEnrollment will unenroll because the branch slug changed" + ); + Assert.ok( + manager.unenroll.calledWith(fooRecipe.slug, "branch-removed"), + "updateEnrollment will unenroll because the branch slug changed" + ); + + await cleanupStore(manager.store); +}); + +add_task(async function test_onRecipe_isEnrollmentPaused() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "enroll"); + sandbox.spy(manager, "updateEnrollment"); + + await manager.onStartup(); + + const pausedRecipe = ExperimentFakes.recipe("xyz", { + isEnrollmentPaused: true, + }); + await manager.onRecipe(pausedRecipe, "test"); + Assert.equal( + manager.enroll.calledWith(pausedRecipe), + false, + "should skip enrollment for recipes that are paused" + ); + Assert.equal( + manager.store.has("xyz"), + false, + "should not add recipe to the store" + ); + + const fooRecipe = ExperimentFakes.recipe("foo"); + const updatedRecipe = ExperimentFakes.recipe("foo", { + isEnrollmentPaused: true, + }); + await manager.enroll(fooRecipe, "test"); + await manager.onRecipe(updatedRecipe, "test"); + Assert.equal( + manager.updateEnrollment.calledWith(updatedRecipe), + true, + "should still update existing recipes, even if enrollment is paused" + ); + + manager.unenroll(fooRecipe.slug); + await cleanupStore(manager.store); +}); + +/** + * onFinalize() + * - should unenroll experiments that weren't seen in the current session + */ + +add_task(async function test_onFinalize_unenroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + // Add an experiment to the store without calling .onRecipe + // This simulates an enrollment having happened in the past. + let recipe0 = ExperimentFakes.experiment("foo", { + experimentType: "unittest", + userFacingName: "foo", + userFacingDescription: "foo", + lastSeen: new Date().toJSON(), + source: "test", + }); + await manager.store.addEnrollment(recipe0); + + const recipe1 = ExperimentFakes.recipe("bar"); + // Unique features to prevent overlap + recipe1.branches[0].features[0].featureId = "red"; + recipe1.branches[1].features[0].featureId = "red"; + await manager.onRecipe(recipe1, "test"); + const recipe2 = ExperimentFakes.recipe("baz"); + recipe2.branches[0].features[0].featureId = "green"; + recipe2.branches[1].features[0].featureId = "green"; + await manager.onRecipe(recipe2, "test"); + + // Finalize + manager.onFinalize("test"); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("foo", "recipe-not-seen"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); + Assert.equal( + manager.sessions.has("test"), + false, + "should clear sessions[test]" + ); + + manager.unenroll(recipe1.slug); + manager.unenroll(recipe2.slug); + await cleanupStore(manager.store); +}); + +add_task(async function test_onFinalize_unenroll_mismatch() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + // Add an experiment to the store without calling .onRecipe + // This simulates an enrollment having happened in the past. + let recipe0 = ExperimentFakes.experiment("foo", { + experimentType: "unittest", + userFacingName: "foo", + userFacingDescription: "foo", + lastSeen: new Date().toJSON(), + source: "test", + }); + await manager.store.addEnrollment(recipe0); + + const recipe1 = ExperimentFakes.recipe("bar"); + // Unique features to prevent overlap + recipe1.branches[0].features[0].featureId = "red"; + recipe1.branches[1].features[0].featureId = "red"; + await manager.onRecipe(recipe1, "test"); + const recipe2 = ExperimentFakes.recipe("baz"); + recipe2.branches[0].features[0].featureId = "green"; + recipe2.branches[1].features[0].featureId = "green"; + await manager.onRecipe(recipe2, "test"); + + // Finalize + manager.onFinalize("test", { recipeMismatches: [recipe0.slug] }); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("foo", "targeting-mismatch"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); + Assert.equal( + manager.sessions.has("test"), + false, + "should clear sessions[test]" + ); + + manager.unenroll(recipe1.slug); + manager.unenroll(recipe2.slug); + await cleanupStore(manager.store); +}); + +add_task(async function test_onFinalize_rollout_unenroll() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + sandbox.spy(manager, "unenroll"); + + await manager.onStartup(); + + let rollout = ExperimentFakes.rollout("rollout"); + await manager.store.addEnrollment(rollout); + + manager.onFinalize("NimbusTestUtils"); + + Assert.equal( + manager.unenroll.callCount, + 1, + "should only call unenroll for the unseen recipe" + ); + Assert.equal( + manager.unenroll.calledWith("rollout", "recipe-not-seen"), + true, + "should unenroll a experiment whose recipe wasn't seen in the current session" + ); + + await cleanupStore(manager.store); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js new file mode 100644 index 0000000000..97d4bfd0c7 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_prefs.js @@ -0,0 +1,3118 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { _ExperimentFeature: ExperimentFeature, NimbusFeatures } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); + +const { PrefUtils } = ChromeUtils.importESModule( + "resource://normandy/lib/PrefUtils.sys.mjs" +); + +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.importESModule( + "resource://normandy/lib/TelemetryEvents.sys.mjs" +); + +/** + * Pick a single entry from an object and return a new object containing only + * that entry. + * + * @param {object} obj The object to copy the value from. + * @param {string} key The key whose value is to be copied. + * + * @returns {object} An object with the property `key` set to `obj[key]`. + */ +function pick(obj, key) { + return { [key]: obj[key] }; +} + +const PREF_FEATURES = [ + new ExperimentFeature("test-set-pref", { + description: "Test feature that sets a pref on the default branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup: false, + variables: { + foo: { + type: "string", + description: "Test variable", + setPref: "nimbus.test-only.foo", + }, + }, + }), + new ExperimentFeature("test-set-user-pref", { + description: "Test feature that sets a pref on the user branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup: true, + variables: { + bar: { + type: "string", + description: "Test variable", + setPref: "nimbus.test-only.bar", + }, + }, + }), +]; + +const DEFAULT_VALUE = "default-value"; +const USER_VALUE = "user-value"; +const EXPERIMENT_VALUE = "experiment-value"; +const ROLLOUT_VALUE = "rollout-value"; +const OVERWRITE_VALUE = "overwrite-value"; + +const USER = "user"; +const DEFAULT = "default"; +const ROLLOUT = "rollout"; +const EXPERIMENT = "experiment"; + +const PREFS = { + [DEFAULT]: "nimbus.test-only.foo", + [USER]: "nimbus.test-only.bar", +}; + +const FEATURE_IDS = { + [DEFAULT]: "test-set-pref", + [USER]: "test-set-user-pref", +}; + +const CONFIGS = { + [DEFAULT]: { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: { + foo: ROLLOUT_VALUE, + }, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: { + foo: EXPERIMENT_VALUE, + }, + }, + }, + [USER]: { + [ROLLOUT]: { + featureId: FEATURE_IDS[USER], + value: { + bar: ROLLOUT_VALUE, + }, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[USER], + value: { + bar: EXPERIMENT_VALUE, + }, + }, + }, +}; + +/** + * Set the given pref values on their respective branches (if they are not + * null). + */ +function setPrefs(pref, { defaultBranchValue = null, userBranchValue = null }) { + if (defaultBranchValue !== null) { + Services.prefs + .getDefaultBranch(null) + .setStringPref(pref, defaultBranchValue); + } + + if (userBranchValue !== null) { + Services.prefs.setStringPref(pref, userBranchValue); + } +} + +function assertExpectedPrefValues(pref, branch, expected, visible, msg) { + info(`Assert pref ${pref} on branch ${branch} matches ${expected} ${msg}`); + const hasBranchValue = expected !== null; + const hasVisibleValue = visible !== null; + + function hasValueMsg(hasValue) { + return `Expected pref "${pref}" on the ${branch} branch to${ + hasValue ? " " : " not " + }have a value ${msg}`; + } + + let prefBranch; + switch (branch) { + case USER: + Assert.equal( + Services.prefs.prefHasUserValue(pref), + hasBranchValue, + hasValueMsg(hasBranchValue) + ); + prefBranch = Services.prefs; + break; + + case DEFAULT: + Assert.equal( + Services.prefs.prefHasDefaultValue(pref), + hasBranchValue, + hasValueMsg(hasBranchValue) + ); + prefBranch = Services.prefs.getDefaultBranch(null); + break; + + default: + Assert.ok(false, "invalid pref branch"); + } + + if (hasBranchValue) { + Assert.equal( + prefBranch.getStringPref(pref), + expected, + `Expected pref "${pref} on the ${branch} branch to be ${JSON.stringify( + expected + )} ${msg}` + ); + } + + if (hasVisibleValue) { + Assert.equal( + Services.prefs.getStringPref(pref), + visible, + `Expected pref "${pref}" to be ${JSON.stringify(visible)} ${msg}` + ); + } else { + Assert.ok( + !Services.prefs.prefHasUserValue(pref) && + !Services.prefs.prefHasDefaultValue(pref), + `Expected pref "${pref} to not be set ${msg}` + ); + } +} + +/** + * Assert the manager has no active pref observers. + */ +function assertNoObservers(manager) { + Assert.equal( + manager._prefs.size, + 0, + "There should be no active pref observers" + ); + Assert.equal( + manager._prefsBySlug.size, + 0, + "There should be no active pref observers" + ); +} + +/** + * Remove all pref observers on the given ExperimentManager. + */ +function removePrefObservers(manager) { + for (const [name, entry] of manager._prefs.entries()) { + Services.prefs.removeObserver(name, entry.observer); + } + + manager._prefs.clear(); + manager._prefsBySlug.clear(); +} + +add_setup(function setup() { + do_get_profile(); + Services.fog.initializeFOG(); + + const cleanupFeatures = ExperimentTestUtils.addTestFeatures(...PREF_FEATURES); + registerCleanupFunction(cleanupFeatures); +}); + +add_task(async function test_enroll_setPref_rolloutsAndExperiments() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + await assertEmptyStore(store); + + /** + * Test that prefs are set correctly before and after enrollment and + * unenrollment. + * + * @param {object} options + * @param {string} options.pref + * The name of the pref. + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {string[]} options.enrollOrder + * The order to do the enrollments. Must only contain + * "experiment" and "rollout" as values. + * + * @param {string[]} options.unenrollOrder + * The order to undo the enrollments. Must only contain + * "experiment" and "rollout" as values. + * + * @param {(string|null)[]} options.expectedValues + * The expected values of the preft on the given branch at each point: + * + * * before enrollment; + * * one entry each each after enrolling in `options.enrollOrder[i]`; and + * * one entry each each after unenrolling in `options.unenrollOrder[i]`. + * + * A value of null indicates that the pref should not be set on that + * branch. + * + * @param {(string|null)[]?} options.visibleValues + * The expected values returned by + * `Services.prefs.getStringPref` (i.e., the user branch if set, + * falling back to the default branch if not), in the same + * order as |options.expectedValues|. + * + * If undefined, then it will default `options.expectedValues`. + */ + async function doBaseTest({ + pref, + branch, + configs, + userBranchValue = undefined, + defaultBranchValue = undefined, + enrollOrder, + unenrollOrder, + expectedValues, + visibleValues = undefined, + }) { + if (visibleValues === undefined) { + visibleValues = expectedValues; + } + + const cleanupFns = {}; + let i = 0; + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + "before enrollment" + ); + i++; + + for (const enrollmentKind of enrollOrder) { + const isRollout = enrollmentKind === ROLLOUT; + cleanupFns[enrollmentKind] = + await ExperimentFakes.enrollWithFeatureConfig(configs[enrollmentKind], { + manager, + isRollout, + }); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} enrollment` + ); + i++; + } + + for (const enrollmentKind of unenrollOrder) { + await cleanupFns[enrollmentKind](); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} unenrollment` + ); + i++; + } + + await assertEmptyStore(store); + Services.prefs.deleteBranch(pref); + } + + // Tests for a feature that would set a pref on the default branch, but the variable is omitted. + { + const branch = DEFAULT; + const pref = PREFS[branch]; + + const configs = { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + }; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in a rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in an experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + // Test for a feature that would set a pref on the user branch, but the variable is omitted. + { + const branch = USER; + const pref = PREFS[branch]; + + const configs = { + [ROLLOUT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + [EXPERIMENT]: { + featureId: FEATURE_IDS[DEFAULT], + value: {}, + }, + }; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in a rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, null, null], + visibleValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in an experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, null, null], + visibleValues: [DEFAULT_VALUE, DEFAULT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + // Tests for a feature that sets prefs on the default branch. + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Enroll in rollout then experiment; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in rollout then experiment; unenroll in same order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in same order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + ROLLOUT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + DEFAULT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, + ], + visibleValues: [ + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + USER_VALUE, + ], + }); + } + + // Tests for a feature that sets prefs on the user branch. + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout then unenroll. + await doTest({ + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [null, ROLLOUT_VALUE, null], + visibleValues: [DEFAULT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + enrollOrder: [ROLLOUT], + unenrollOrder: [ROLLOUT], + expectedValues: [USER_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + // Enroll in experiment then unenroll. + await doTest({ + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, null], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [null, EXPERIMENT_VALUE, null], + visibleValues: [DEFAULT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + enrollOrder: [EXPERIMENT], + unenrollOrder: [EXPERIMENT], + expectedValues: [USER_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + // Enroll in rollout then experiment; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in rollout then experiment; unenroll in same order. + await doTest({ + configs, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [ROLLOUT, EXPERIMENT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + ROLLOUT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in reverse order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + USER_VALUE, + ], + }); + + // Enroll in experiment then rollout; unenroll in same order. + await doTest({ + configs, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + null, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + null, + ], + visibleValues: [ + DEFAULT_VALUE, // User branch falls back to default branch. + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + DEFAULT_VALUE, // User branch falls back to default branch. + ], + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + enrollOrder: [EXPERIMENT, ROLLOUT], + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [ + USER_VALUE, + EXPERIMENT_VALUE, + EXPERIMENT_VALUE, + ROLLOUT_VALUE, + USER_VALUE, + ], + }); + } + + await assertEmptyStore(store, { cleanup: true }); +}); + +add_task(async function test_restorePrefs_experimentAndRollout() { + /** + * Test that prefs are set correctly after restoring from a saved store file + * and unrnollment. + * + * This test sets up some enrollments and saves them to disk. + * + * A browser restart will be simulated by creating a new ExperimentStore and + * ExperimentManager to restore the saved enrollments. + * + * @param {object} options + * @param {string} options.pref + * The name of the pref. + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {string[]} options.unenrollOrder + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {(string|null)[]} options.expectedValues + * The expected values of the preft on the given branch at each point: + * + * * before enrollment; + * * one entry each each after enrolling in `options.enrollOrder[i]`; and + * * one entry each each after unenrolling in `options.unenrollOrder[i]`. + * + * A value of null indicates that the pref should not be set on that + * branch. + * + * @param {(string|null)[]?} options.visibleValues + * The expected values returned by + * Services.prefs.getStringPref (i.e., the user branch if set, + * falling back to the default branch if not), in the same + * order as `options.expectedValues`. + * + * If undefined, then it will default to `options.expectedValues`. + */ + async function doBaseTest({ + featureId, + pref, + branch, + configs, + defaultBranchValue = null, + userBranchValue = null, + unenrollOrder, + expectedValues, + visibleValues = undefined, + }) { + if (![USER, DEFAULT].includes(branch)) { + Assert.ok(false, `invalid branch ${branch}`); + } + + if (visibleValues === undefined) { + visibleValues = expectedValues; + } + + // Set the initial conditions. + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + // Enroll in some experiments and save the state to disk. + { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + await assertEmptyStore(store); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout: enrollmentKind === ROLLOUT, + }); + } + + store._store.saveSoon(); + await store._store.finalize(); + + // User branch prefs persist through restart, so we only want to delete + // the prefs if we changed the default branch. + if (branch === "default") { + Services.prefs.deleteBranch(pref); + } + + removePrefObservers(manager); + assertNoObservers(manager); + } + + // Restore the default branch value as it was before "restarting". + setPrefs(pref, { defaultBranchValue }); + // If this is not a user branch pref, restore the user branch value. User + // branch values persist through restart, so we don't want to overwrite a + // value we just set. + if (branch === "default") { + setPrefs(pref, { userBranchValue }); + } + + const sandbox = sinon.createSandbox(); + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + const setPrefSpy = sandbox.spy(PrefUtils, "setPref"); + + await manager.onStartup(); + + if (branch === DEFAULT) { + Assert.ok(setPrefSpy.calledOnce, "Should have called setPref once total"); + Assert.ok( + setPrefSpy.calledOnceWith(pref, expectedValues[0], { branch }), + `Should have only called setPref with correct args (called with: ${JSON.stringify( + setPrefSpy.getCall(0).args + )}) expected ${JSON.stringify([pref, expectedValues[0], { branch }])})` + ); + } else if (branch === USER) { + Assert.ok( + setPrefSpy.notCalled, + "Should have not called setPref for a user branch pref" + ); + } + + assertExpectedPrefValues( + pref, + branch, + expectedValues[0], + visibleValues[0], + "after manager startup" + ); + + const slugs = { + [ROLLOUT]: store.getRolloutForFeature(featureId)?.slug, + [EXPERIMENT]: store.getExperimentForFeature(featureId)?.slug, + }; + + let i = 1; + for (const enrollmentKind of unenrollOrder) { + manager.unenroll(slugs[enrollmentKind]); + + assertExpectedPrefValues( + pref, + branch, + expectedValues[i], + visibleValues[i], + `after ${enrollmentKind} unenrollment` + ); + + i++; + } + + for (const enrollmentKind of unenrollOrder) { + // The unenrollment happened normally, not through a cleanup function. + store._deleteForTests(slugs[enrollmentKind]); + } + + assertNoObservers(manager); + await assertEmptyStore(store, { cleanup: true }); + + Services.prefs.deleteBranch(pref); + sandbox.restore(); + } + + { + const branch = DEFAULT; + const featureId = FEATURE_IDS[branch]; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ featureId, pref, branch, ...args }); + + // Tests with no prefs set beforehand. + await doTest({ + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, ROLLOUT_VALUE], + }); + + await doTest({ + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + }); + + // Tests where the default branch is set beforehand. + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + // Tests where the user branch is set beforehand. + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, ROLLOUT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, EXPERIMENT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + // Tests with both branches set beforehand + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + visibleValues: [USER_VALUE, USER_VALUE, USER_VALUE], + }); + } + + { + const branch = USER; + const featureId = FEATURE_IDS[branch]; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => + doBaseTest({ featureId, pref, branch, configs, ...args }); + + // Tests with no prefs set beforehand. + await doTest({ + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, null], + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, null], + }); + + await doTest({ + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, null], + }); + + await doTest({ + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, null], + }); + + // Tests with the default branch set beforehand. + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, null], + visibleValues: [ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, DEFAULT_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, null], + visibleValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, DEFAULT_VALUE], + }); + + // Tests with the user branch set beforehand. + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + + // Tests with both branches set beforehand + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, EXPERIMENT), + unenrollOrder: [EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs: pick(configs, ROLLOUT), + unenrollOrder: [ROLLOUT], + expectedValues: [ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [EXPERIMENT, ROLLOUT], + expectedValues: [EXPERIMENT_VALUE, ROLLOUT_VALUE, USER_VALUE], + }); + + await doTest({ + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + configs, + unenrollOrder: [ROLLOUT, EXPERIMENT], + expectedValues: [EXPERIMENT_VALUE, EXPERIMENT_VALUE, USER_VALUE], + }); + } +}); + +add_task(async function test_prefChange() { + /** + * Test that pref tampering causes unenrollment. + * + * This test sets up some enrollments and then modifies the given `pref` on a + * branch specified by `setBranch` and checks that unenrollments happen as + * appropriate. + * + * @param {object} options + * + * @param {string} options.pref + * The name of the pref. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string} options.setBranch + * The branch that the test will set (either "user" or "default"). + * + * @param {string[]} options.expectedEnrollments + * The list of enrollment kinds (e.g., "rollout" or "experiment") that + * should be active after setting the pref on the requested branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after setting the pref on + * the requested branch. + * + * A value of null indicates that the pref should not be set on the + * default branch. + * + * @param {string} options.expectedUser + * The expected value of the user branch after setting the pref on the + * requested branch. + * + * A value of null indicates that the pref should not be set on the + * user branch. + */ + async function doBaseTest({ + pref, + defaultBranchValue = null, + userBranchValue = null, + configs, + setBranch, + expectedEnrollments = [], + expectedDefault = null, + expectedUser = null, + }) { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + const cleanup = {}; + const slugs = {}; + + await manager.onStartup(); + + await assertEmptyStore(store); + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + info(`Enrolling in ${Array.from(Object.keys(configs)).join(", ")} ...`); + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + cleanup[enrollmentKind] = await ExperimentFakes.enrollWithFeatureConfig( + config, + { + manager, + isRollout, + } + ); + + const enrollments = isRollout + ? store.getAllActiveRollouts() + : store.getAllActiveExperiments(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + info( + `Overwriting ${pref} with "${OVERWRITE_VALUE}" on ${setBranch} branch` + ); + + PrefUtils.setPref(pref, OVERWRITE_VALUE, { branch: setBranch }); + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + if (expectedUser === null) { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getStringPref(pref), + expectedUser, + `Expected the value of ${pref} on the user branch to match the expected value` + ); + } + + for (const enrollmentKind of expectedEnrollments) { + const enrollment = store.get(slugs[enrollmentKind]); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(enrollment.active, "It should still be active"); + } + + for (const enrollmentKind of Object.keys(configs)) { + if (!expectedEnrollments.includes(enrollmentKind)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(!enrollment.active, "It should not be active"); + + store._deleteForTests(slug); + } + } + + for (const enrollmentKind of expectedEnrollments) { + await cleanup[enrollmentKind](); + } + + assertNoObservers(manager); + await assertEmptyStore(store, { cleanup: true }); + + Services.prefs.deleteBranch(pref); + } + + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enrolled in rollout, set default branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enrolled in rollout, set user branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: USER, + expectedDefault: ROLLOUT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: ROLLOUT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in experiment, set default branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enrolled in experiment, set user branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enroll in both, set default branch. + await doTest({ + configs, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedDefault: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: USER_VALUE, + }); + + // Enroll in both, set user branch. + await doTest({ + configs, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: EXPERIMENT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + } + + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enrolled in rollout, set default branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: ROLLOUT_VALUE, + }); + + // Enrolled in rollout, set user branch. + await doTest({ + configs: pick(configs, ROLLOUT), + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in experiment, set default branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + // Enrolled in experiment, set user branch. + await doTest({ + configs: pick(configs, EXPERIMENT), + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + // Enrolled in both, set default branch. + await doTest({ + configs, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: DEFAULT, + expectedEnrollments: [EXPERIMENT, ROLLOUT], + expectedDefault: OVERWRITE_VALUE, + expectedUser: EXPERIMENT_VALUE, + }); + + // Enrolled in both, set user branch. + await doTest({ + configs, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedUser: OVERWRITE_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + setBranch: USER, + expectedDefault: DEFAULT_VALUE, + expectedUser: OVERWRITE_VALUE, + }); + } +}); + +add_task(async function test_deleteBranch() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + await assertEmptyStore(store); + + const cleanup = []; + cleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[USER][EXPERIMENT], { + manager, + }), + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[USER][ROLLOUT], { + manager, + isRollout: true, + }), + await ExperimentFakes.enrollWithFeatureConfig( + CONFIGS[DEFAULT][EXPERIMENT], + { manager } + ), + await ExperimentFakes.enrollWithFeatureConfig(CONFIGS[DEFAULT][ROLLOUT], { + manager, + isRollout: true, + }) + ); + + Services.prefs.deleteBranch(PREFS[USER]); + Services.prefs.deleteBranch(PREFS[DEFAULT]); + + // deleteBranch does not trigger pref observers! + Assert.equal( + store.getAll().length, + 4, + "nsIPrefBranch::deleteBranch does not trigger unenrollment" + ); + + for (const cleanupFn of cleanup) { + await cleanupFn(); + } + + assertNoObservers(manager); + await assertEmptyStore(store, { cleanup: true }); +}); + +add_task(async function test_clearUserPref() { + /** + * Test that nsIPrefBranch::clearUserPref() correctly interacts with pref + * tampering logic. + * + * This test sets up some enrollments and then clears the pref specified and + * checks that unenrollments happen as * appropriate. + * + * @param {object} options + * + * @param {string} options.pref + * The name of the pref. + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {boolean} options.expectedEnrolled + * Whether or not the enrollments defined in `configs` should still be + * active after clearing the user branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after clearing the user branch. + * + * A value of null indicates that the pref should not be set on the default + * branch. + */ + async function doBaseTest({ + pref, + defaultBranchValue = null, + userBranchValue = null, + configs, + expectedEnrolled, + expectedDefault = null, + }) { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + await assertEmptyStore(store); + + const cleanup = []; + const slugs = {}; + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + cleanup.push( + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout, + }) + ); + + const enrollments = isRollout + ? store.getAllActiveRollouts() + : store.getAllActiveExperiments(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + Services.prefs.clearUserPref(pref); + + for (const enrollmentKind of Object.keys(configs)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + + if (expectedEnrolled) { + Assert.ok(enrollment.active, "It should be active"); + } else { + Assert.ok(!enrollment.active, "It should not be active"); + } + } + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref}` + ); + + if (expectedEnrolled) { + for (const cleanupFn of Object.values(cleanup)) { + await cleanupFn(); + } + } else { + for (const slug of Object.values(slugs)) { + store._deleteForTests(slug); + } + } + + assertNoObservers(manager); + await assertEmptyStore(store, { cleanup: true }); + + Services.prefs.deleteBranch(pref); + } + + { + const branch = DEFAULT; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + const doTest = args => doBaseTest({ pref, branch, ...args }); + + // Enroll in rollout. + await doTest({ + configs: pick(configs, ROLLOUT), + expectedEnrolled: true, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: ROLLOUT_VALUE, + }); + + await doTest({ + configs: pick(configs, ROLLOUT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + + // Enroll in experiment. + await doTest({ + configs: pick(configs, EXPERIMENT), + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs: pick(configs, EXPERIMENT), + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + + // Enroll in both. + await doTest({ + configs, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + expectedEnrolled: true, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: EXPERIMENT_VALUE, + }); + + await doTest({ + configs, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedEnrolled: false, + expectedDefault: DEFAULT_VALUE, + }); + } + + { + const branch = USER; + const pref = PREFS[branch]; + const configs = CONFIGS[branch]; + const doTest = args => + doBaseTest({ pref, branch, expectedEnrolled: false, ...args }); + + // Because this pref is set on the user branch, clearing the user branch has + // the same effect for every suite of configs. + for (const selectedConfig of [ + pick(configs, ROLLOUT), + pick(configs, EXPERIMENT), + configs, + ]) { + await doTest({ + configs: selectedConfig, + }); + + await doTest({ + configs: selectedConfig, + defaultBranchValue: DEFAULT_VALUE, + expectedDefault: DEFAULT_VALUE, + }); + + await doTest({ + configs: selectedConfig, + userBranchValue: USER_VALUE, + }); + + await doTest({ + configs: selectedConfig, + defaultBranchValue: DEFAULT_VALUE, + userBranchValue: USER_VALUE, + expectedDefault: DEFAULT_VALUE, + }); + } + } +}); + +// Test that unenrollment doesn't happen if a pref changes but it wasn't set. +add_task(async function test_prefChanged_noPrefSet() { + const featureId = "test-set-pref-2"; + const pref = "nimbus.test-only.baz"; + + function featureFactory(isEarlyStartup) { + return new ExperimentFeature(featureId, { + description: "Test feature that sets a pref", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup, + variables: { + baz: { + type: "string", + description: "Test variable", + setPref: pref, + }, + qux: { + type: "string", + description: "Test variable", + }, + }, + }); + } + + const config = { + featureId, + value: { + qux: "qux", + }, + }; + + for (const isEarlyStartup of [true, false]) { + const feature = featureFactory(isEarlyStartup); + const cleanupFeature = ExperimentTestUtils.addTestFeatures(feature); + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + await manager.onStartup(); + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const isRollout of [true, false]) { + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + const doEnrollmentCleanup = + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout, + }); + + PrefUtils.setPref(pref, OVERWRITE_VALUE, { branch }); + + const enrollments = await store.getAll(); + Assert.equal( + enrollments.length, + 1, + "There should be one enrollment" + ); + Assert.ok(enrollments[0].active, "The enrollment should be active"); + + Assert.equal( + PrefUtils.getPref(pref, { branch }), + OVERWRITE_VALUE, + `The value of ${pref} on the ${branch} branch should be the expected value` + ); + + if (branch === USER) { + if (defaultBranchValue) { + Assert.equal( + PrefUtils.getPref(pref, { branch: DEFAULT }), + defaultBranchValue, + "The default branch should have the expected value" + ); + } else { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + "The default branch should not have a value" + ); + } + } else if (userBranchValue) { + Assert.equal( + PrefUtils.getPref(pref, { branch: USER }), + userBranchValue, + "The user branch should have the expected value" + ); + } else { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + "The user branch should not have a value" + ); + } + + assertNoObservers(manager); + + await doEnrollmentCleanup(); + await assertEmptyStore(store); + + Services.prefs.deleteBranch(pref); + } + } + } + } + + cleanupFeature(); + await assertEmptyStore(store, { cleanup: true }); + } +}); + +add_task(async function test_restorePrefs_manifestChanged() { + TelemetryEvents.init(); + + const LEGACY_FILTER = { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + }; + + const BOGUS_PREF = "nimbus.test-only.bogus"; + + const REMOVE_FEATURE = "remove-feature"; + const REMOVE_PREF_VARIABLE = "remove-pref-variable"; + const REMOVE_OTHER_VARIABLE = "remove-other-variable"; + const REMOVE_SETPREF = "remove-setpref"; + const CHANGE_SETPREF = "change-setpref"; + + const OPERATIONS = [ + REMOVE_FEATURE, + REMOVE_PREF_VARIABLE, + REMOVE_OTHER_VARIABLE, + REMOVE_SETPREF, + CHANGE_SETPREF, + ]; + + const REASONS = { + [REMOVE_FEATURE]: "invalid-feature", + [REMOVE_PREF_VARIABLE]: "pref-variable-missing", + [REMOVE_SETPREF]: "pref-variable-no-longer", + [CHANGE_SETPREF]: "pref-variable-changed", + }; + + const featureId = "test-set-pref-temp"; + const pref = "nimbus.test-only.baz"; + + // Return a new object so we can modified the returned value. + function featureFactory(isEarlyStartup) { + return new ExperimentFeature(featureId, { + description: "Test feature that sets a pref on the default branch.", + owner: "test@test.test", + hasExposure: false, + isEarlyStartup, + variables: { + baz: { + type: "string", + description: "Test variable", + setPref: pref, + }, + qux: { + type: "string", + description: "Test variable", + }, + }, + }); + } + + /* + * Test that enrollments end when the manifest is sufficiently changed and + * that the appropriate telemetry is submitted. + * + * This test sets up some enrollments and saves them to disk. Then the + * manifest will be modified according to `operation`. + * + * A browser restart will be simulated by creating a new ExperimentStore and + * ExperimentManager to restore the saved enrollments. + * + * @param {object} options + * + * @param {string} options.branch + * The name of the pref branch ("user" or "default"). + * + * @param {string?} options.defaultBranchValue + * An optional value to set for the pref on the default branch + * before the first enrollment. + * + * @param {string?} options.userBranchValue + * An optional value to set for the pref on the user branch + * before the first enrollment. + * + * @param {object} options.configs + * The rollout and experiment feature configurations. + * + * @param {string} options.operation + * The operation that will be performed on the manifest. + * + * See `OPERATIONS` above. + * + * @param {string[]} options.expectedEnrollments + * The list of enrollment kinds (e.g., "rollout" or "experiment") that + * should be active after setting the pref on the requested branch. + * + * @param {string} options.expectedDefault + * The expected value of the default branch after restoring enrollments. + * + * A value of null indicates that the pref should not be set on the + * default branch. + * + * @param {string} options.expectedUser + * The expected value of the user branch after restoring enrollments. + * + * A value of null indicates that the pref should not be set on the + * user branch. + */ + async function doBaseTest({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + expectedEnrollments = [], + expectedDefault = null, + expectedUser = null, + }) { + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); + + const feature = featureFactory(branch === USER); + const cleanupFeatures = ExperimentTestUtils.addTestFeatures(feature); + + setPrefs(pref, { defaultBranchValue, userBranchValue }); + + const slugs = {}; + let userPref = null; + + // Enroll in some experiments and save the state to disk. + { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + await assertEmptyStore(store); + + for (const [enrollmentKind, config] of Object.entries(configs)) { + const isRollout = enrollmentKind === ROLLOUT; + await ExperimentFakes.enrollWithFeatureConfig(config, { + manager, + isRollout, + }); + + const enrollments = isRollout + ? store.getAllActiveRollouts() + : store.getAllActiveExperiments(); + + Assert.equal( + enrollments.length, + 1, + `Expected one ${enrollmentKind} enrollment` + ); + slugs[enrollmentKind] = enrollments[0].slug; + } + + store._store.saveSoon(); + await store._store.finalize(); + + // User branch prefs persist through restart, so we only want to delete + // the prefs if we changed the default branch. + if (branch === "user") { + userPref = PrefUtils.getPref(pref, { branch }); + } + + Services.prefs.deleteBranch(pref); + + removePrefObservers(manager); + assertNoObservers(manager); + } + + // Restore the default branch value as it was before "restarting". + setPrefs(pref, { + defaultBranchValue, + userBranchValue: userPref ?? userBranchValue, + }); + + // Mangle the manifest. + switch (operation) { + case REMOVE_FEATURE: + cleanupFeatures(); + break; + + case REMOVE_PREF_VARIABLE: + delete NimbusFeatures[featureId].manifest.variables.baz; + break; + + case REMOVE_OTHER_VARIABLE: + delete NimbusFeatures[featureId].manifest.variables.qux; + break; + + case REMOVE_SETPREF: + delete NimbusFeatures[featureId].manifest.variables.baz.setPref; + break; + + case CHANGE_SETPREF: + NimbusFeatures[featureId].manifest.variables.baz.setPref = BOGUS_PREF; + break; + + default: + Assert.ok(false, "invalid operation"); + } + + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + + await manager.onStartup(); + + for (const enrollmentKind of expectedEnrollments) { + const enrollment = store.get(slugs[enrollmentKind]); + + Assert.ok( + enrollment !== null, + `An experiment of kind ${enrollmentKind} should exist` + ); + Assert.ok(enrollment.active, "It should still be active"); + } + + if (expectedDefault === null) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(pref), + `Expected the default branch not to be set for ${pref} value: ${PrefUtils.getPref( + pref, + { branch: "default" } + )}` + ); + } else { + Assert.equal( + Services.prefs.getDefaultBranch(null).getStringPref(pref), + expectedDefault, + `Expected the value of ${pref} on the default branch to match the expected value` + ); + } + + if (expectedUser === null) { + Assert.ok( + !Services.prefs.prefHasUserValue(pref), + `Expected the user branch not to be set for ${pref} value: ${PrefUtils.getPref( + pref, + { branch: "user" } + )}` + ); + } else { + Assert.equal( + Services.prefs.getStringPref(pref), + expectedUser, + `Expected the value of ${pref} on the user branch to match the expected value` + ); + } + + if (operation === CHANGE_SETPREF) { + Assert.ok( + !Services.prefs.prefHasDefaultValue(BOGUS_PREF), + "The new pref should not have a value on the default branch" + ); + Assert.ok( + !Services.prefs.prefHasUserValue(BOGUS_PREF), + "The new pref should not have a value on the user branch" + ); + } + + for (const enrollmentKind of Object.keys(configs)) { + if (!expectedEnrollments.includes(enrollmentKind)) { + const slug = slugs[enrollmentKind]; + const enrollment = store.get(slug); + + Assert.ok( + enrollment !== null, + `An enrollment of kind ${enrollmentKind} should exist` + ); + Assert.ok(!enrollment.active, "It should not be active"); + + store._deleteForTests(slug); + } + } + + const gleanEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + if (expectedEnrollments.length === 0) { + const expectedEvents = [EXPERIMENT, ROLLOUT] + .filter(enrollmentKind => Object.hasOwn(slugs, enrollmentKind)) + .map(enrollmentKind => ({ + reason: REASONS[operation], + experiment: slugs[enrollmentKind], + })); + + // Extract only the values we care about. + const processedEvents = gleanEvents.map(event => ({ + reason: event.extra.reason, + experiment: event.extra.experiment, + })); + + Assert.deepEqual( + processedEvents, + expectedEvents, + "Glean should have the expected unenrollment events" + ); + + const expectedLegacyEvents = expectedEvents.map(extra => ({ + value: extra.experiment, + extra: pick(extra, "reason"), + })); + + TelemetryTestUtils.assertEvents(expectedLegacyEvents, LEGACY_FILTER); + } else { + Assert.equal( + gleanEvents, + undefined, + "Glean should have no unenrollment events" + ); + + TelemetryTestUtils.assertEvents([], LEGACY_FILTER); + } + + for (const enrollmentKind of expectedEnrollments) { + const slug = slugs[enrollmentKind]; + manager.unenroll(slug); + store._deleteForTests(slug); + } + + await assertEmptyStore(store, { cleanup: true }); + + assertNoObservers(manager); + Services.prefs.deleteBranch(pref); + + if (operation !== REMOVE_FEATURE) { + // If we try to remove the feature twice, we will throw an exception. + cleanupFeatures(); + } + } + + // Test only qux set. These tests should not cause any unenrollments. + { + const quxConfigs = { + [EXPERIMENT]: { + featureId, + value: { + qux: EXPERIMENT_VALUE, + }, + }, + [ROLLOUT]: { + featureId, + value: { + qux: ROLLOUT_VALUE, + }, + }, + }; + + const doTest = ({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + }) => + doBaseTest({ + branch, + configs, + defaultBranchValue, + userBranchValue, + operation, + expectedEnrollments: Object.keys(configs), + expectedDefault: defaultBranchValue, + expectedUser: userBranchValue, + }); + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const specifiedConfigs of [ + pick(quxConfigs, ROLLOUT), + pick(quxConfigs, EXPERIMENT), + quxConfigs, + ]) { + for (const operation of OPERATIONS) { + await doTest({ + branch, + defaultBranchValue, + userBranchValue, + configs: specifiedConfigs, + operation, + }); + } + } + } + } + } + } + + // Test only baz set. All operations except REMOVE_OTHER_VARIABLE will trigger + // unenrollment. + { + const bazConfigs = { + [EXPERIMENT]: { + featureId, + value: { + baz: EXPERIMENT_VALUE, + }, + }, + [ROLLOUT]: { + featureId, + value: { + baz: ROLLOUT_VALUE, + }, + }, + }; + + const doTest = ({ + branch, + defaultBranchValue = null, + userBranchValue = null, + configs, + operation, + }) => { + const expectedEnrollments = + operation === REMOVE_OTHER_VARIABLE ? Object.keys(configs) : []; + + function expectedPref(forBranch, originalValue) { + if (forBranch === branch) { + if (expectedEnrollments.includes(EXPERIMENT)) { + return EXPERIMENT_VALUE; + } else if (expectedEnrollments.includes(ROLLOUT)) { + return ROLLOUT_VALUE; + } + } + return originalValue; + } + + const expectedDefault = expectedPref(DEFAULT, defaultBranchValue); + const expectedUser = expectedPref(USER, userBranchValue); + + return doBaseTest({ + branch, + configs, + defaultBranchValue, + userBranchValue, + operation, + expectedEnrollments, + expectedDefault, + expectedUser, + }); + }; + + for (const branch of [USER, DEFAULT]) { + for (const defaultBranchValue of [null, DEFAULT_VALUE]) { + for (const userBranchValue of [null, USER_VALUE]) { + for (const specifiedConfigs of [ + pick(bazConfigs, ROLLOUT), + pick(bazConfigs, EXPERIMENT), + bazConfigs, + ]) { + for (const operation of OPERATIONS) { + await doTest({ + branch, + defaultBranchValue, + userBranchValue, + configs: specifiedConfigs, + operation, + }); + } + } + } + } + } + } + + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js b/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js new file mode 100644 index 0000000000..bc74e47eb4 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js @@ -0,0 +1,513 @@ +"use strict"; + +const { TelemetryEvents } = ChromeUtils.importESModule( + "resource://normandy/lib/TelemetryEvents.sys.mjs" +); +const { TelemetryEnvironment } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryEnvironment.sys.mjs" +); +const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled"; +const UPLOAD_ENABLED_PREF = "datareporting.healthreport.uploadEnabled"; + +const globalSandbox = sinon.createSandbox(); +globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); +globalSandbox.spy(TelemetryEvents, "sendEvent"); +registerCleanupFunction(() => { + globalSandbox.restore(); +}); + +/** + * FOG requires a little setup in order to test it + */ +add_setup(function test_setup() { + // FOG needs a profile directory to put its data in. + do_get_profile(); + + // FOG needs to be initialized in order for data to flow. + Services.fog.initializeFOG(); +}); + +/** + * Normal unenrollment for experiments: + * - set .active to false + * - set experiment inactive in telemetry + * - send unrollment event + */ +add_task(async function test_set_inactive() { + const manager = ExperimentFakes.manager(); + + await manager.onStartup(); + await manager.store.addEnrollment(ExperimentFakes.experiment("foo")); + + manager.unenroll("foo", "some-reason"); + + Assert.equal( + manager.store.get("foo").active, + false, + "should set .active to false" + ); +}); + +add_task(async function test_unenroll_opt_out() { + globalSandbox.reset(); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + + Assert.equal( + manager.store.get(experiment.slug).active, + false, + "should set .active to false" + ); + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + experiment.slug, + { + reason: "studies-opt-out", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "studies-opt-out", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + experiment.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + // reset pref + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_unenroll_rollout_opt_out() { + globalSandbox.reset(); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + const manager = ExperimentFakes.manager(); + const rollout = ExperimentFakes.rollout("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(rollout); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + + Assert.equal( + manager.store.get(rollout.slug).active, + false, + "should set .active to false" + ); + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + rollout.slug, + { + reason: "studies-opt-out", + branch: rollout.branch.slug, + enrollmentId: rollout.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + rollout.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct rollout slug" + ); + Assert.equal( + rollout.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "studies-opt-out", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + rollout.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + // reset pref + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_unenroll_uploadPref() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const recipe = ExperimentFakes.recipe("foo"); + + await manager.onStartup(); + await ExperimentFakes.enrollmentHelper(recipe, { manager }).enrollmentPromise; + + Assert.equal( + manager.store.get(recipe.slug).active, + true, + "Should set .active to true" + ); + + Services.prefs.setBoolPref(UPLOAD_ENABLED_PREF, false); + + Assert.equal( + manager.store.get(recipe.slug).active, + false, + "Should set .active to false" + ); + Services.prefs.clearUserPref(UPLOAD_ENABLED_PREF); +}); + +add_task(async function test_setExperimentInactive_called() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Because `manager.store.addEnrollment()` sidesteps telemetry recording + // we will also call on the Glean experiment API directly to test that + // `manager.unenroll()` does in fact call `Glean.setExperimentActive()` + Services.fog.setExperimentActive( + experiment.slug, + experiment.branch.slug, + null + ); + + // Test Glean experiment API interaction + Assert.notEqual( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "experiment should be active before unenroll" + ); + + manager.unenroll("foo", "some-reason"); + + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledWith("foo"), + "should call TelemetryEnvironment.setExperimentInactive with slug" + ); + + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData(experiment.slug), + "experiment should be inactive after unenroll" + ); +}); + +add_task(async function test_send_unenroll_event() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + manager.unenroll("foo", "some-reason"); + + Assert.ok(TelemetryEvents.sendEvent.calledOnce); + Assert.deepEqual( + TelemetryEvents.sendEvent.firstCall.args, + [ + "unenroll", + "nimbus_experiment", + "foo", // slug + { + reason: "some-reason", + branch: experiment.branch.slug, + enrollmentId: experiment.enrollmentId, + }, + ], + "should send an unenrollment ping with the slug, reason, branch slug, and enrollmentId" + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + experiment.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct experiment slug" + ); + Assert.equal( + experiment.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "some-reason", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + experiment.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); +}); + +add_task(async function test_undefined_reason() { + globalSandbox.reset(); + const manager = ExperimentFakes.manager(); + const experiment = ExperimentFakes.experiment("foo"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + await manager.store.addEnrollment(experiment); + + manager.unenroll("foo"); + + const options = TelemetryEvents.sendEvent.firstCall?.args[3]; + Assert.ok( + "reason" in options, + "options object with .reason should be the fourth param" + ); + Assert.equal( + options.reason, + "unknown", + "should include unknown as the reason if none was supplied" + ); + + // Check that the Glean unenrollment event was recorded. + let unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event reason matches the expected reason + Assert.equal( + "unknown", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct (unknown) reason" + ); +}); + +/** + * Normal unenrollment for rollouts: + * - remove stored enrollment and synced data (prefs) + * - set rollout inactive in telemetry + * - send unrollment event + */ + +add_task(async function test_remove_rollouts() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + + sinon.stub(store, "get").returns(rollout); + sinon.spy(store, "updateExperiment"); + + await manager.onStartup(); + + manager.unenroll("foo", "some-reason"); + + Assert.ok( + manager.store.updateExperiment.calledOnce, + "Called to set the rollout as !active" + ); + Assert.ok( + manager.store.updateExperiment.calledWith(rollout.slug, { + active: false, + unenrollReason: "some-reason", + }), + "Called with expected parameters" + ); +}); + +add_task(async function test_remove_rollout_onFinalize() { + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + + sinon.stub(store, "getAllActiveRollouts").returns([rollout]); + sinon.stub(store, "get").returns(rollout); + sinon.spy(manager, "unenroll"); + sinon.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + manager.onFinalize("NimbusTestUtils"); + + // Check that there aren't any Glean unenroll_failed events + var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue(); + Assert.equal( + undefined, + unenrollFailedEvents, + "no Glean unenroll_failed events when removing rollout" + ); + + Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail"); + Assert.ok(manager.unenroll.calledOnce, "Should unenroll recipe not seen"); + Assert.ok(manager.unenroll.calledWith(rollout.slug, "recipe-not-seen")); +}); + +add_task(async function test_rollout_telemetry_events() { + globalSandbox.restore(); + const store = ExperimentFakes.store(); + const manager = ExperimentFakes.manager(store); + const rollout = ExperimentFakes.rollout("foo"); + globalSandbox.spy(TelemetryEnvironment, "setExperimentInactive"); + globalSandbox.spy(TelemetryEvents, "sendEvent"); + + sinon.stub(store, "getAllActiveRollouts").returns([rollout]); + sinon.stub(store, "get").returns(rollout); + sinon.spy(manager, "sendFailureTelemetry"); + + // Clear any pre-existing data in Glean + Services.fog.testResetFOG(); + + await manager.onStartup(); + + // Check that there aren't any Glean unenrollment events yet + var unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + undefined, + unenrollmentEvents, + "no Glean unenrollment events before unenrollment" + ); + + manager.onFinalize("NimbusTestUtils"); + + // Check that there aren't any Glean unenroll_failed events + var unenrollFailedEvents = Glean.nimbusEvents.unenrollFailed.testGetValue(); + Assert.equal( + undefined, + unenrollFailedEvents, + "no Glean unenroll_failed events when removing rollout" + ); + + Assert.ok(manager.sendFailureTelemetry.notCalled, "Nothing should fail"); + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledOnce, + "Should unenroll recipe not seen" + ); + Assert.ok( + TelemetryEnvironment.setExperimentInactive.calledWith(rollout.slug), + "Should set rollout to inactive." + ); + // Test Glean experiment API interaction + Assert.equal( + undefined, + Services.fog.testGetExperimentData(rollout.slug), + "Should set rollout to inactive" + ); + + Assert.ok( + TelemetryEvents.sendEvent.calledWith( + "unenroll", + sinon.match.string, + rollout.slug, + sinon.match.object + ), + "Should send unenroll event for rollout." + ); + + // Check that the Glean unenrollment event was recorded. + unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + // We expect only one event + Assert.equal(1, unenrollmentEvents.length); + // And that one event matches the expected enrolled experiment + Assert.equal( + rollout.slug, + unenrollmentEvents[0].extra.experiment, + "Glean.nimbusEvents.unenrollment recorded with correct rollout slug" + ); + Assert.equal( + rollout.branch.slug, + unenrollmentEvents[0].extra.branch, + "Glean.nimbusEvents.unenrollment recorded with correct branch slug" + ); + Assert.equal( + "recipe-not-seen", + unenrollmentEvents[0].extra.reason, + "Glean.nimbusEvents.unenrollment recorded with correct reason" + ); + Assert.equal( + rollout.enrollmentId, + unenrollmentEvents[0].extra.enrollment_id, + "Glean.nimbusEvents.unenrollment recorded with correct enrollment id" + ); + + globalSandbox.restore(); +}); diff --git a/toolkit/components/nimbus/test/unit/test_ExperimentStore.js b/toolkit/components/nimbus/test/unit/test_ExperimentStore.js new file mode 100644 index 0000000000..8f8022b99e --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_ExperimentStore.js @@ -0,0 +1,874 @@ +"use strict"; + +const { ExperimentStore } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentStore.sys.mjs" +); +const { FeatureManifest } = ChromeUtils.importESModule( + "resource://nimbus/FeatureManifest.sys.mjs" +); + +const { SYNC_DATA_PREF_BRANCH, SYNC_DEFAULTS_PREF_BRANCH } = ExperimentStore; +const { cleanupStorePrefCache } = ExperimentFakes; + +add_task(async function test_sharedDataMap_key() { + const store = new ExperimentStore(); + + // Outside of tests we use sharedDataKey for the profile dir filepath + // where we store experiments + Assert.ok(store._sharedDataKey, "Make sure it's defined"); +}); + +add_task(async function test_usageBeforeInitialization() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + + Assert.equal(store.getAll().length, 0, "It should not fail"); + + await store.init(); + store.addEnrollment(experiment); + + Assert.equal( + store.getExperimentForFeature("purple"), + experiment, + "should return a matching experiment for the given feature" + ); +}); + +add_task(async function test_event_add_experiment() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const expected = ExperimentFakes.experiment("foo"); + const updateEventCbStub = sandbox.stub(); + + // Setup ExperimentManager and child store for ExperimentAPI + await store.init(); + + // Set update cb + store.on("update:foo", updateEventCbStub); + + // Add some data + store.addEnrollment(expected); + + Assert.equal(updateEventCbStub.callCount, 1, "Called once for add"); + + store.off("update:foo", updateEventCbStub); +}); + +add_task(async function test_event_updates_main() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo"); + const updateEventCbStub = sandbox.stub(); + + // Setup ExperimentManager and child store for ExperimentAPI + await store.init(); + + // Set update cb + store.on( + `featureUpdate:${experiment.branch.features[0].featureId}`, + updateEventCbStub + ); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + Assert.equal( + updateEventCbStub.callCount, + 2, + "Should be called twice: add, update" + ); + Assert.equal( + updateEventCbStub.firstCall.args[1], + "experiment-updated", + "Should be called with updated experiment status" + ); + Assert.equal( + updateEventCbStub.secondCall.args[1], + "experiment-updated", + "Should be called with updated experiment status" + ); + + store.off( + `featureUpdate:${experiment.branch.features[0].featureId}`, + updateEventCbStub + ); +}); + +add_task(async function test_getExperimentForGroup() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + + await store.init(); + store.addEnrollment(ExperimentFakes.experiment("bar")); + store.addEnrollment(experiment); + + Assert.equal( + store.getExperimentForFeature("purple"), + experiment, + "should return a matching experiment for the given feature" + ); +}); + +add_task(async function test_hasExperimentForFeature() { + const store = ExperimentFakes.store(); + + await store.init(); + store.addEnrollment( + ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + feature: { featureId: "green" }, + }, + }) + ); + store.addEnrollment( + ExperimentFakes.experiment("foo2", { + branch: { + slug: "variant", + feature: { featureId: "yellow" }, + }, + }) + ); + store.addEnrollment( + ExperimentFakes.experiment("bar_expired", { + active: false, + branch: { + slug: "variant", + feature: { featureId: "purple" }, + }, + }) + ); + Assert.equal( + store.hasExperimentForFeature(), + false, + "should return false if the input is empty" + ); + + Assert.equal( + store.hasExperimentForFeature(undefined), + false, + "should return false if the input is undefined" + ); + + Assert.equal( + store.hasExperimentForFeature("green"), + true, + "should return true if there is an experiment with any of the given groups" + ); + + Assert.equal( + store.hasExperimentForFeature("purple"), + false, + "should return false if there is a non-active experiment with the given groups" + ); +}); + +add_task(async function test_getAll_getAllActiveExperiments() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.experiment(slug, { active: false })) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux"], + ".getAll() should return all experiments" + ); + Assert.deepEqual( + store.getAllActiveExperiments().map(e => e.slug), + ["qux"], + ".getAllActiveExperiments() should return all experiments that are active" + ); +}); + +add_task(async function test_getAll_getAllActiveExperiments() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.experiment(slug, { active: false })) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + store.addEnrollment(ExperimentFakes.rollout("rol")); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux", "rol"], + ".getAll() should return all experiments and rollouts" + ); + Assert.deepEqual( + store.getAllActiveExperiments().map(e => e.slug), + ["qux"], + ".getAllActiveExperiments() should return all experiments that are active and no rollouts" + ); +}); + +add_task(async function test_getAllActiveRollouts() { + const store = ExperimentFakes.store(); + + await store.init(); + ["foo", "bar", "baz"].forEach(slug => + store.addEnrollment(ExperimentFakes.rollout(slug)) + ); + store.addEnrollment(ExperimentFakes.experiment("qux", { active: true })); + + Assert.deepEqual( + store.getAll().map(e => e.slug), + ["foo", "bar", "baz", "qux"], + ".getAll() should return all experiments and rollouts" + ); + Assert.deepEqual( + store.getAllActiveRollouts().map(e => e.slug), + ["foo", "bar", "baz"], + ".getAllActiveRollouts() should return all rollouts" + ); +}); + +add_task(async function test_addEnrollment_experiment() { + const store = ExperimentFakes.store(); + const exp = ExperimentFakes.experiment("foo"); + + await store.init(); + store.addEnrollment(exp); + + Assert.equal(store.get("foo"), exp, "should save experiment by slug"); +}); + +add_task(async function test_addEnrollment_rollout() { + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo"); + + await store.init(); + store.addEnrollment(rollout); + + Assert.equal(store.get("foo"), rollout, "should save rollout by slug"); +}); + +add_task(async function test_updateExperiment() { + const features = [{ featureId: "cfr" }]; + const experiment = Object.freeze( + ExperimentFakes.experiment("foo", { features, active: true }) + ); + const store = ExperimentFakes.store(); + + await store.init(); + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + const actual = store.get("foo"); + Assert.equal(actual.active, false, "should change updated props"); + Assert.deepEqual( + actual.branch.features, + features, + "should not update other props" + ); +}); + +add_task(async function test_sync_access_before_init() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + + Assert.equal(store.getAll().length, 0, "Start with an empty store"); + + const syncAccessExp = ExperimentFakes.experiment("foo", { + features: [{ featureId: "newtab" }], + }); + await store.init(); + store.addEnrollment(syncAccessExp); + + let prefValue; + try { + prefValue = JSON.parse( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}newtab`) + ); + } catch (e) { + Assert.ok(false, "Failed to parse pref value"); + } + + Assert.ok(prefValue, "Parsed stored experiment"); + Assert.equal(prefValue.slug, syncAccessExp.slug, "Got back the experiment"); + + // New un-initialized store that should read the pref value + store = ExperimentFakes.store(); + + Assert.equal( + store.getExperimentForFeature("newtab").slug, + "foo", + "Returns experiment from pref" + ); +}); + +add_task(async function test_sync_access_update() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "aboutwelcome" }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { + branch: { + ...experiment.branch, + features: [ + { + featureId: "aboutwelcome", + value: { bar: "bar", enabled: true }, + }, + ], + }, + }); + + store = ExperimentFakes.store(); + let cachedExperiment = store.getExperimentForFeature("aboutwelcome"); + + Assert.ok(cachedExperiment, "Got back 1 experiment"); + Assert.deepEqual( + // `branch.feature` and not `features` because for sync access (early startup) + // experiments we only store the `isEarlyStartup` feature + cachedExperiment.branch.feature.value, + { bar: "bar", enabled: true }, + "Got updated value" + ); +}); + +add_task(async function test_sync_features_only() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "cfr" }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store = ExperimentFakes.store(); + + Assert.equal(store.getAll().length, 0, "cfr is not a sync access experiment"); +}); + +add_task(async function test_sync_features_remotely() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "cfr", isEarlyStartup: true }], + }); + + await store.init(); + + store.addEnrollment(experiment); + store = ExperimentFakes.store(); + + Assert.ok( + Services.prefs.prefHasUserValue("nimbus.syncdatastore.cfr"), + "The cfr feature was stored as early access in prefs" + ); + Assert.equal(store.getAll().length, 0, "Featre restored from prefs"); +}); + +add_task(async function test_sync_access_unenroll() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment = ExperimentFakes.experiment("foo", { + features: [{ featureId: "aboutwelcome" }], + active: true, + }); + + await store.init(); + + store.addEnrollment(experiment); + store.updateExperiment("foo", { active: false }); + + store = ExperimentFakes.store(); + let experiments = store.getAll(); + + Assert.equal(experiments.length, 0, "Unenrolled experiment is deleted"); +}); + +add_task(async function test_sync_access_unenroll_2() { + cleanupStorePrefCache(); + + let store = ExperimentFakes.store(); + let experiment1 = ExperimentFakes.experiment("foo", { + features: [{ featureId: "newtab" }], + }); + let experiment2 = ExperimentFakes.experiment("bar", { + features: [{ featureId: "aboutwelcome" }], + }); + + await store.init(); + + store.addEnrollment(experiment1); + store.addEnrollment(experiment2); + + Assert.equal(store.getAll().length, 2, "2/2 experiments"); + + let other_store = ExperimentFakes.store(); + + Assert.ok( + other_store.getExperimentForFeature("aboutwelcome"), + "Fetches experiment from pref cache even before init (aboutwelcome)" + ); + + store.updateExperiment("bar", { active: false }); + + Assert.ok( + other_store.getExperimentForFeature("newtab").slug, + "Fetches experiment from pref cache even before init (newtab)" + ); + Assert.ok( + !other_store.getExperimentForFeature("aboutwelcome")?.slug, + "Experiment was updated and should not be found" + ); + + store.updateExperiment("foo", { active: false }); + Assert.ok( + !other_store.getExperimentForFeature("newtab")?.slug, + "Unenrolled from 2/2 experiments" + ); + + Assert.equal( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}newtab`, "").length, + 0, + "Cleared pref 1" + ); + Assert.equal( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}aboutwelcome`, "") + .length, + 0, + "Cleared pref 2" + ); +}); + +add_task(async function test_getRolloutForFeature_fromStore() { + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo"); + + await store.init(); + store.addEnrollment(rollout); + + Assert.deepEqual( + store.getRolloutForFeature(rollout.featureIds[0]), + rollout, + "Should return back the same rollout" + ); +}); + +add_task(async function test_getRolloutForFeature_fromSyncCache() { + let store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "early-startup", + features: [{ featureId: "aboutwelcome", value: { enabled: true } }], + }, + }); + let updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + + await store.init(); + store.addEnrollment(rollout); + await updatePromise; + // New uninitialized store will return data from sync cache + // before init + store = ExperimentFakes.store(); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Sync cache is set" + ); + Assert.equal( + store.getRolloutForFeature(rollout.featureIds[0]).slug, + rollout.slug, + "Should return back the same rollout" + ); + Assert.deepEqual( + store.getRolloutForFeature(rollout.featureIds[0]).branch.feature, + rollout.branch.features[0], + "Should return back the same feature" + ); + cleanupStorePrefCache(); +}); + +add_task(async function test_remoteRollout() { + let store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("foo", { + branch: { + slug: "early-startup", + features: [{ featureId: "aboutwelcome", value: { enabled: true } }], + }, + }); + let featureUpdateStub = sinon.stub(); + let updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + store.on("featureUpdate:aboutwelcome", featureUpdateStub); + + await store.init(); + store.addEnrollment(rollout); + await updatePromise; + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Sync cache is set" + ); + + updatePromise = new Promise(resolve => + store.on(`update:${rollout.slug}`, resolve) + ); + store.updateExperiment(rollout.slug, { active: false }); + + // wait for it to be removed + await updatePromise; + + Assert.ok(featureUpdateStub.calledTwice, "Called for add and remove"); + Assert.ok( + store.get(rollout.slug), + "Rollout is still in the store just not active" + ); + Assert.ok( + !store.getRolloutForFeature("aboutwelcome"), + "Feature rollout should not exist" + ); + Assert.ok( + !Services.prefs.getStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`, + "" + ), + "Sync cache is cleared" + ); +}); + +add_task(async function test_syncDataStore_setDefault() { + cleanupStorePrefCache(); + const store = ExperimentFakes.store(); + + await store.init(); + + Assert.equal( + Services.prefs.getStringPref( + `${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`, + "" + ), + "", + "Pref is empty" + ); + + let rollout = ExperimentFakes.rollout("foo", { + features: [{ featureId: "aboutwelcome", value: { remote: true } }], + }); + store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`), + "Stored in pref" + ); + + cleanupStorePrefCache(); +}); + +add_task(async function test_syncDataStore_getDefault() { + cleanupStorePrefCache(); + const store = ExperimentFakes.store(); + const rollout = ExperimentFakes.rollout("aboutwelcome-slug", { + branch: { + features: [ + { + featureId: "aboutwelcome", + value: { remote: true }, + }, + ], + }, + }); + + await store.init(); + await store.addEnrollment(rollout); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DEFAULTS_PREF_BRANCH}aboutwelcome`) + ); + + let restoredRollout = store.getRolloutForFeature("aboutwelcome"); + + Assert.ok(restoredRollout); + Assert.ok( + restoredRollout.branch.features[0].value.remote, + "Restore data from pref" + ); + + cleanupStorePrefCache(); +}); + +add_task(async function test_addEnrollment_rollout() { + const sandbox = sinon.createSandbox(); + const store = ExperimentFakes.store(); + const stub = sandbox.stub(); + const value = { bar: true }; + let rollout = ExperimentFakes.rollout("foo", { + features: [{ featureId: "aboutwelcome", value }], + }); + + store._onFeatureUpdate("aboutwelcome", stub); + + await store.init(); + store.addEnrollment(rollout); + + Assert.deepEqual( + store.getRolloutForFeature("aboutwelcome"), + rollout, + "should return the stored value" + ); + Assert.equal(stub.callCount, 1, "Called once on update"); + Assert.equal( + stub.firstCall.args[1], + "rollout-updated", + "Called for correct reason" + ); +}); + +add_task(async function test_storeValuePerPref_noVariables() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + Assert.ok( + Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment metadata saved to prefs" + ); + + Assert.equal(branch.getChildList("").length, 0, "No variables to store"); + + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); +}); + +add_task(async function test_storeValuePerPref_withVariables() { + const store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { color: "purple", enabled: true }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + let val = Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`); + Assert.equal( + val.indexOf("color"), + -1, + `Experiment metadata does not contain variables ${val}` + ); + + Assert.equal(branch.getChildList("").length, 2, "Enabled and color"); + + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.equal(branch.getChildList("").length, 0, "Variables are also removed"); +}); + +add_task(async function test_storeValuePerPref_returnsSameValue() { + let store = ExperimentFakes.store(); + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { color: "purple", enabled: true }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + store = ExperimentFakes.store(); + const cachedExperiment = store.getExperimentForFeature("purple"); + // Cached experiment format only stores early access feature + cachedExperiment.branch.features = [cachedExperiment.branch.feature]; + delete cachedExperiment.branch.feature; + Assert.deepEqual(cachedExperiment, experiment, "Returns the same value"); + + // Cleanup + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.deepEqual(branch.getChildList(""), [], "Variables are also removed"); +}); + +add_task(async function test_storeValuePerPref_returnsSameValue_allTypes() { + let store = ExperimentFakes.store(); + // Add a fake feature that matches the variables we're testing + FeatureManifest.purple = { + variables: { + string: { type: "string" }, + bool: { type: "boolean" }, + array: { type: "json" }, + number1: { type: "int" }, + number2: { type: "int" }, + number3: { type: "int" }, + json: { type: "json" }, + }, + }; + const experiment = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [ + { + // Ensure it gets saved to prefs + isEarlyStartup: true, + featureId: "purple", + value: { + string: "string", + bool: true, + array: [1, 2, 3], + number1: 42, + number2: 0, + number3: -5, + json: { jsonValue: true }, + }, + }, + ], + }, + }); + + await store.init(); + store.addEnrollment(experiment); + let branch = Services.prefs.getBranch(`${SYNC_DATA_PREF_BRANCH}purple.`); + + store = ExperimentFakes.store(); + Assert.deepEqual( + store.getExperimentForFeature("purple").branch.feature.value, + experiment.branch.features[0].value, + "Returns the same value" + ); + + // Cleanup + store._updateSyncStore({ ...experiment, active: false }); + Assert.ok( + !Services.prefs.getStringPref(`${SYNC_DATA_PREF_BRANCH}purple`, ""), + "Experiment cleanup" + ); + Assert.deepEqual(branch.getChildList(""), [], "Variables are also removed"); + delete FeatureManifest.purple; +}); + +add_task(async function test_cleanupOldRecipes() { + let store = ExperimentFakes.store(); + let sandbox = sinon.createSandbox(); + let stub = sandbox.stub(store, "_removeEntriesByKeys"); + const experiment1 = ExperimentFakes.experiment("foo", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment2 = ExperimentFakes.experiment("bar", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment3 = ExperimentFakes.experiment("baz", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + const experiment4 = ExperimentFakes.experiment("faz", { + branch: { + slug: "variant", + features: [{ featureId: "purple" }], + }, + }); + // Exp 2 is kept because it's recent (even though it's not active) + // Exp 4 is kept because it's active + experiment2.lastSeen = new Date().toISOString(); + experiment2.active = false; + experiment1.lastSeen = new Date("2020-01-01").toISOString(); + experiment1.active = false; + experiment3.active = false; + delete experiment3.lastSeen; + store._data = { + foo: experiment1, + bar: experiment2, + baz: experiment3, + faz: experiment4, + }; + + store._cleanupOldRecipes(); + + Assert.ok(stub.calledOnce, "Recipe cleanup called"); + Assert.equal( + stub.firstCall.args[0].length, + 2, + "We call to remove enrollments" + ); + Assert.equal( + stub.firstCall.args[0][0], + experiment1.slug, + "Should remove expired enrollment" + ); + Assert.equal( + stub.firstCall.args[0][1], + experiment3.slug, + "Should remove invalid enrollment" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js b/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js new file mode 100644 index 0000000000..5b9aa301d0 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js @@ -0,0 +1,82 @@ +"use strict"; + +const { ExperimentFakes, ExperimentTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); + +add_task(async function test_recipe_fake_validates() { + const recipe = ExperimentFakes.recipe("foo"); + Assert.ok( + await ExperimentTestUtils.validateExperiment(recipe), + "should produce a valid experiment recipe" + ); +}); + +add_task(async function test_enrollmentHelper() { + let recipe = ExperimentFakes.recipe("bar", { + branches: [ + { + slug: "control", + ratio: 1, + features: [{ featureId: "aboutwelcome", value: {} }], + }, + ], + }); + let manager = ExperimentFakes.manager(); + + Assert.deepEqual( + recipe.featureIds, + ["aboutwelcome"], + "Helper sets correct featureIds" + ); + + await manager.onStartup(); + + let { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(recipe, { manager }); + + await enrollmentPromise; + + Assert.ok(manager.store.getAllActiveExperiments().length === 1, "Enrolled"); + Assert.equal( + manager.store.getAllActiveExperiments()[0].slug, + recipe.slug, + "Has expected slug" + ); + Assert.ok( + Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"), + "Sync pref cache set" + ); + + await doExperimentCleanup(); + + Assert.ok(manager.store.getAll().length === 0, "Cleanup done"); + Assert.ok( + !Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"), + "Sync pref cache is cleared" + ); +}); + +add_task(async function test_enrollWithFeatureConfig() { + let manager = ExperimentFakes.manager(); + await manager.onStartup(); + let doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "enrollWithFeatureConfig", + value: { enabled: true }, + }, + { manager } + ); + + Assert.ok( + manager.store.hasExperimentForFeature("enrollWithFeatureConfig"), + "Enrolled successfully" + ); + + await doExperimentCleanup(); + + Assert.ok( + !manager.store.hasExperimentForFeature("enrollWithFeatureConfig"), + "Unenrolled successfully" + ); +}); diff --git a/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js new file mode 100644 index 0000000000..a7e06a03e6 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader.js @@ -0,0 +1,344 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { ExperimentManager } = ChromeUtils.importESModule( + "resource://nimbus/lib/ExperimentManager.sys.mjs" +); +const { RemoteSettingsExperimentLoader, EnrollmentsContext } = + ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" + ); +const { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); + +const ENABLED_PREF = "messaging-system.rsexperimentloader.enabled"; +const RUN_INTERVAL_PREF = "app.normandy.run_interval_seconds"; +const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled"; +const UPLOAD_PREF = "datareporting.healthreport.uploadEnabled"; +const DEBUG_PREF = "nimbus.debug"; + +add_task(async function test_real_exp_manager() { + equal( + RemoteSettingsExperimentLoader.manager, + ExperimentManager, + "should reference ExperimentManager singleton by default" + ); +}); + +add_task(async function test_lazy_pref_getters() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setIntPref(RUN_INTERVAL_PREF, 123456); + equal( + loader.intervalInSeconds, + 123456, + `should set intervalInSeconds to the value of ${RUN_INTERVAL_PREF}` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + equal( + loader.enabled, + true, + `should set enabled to the value of ${ENABLED_PREF}` + ); + Services.prefs.setBoolPref(ENABLED_PREF, false); + equal(loader.enabled, false); + + Services.prefs.clearUserPref(RUN_INTERVAL_PREF); + Services.prefs.clearUserPref(ENABLED_PREF); +}); + +add_task(async function test_init() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setBoolPref(ENABLED_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${ENABLED_PREF} pref is false` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.setTimer.calledOnce, "should call .setTimer"); + ok(loader.updateRecipes.calledOnce, "should call .updatpickeRecipes"); +}); + +add_task(async function test_init_with_opt_in() { + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${STUDIES_OPT_OUT_PREF} pref is false` + ); + + Services.prefs.setBoolPref(ENABLED_PREF, false); + await loader.init(); + equal( + loader.setTimer.callCount, + 0, + `should not initialize if ${ENABLED_PREF} pref is false` + ); + + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.setTimer.calledOnce, "should call .setTimer"); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); +}); + +add_task(async function test_updateRecipes() { + const loader = ExperimentFakes.rsLoader(); + + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "true", + }); + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "false", + }); + sinon.stub(loader, "setTimer"); + sinon.spy(loader, "updateRecipes"); + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([PASS_FILTER_RECIPE, FAIL_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should call .onRecipe only for recipes that pass" + ); + ok( + loader.manager.onRecipe.calledWith(PASS_FILTER_RECIPE, "rs-loader"), + "should call .onRecipe with argument data" + ); +}); + +add_task(async function test_updateRecipes_someMismatch() { + const loader = ExperimentFakes.rsLoader(); + + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "true", + }); + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "false", + }); + sinon.stub(loader, "setTimer"); + sinon.spy(loader, "updateRecipes"); + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([PASS_FILTER_RECIPE, FAIL_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should call .onRecipe only for recipes that pass" + ); + ok(loader.manager.onFinalize.calledOnce, "Should call onFinalize."); + ok( + onFinalizeCalled(loader.manager.onFinalize, "rs-loader", { + recipeMismatches: [FAIL_FILTER_RECIPE.slug], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingL10nIds: new Map(), + missingLocale: [], + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with the recipes that failed targeting" + ); +}); + +add_task(async function test_updateRecipes_forFirstStartup() { + const loader = ExperimentFakes.rsLoader(); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "isFirstStartup", + }); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + sinon + .stub(loader.manager, "createTargetingContext") + .returns({ isFirstStartup: true }); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init({ isFirstStartup: true }); + + ok(loader.manager.onRecipe.calledOnce, "should pass the targeting filter"); +}); + +add_task(async function test_updateRecipes_forNoneFirstStartup() { + const loader = ExperimentFakes.rsLoader(); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "isFirstStartup", + }); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + sinon + .stub(loader.manager, "createTargetingContext") + .returns({ isFirstStartup: false }); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init({ isFirstStartup: true }); + + ok(loader.manager.onRecipe.notCalled, "should not pass the targeting filter"); +}); + +add_task(async function test_checkTargeting() { + const loader = ExperimentFakes.rsLoader(); + const ctx = new EnrollmentsContext(loader.manager); + equal( + await ctx.checkTargeting({}), + true, + "should return true if .targeting is not defined" + ); + equal( + await ctx.checkTargeting({ + targeting: "'foo'", + slug: "test_checkTargeting", + }), + true, + "should return true for truthy expression" + ); + equal( + await ctx.checkTargeting({ + targeting: "aPropertyThatDoesNotExist", + slug: "test_checkTargeting", + }), + false, + "should return false for falsey expression" + ); +}); + +add_task(async function test_checkExperimentSelfReference() { + const loader = ExperimentFakes.rsLoader(); + const ctx = new EnrollmentsContext(loader.manager); + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: + "experiment.slug == 'foo' && experiment.branches[0].slug == 'control'", + }); + + const FAIL_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "experiment.slug == 'bar'", + }); + + equal( + await ctx.checkTargeting(PASS_FILTER_RECIPE), + true, + "Should return true for matching on slug name and branch" + ); + equal( + await ctx.checkTargeting(FAIL_FILTER_RECIPE), + false, + "Should fail targeting" + ); +}); + +add_task(async function test_optIn_debug_disabled() { + info("Testing users cannot opt-in when nimbus.debug is false"); + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + const recipe = ExperimentFakes.recipe("foo"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + Services.prefs.setBoolPref(DEBUG_PREF, false); + Services.prefs.setBoolPref(UPLOAD_PREF, true); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + + await Assert.rejects( + loader.optInToExperiment({ + slug: recipe.slug, + branchSlug: recipe.branches[0].slug, + }), + /Could not opt in/ + ); + + Services.prefs.clearUserPref(DEBUG_PREF); + Services.prefs.clearUserPref(UPLOAD_PREF); + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_optIn_studies_disabled() { + info( + "Testing users cannot opt-in when telemetry is disabled or studies are disabled." + ); + + const prefs = [UPLOAD_PREF, STUDIES_OPT_OUT_PREF]; + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader, "setTimer"); + sinon.stub(loader, "updateRecipes").resolves(); + + const recipe = ExperimentFakes.recipe("foo"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + Services.prefs.setBoolPref(DEBUG_PREF, true); + + for (const pref of prefs) { + Services.prefs.setBoolPref(UPLOAD_PREF, true); + Services.prefs.setBoolPref(STUDIES_OPT_OUT_PREF, true); + + Services.prefs.setBoolPref(pref, false); + + await Assert.rejects( + loader.optInToExperiment({ + slug: recipe.slug, + branchSlug: recipe.branches[0].slug, + }), + /Could not opt in: studies are disabled/ + ); + } + + Services.prefs.clearUserPref(DEBUG_PREF); + Services.prefs.clearUserPref(UPLOAD_PREF); + Services.prefs.clearUserPref(STUDIES_OPT_OUT_PREF); +}); + +add_task(async function test_enrollment_changed_notification() { + const loader = ExperimentFakes.rsLoader(); + + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: "true", + }); + sinon.stub(loader, "setTimer"); + sinon.spy(loader, "updateRecipes"); + const enrollmentChanged = TestUtils.topicObserved( + "nimbus:enrollments-updated" + ); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + Services.prefs.setBoolPref(ENABLED_PREF, true); + await loader.init(); + await enrollmentChanged; + ok(loader.updateRecipes.called, "should call .updateRecipes"); +}); diff --git a/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js new file mode 100644 index 0000000000..bcf016b3ab --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js @@ -0,0 +1,1271 @@ +"use strict"; + +const { ExperimentFakes } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { FirstStartup } = ChromeUtils.importESModule( + "resource://gre/modules/FirstStartup.sys.mjs" +); +const { NimbusFeatures } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); +const { EnrollmentsContext } = ChromeUtils.importESModule( + "resource://nimbus/lib/RemoteSettingsExperimentLoader.sys.mjs" +); +const { PanelTestProvider } = ChromeUtils.importESModule( + "resource://activity-stream/lib/PanelTestProvider.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.importESModule( + "resource://normandy/lib/TelemetryEvents.sys.mjs" +); + +add_setup(async function setup() { + do_get_profile(); + Services.fog.initializeFOG(); +}); + +add_task(async function test_updateRecipes_activeExperiments() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", { + targeting: `"${recipe.slug}" in activeExperiments`, + }); + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([recipe]); + + await loader.init(); + + ok(onRecipe.calledOnce, "Should match active experiments"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_updateRecipes_isFirstRun() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const recipe = ExperimentFakes.recipe("foo"); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + const PASS_FILTER_RECIPE = { ...recipe, targeting: "isFirstStartup" }; + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([recipe]); + + // Pretend to be in the first startup + FirstStartup._state = FirstStartup.IN_PROGRESS; + await loader.init(); + + Assert.ok(onRecipe.calledOnce, "Should match first run"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_updateRecipes_invalidFeatureId() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "invalid-feature-id", + value: { hello: "world" }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "invalid-feature-id", + value: { hello: "goodbye" }, + }, + ], + }, + ], + }); + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); + + await assertEmptyStore(manager.store); +}); + +add_task(async function test_updateRecipes_invalidFeatureValue() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { + template: "spotlight", + }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { + template: "spotlight", + }, + }, + ], + }, + ], + }); + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_invalidRecipe() { + const manager = ExperimentFakes.manager(); + const sandbox = sinon.createSandbox(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const badRecipe = ExperimentFakes.recipe("foo"); + delete badRecipe.slug; + + const onRecipe = sandbox.stub(manager, "onRecipe"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + sandbox.stub(manager.store, "ready").resolves(); + sandbox.stub(manager.store, "getAllActiveExperiments").returns([]); + + await loader.init(); + ok(onRecipe.notCalled, "No recipes"); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_invalidRecipeAfterUpdate() { + Services.fog.testResetFOG(); + + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const recipe = ExperimentFakes.recipe("foo"); + const badRecipe = { ...recipe }; + delete badRecipe.branches; + + sinon.stub(loader, "setTimer"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + sinon.stub(manager.store, "ready").resolves(); + sinon.spy(loader, "updateRecipes"); + + await loader.init(); + + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + + ok( + onFinalizeCalled(loader.manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with no mismatches or invalid recipes" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + loader.manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + onFinalizeCalled(loader.manager.onFinalize.secondCall.args, "rs-loader", { + recipeMismatches: [], + invalidRecipes: ["foo"], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with an invalid recipe" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_invalidBranchAfterUpdate() { + const message = await PanelTestProvider.getMessages().then(msgs => + msgs.find(m => m.id === "MULTISTAGE_SPOTLIGHT_MESSAGE") + ); + + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + loader.manager = manager; + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { ...message }, + }, + ], + }, + { + slug: "treatment", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: { ...message }, + }, + ], + }, + ], + }); + + const badRecipe = { + ...recipe, + branches: [ + { ...recipe.branches[0] }, + { + ...recipe.branches[1], + features: [ + { + ...recipe.branches[1].features[0], + value: { ...message }, + }, + ], + }, + ], + }; + delete badRecipe.branches[1].features[0].value.template; + + sinon.stub(loader, "setTimer"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + sinon.stub(manager.store, "ready").resolves(); + sinon.spy(loader, "updateRecipes"); + + await loader.init(); + + ok(loader.updateRecipes.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + ok( + onFinalizeCalled(loader.manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with no mismatches or invalid recipes" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + loader.manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + loader.manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + onFinalizeCalled(loader.manager.onFinalize.secondCall.args, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([["foo", [badRecipe.branches[1].slug]]]), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with an invalid branch" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_simpleFeatureInvalidAfterUpdate() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("foo"); + const badRecipe = ExperimentFakes.recipe("foo", { + branches: [ + { + ...recipe.branches[0], + features: [ + { + featureId: "testFeature", + value: { testInt: "abc123", enabled: true }, + }, + ], + }, + { + ...recipe.branches[1], + features: [ + { + featureId: "testFeature", + value: { testInt: 456, enabled: true }, + }, + ], + }, + ], + }); + + const EXPECTED_SCHEMA = { + $schema: "https://json-schema.org/draft/2019-09/schema", + title: "testFeature", + description: NimbusFeatures.testFeature.manifest.description, + type: "object", + properties: { + testInt: { + type: "integer", + }, + enabled: { + type: "boolean", + }, + testSetString: { + type: "string", + }, + }, + additionalProperties: true, + }; + + sinon.spy(loader, "updateRecipes"); + sinon.spy(EnrollmentsContext.prototype, "_generateVariablesOnlySchema"); + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onFinalize"); + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + + await loader.init(); + ok(manager.onRecipe.calledOnce, "should call .updateRecipes"); + equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once"); + ok( + loader.manager.onRecipe.calledWith(recipe, "rs-loader"), + "should call .onRecipe with argument data" + ); + equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once"); + ok( + onFinalizeCalled(loader.manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with nomismatches or invalid recipes" + ); + + ok( + EnrollmentsContext.prototype._generateVariablesOnlySchema.calledOnce, + "Should have generated a schema for testFeature" + ); + + Assert.deepEqual( + EnrollmentsContext.prototype._generateVariablesOnlySchema.returnValues[0], + EXPECTED_SCHEMA, + "should have generated a schema with three fields" + ); + + info("Replacing recipe with an invalid one"); + + loader.remoteSettingsClient.get.resolves([badRecipe]); + + await loader.updateRecipes("timer"); + equal( + manager.onRecipe.callCount, + 1, + "should not have called .onRecipe again" + ); + equal( + manager.onFinalize.callCount, + 2, + "should have called .onFinalize again" + ); + + ok( + onFinalizeCalled(loader.manager.onFinalize.secondCall.args, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([["foo", [badRecipe.branches[0].slug]]]), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with an invalid branch" + ); + + EnrollmentsContext.prototype._generateVariablesOnlySchema.restore(); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_validationTelemetry() { + TelemetryEvents.init(); + + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); + + const invalidRecipe = ExperimentFakes.recipe("invalid-recipe"); + delete invalidRecipe.channel; + + const invalidBranch = ExperimentFakes.recipe("invalid-branch"); + invalidBranch.branches[0].features[0].value.testInt = "hello"; + invalidBranch.branches[1].features[0].value.testInt = "world"; + + const invalidFeature = ExperimentFakes.recipe("invalid-feature", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "unknown-feature", + value: { foo: "bar" }, + }, + { + featureId: "second-unknown-feature", + value: { baz: "qux" }, + }, + ], + }, + ], + }); + + const TEST_CASES = [ + { + recipe: invalidRecipe, + reason: "invalid-recipe", + events: [{}], + callCount: 1, + }, + { + recipe: invalidBranch, + reason: "invalid-branch", + events: invalidBranch.branches.map(branch => ({ branch: branch.slug })), + callCount: 2, + }, + { + recipe: invalidFeature, + reason: "invalid-feature", + events: invalidFeature.branches[0].features.map(feature => ({ + feature: feature.featureId, + })), + callCount: 2, + }, + ]; + + const LEGACY_FILTER = { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + }; + + for (const { recipe, reason, events, callCount } of TEST_CASES) { + info(`Testing validation failed telemetry for reason = "${reason}" ...`); + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActiveExperiments").returns([]); + sinon.stub(manager.store, "getAllActiveRollouts").returns([]); + + const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry"); + + await loader.init(); + + Assert.equal( + telemetrySpy.callCount, + callCount, + `Should call sendValidationFailedTelemetry ${callCount} times for reason ${reason}` + ); + + const gleanEvents = Glean.nimbusEvents.validationFailed + .testGetValue() + .map(event => { + event = { ...event }; + // We do not care about the timestamp. + delete event.timestamp; + return event; + }); + + const expectedGleanEvents = events.map(event => ({ + category: "nimbus_events", + name: "validation_failed", + extra: { + experiment: recipe.slug, + reason, + ...event, + }, + })); + + Assert.deepEqual( + gleanEvents, + expectedGleanEvents, + "Glean telemetry matches" + ); + + const expectedLegacyEvents = events.map(event => ({ + ...LEGACY_FILTER, + value: recipe.slug, + extra: { + reason, + ...event, + }, + LEGACY_FILTER, + })); + + TelemetryTestUtils.assertEvents(expectedLegacyEvents, LEGACY_FILTER, { + clear: true, + }); + + Services.fog.testResetFOG(); + + await assertEmptyStore(manager.store, { cleanup: true }); + } +}); + +add_task(async function test_updateRecipes_validationDisabled() { + Services.prefs.setBoolPref("nimbus.validation.enabled", false); + + const invalidRecipe = ExperimentFakes.recipe("invalid-recipe"); + delete invalidRecipe.channel; + + const invalidBranch = ExperimentFakes.recipe("invalid-branch"); + invalidBranch.branches[0].features[0].value.testInt = "hello"; + invalidBranch.branches[1].features[0].value.testInt = "world"; + + const invalidFeature = ExperimentFakes.recipe("invalid-feature", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "unknown-feature", + value: { foo: "bar" }, + }, + { + featureId: "second-unknown-feature", + value: { baz: "qux" }, + }, + ], + }, + ], + }); + + for (const recipe of [invalidRecipe, invalidBranch, invalidFeature]) { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActiveExperiments").returns([]); + sinon.stub(manager.store, "getAllActiveRollouts").returns([]); + + const finalizeStub = sinon.stub(manager, "onFinalize"); + const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry"); + + await loader.init(); + + Assert.equal( + telemetrySpy.callCount, + 0, + "Should not send validation failed telemetry" + ); + Assert.ok( + onFinalizeCalled(finalizeStub, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: false, + }), + "should call .onFinalize with no validation issues" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); + } + + Services.prefs.clearUserPref("nimbus.validation.enabled"); +}); + +add_task(async function test_updateRecipes_appId() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("background-task-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "backgroundTaskMessage", + value: {}, + }, + ], + }, + ], + }); + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + + info("Testing updateRecipes() with the default application ID"); + await loader.init(); + + Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called"); + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); + + info("Testing updateRecipes() with a custom application ID"); + + Services.prefs.setStringPref( + "nimbus.appId", + "firefox-desktop-background-task" + ); + + await loader.updateRecipes(); + Assert.ok( + manager.onRecipe.calledWith(recipe, "rs-loader"), + `.onRecipe called with ${recipe.slug}` + ); + + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); + + Services.prefs.clearUserPref("nimbus.appId"); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_withPropNotInManifest() { + // Need to randomize the slug so subsequent test runs don't skip enrollment + // due to a conflicting slug + const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo" + Math.random(), { + arguments: {}, + branches: [ + { + features: [ + { + enabled: true, + featureId: "testFeature", + value: { + enabled: true, + testInt: 5, + testSetString: "foo", + additionalPropNotInManifest: 7, + }, + }, + ], + ratio: 1, + slug: "treatment-2", + }, + ], + channel: "nightly", + schemaVersion: "1.9.0", + targeting: "true", + }); + + const loader = ExperimentFakes.rsLoader(); + sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]); + sinon.stub(loader.manager, "onRecipe").resolves(); + sinon.stub(loader.manager, "onFinalize"); + + await loader.init(); + + ok( + loader.manager.onRecipe.calledWith(PASS_FILTER_RECIPE, "rs-loader"), + "should call .onRecipe with this recipe" + ); + equal(loader.manager.onRecipe.callCount, 1, "should only call onRecipe once"); + + await assertEmptyStore(loader.manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_recipeAppId() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const recipe = ExperimentFakes.recipe("mobile-experiment", { + appId: "org.mozilla.firefox", + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "mobile-feature", + value: { + enabled: true, + }, + }, + ], + }, + ], + }); + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + + await loader.init(); + + Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called"); + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "Should call .onFinalize with no validation issues" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_featureValidationOptOut() { + const invalidTestRecipe = ExperimentFakes.recipe("invalid-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: { + enabled: "true", + testInt: false, + }, + }, + ], + }, + ], + }); + + const message = await PanelTestProvider.getMessages().then(msgs => + msgs.find(m => m.id === "MULTISTAGE_SPOTLIGHT_MESSAGE") + ); + delete message.template; + + const invalidMsgRecipe = ExperimentFakes.recipe("invalid-recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "spotlight", + value: message, + }, + ], + }, + ], + }); + + for (const invalidRecipe of [invalidTestRecipe, invalidMsgRecipe]) { + const optOutRecipe = { + ...invalidMsgRecipe, + slug: "optout-recipe", + featureValidationOptOut: true, + }; + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([invalidRecipe, optOutRecipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActiveExperiments").returns([]); + sinon.stub(manager.store, "getAllActiveRollouts").returns([]); + + await loader.init(); + ok( + manager.onRecipe.calledOnceWith(optOutRecipe, "rs-loader"), + "should call .onRecipe for the opt-out recipe" + ); + + ok( + manager.onFinalize.calledOnce && + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map([[invalidRecipe.slug, ["control"]]]), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map(), + locale: Services.locale.appLocaleAsBCP47, + validationEnabled: true, + }), + "should call .onFinalize with only one invalid recipe" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); + } +}); + +add_task(async function test_updateRecipes_invalidFeature_mismatch() { + info( + "Testing that we do not submit validation telemetry when the targeting does not match" + ); + const recipe = ExperimentFakes.recipe("recipe", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "bogus", + value: { + bogus: "bogus", + }, + }, + ], + }, + ], + targeting: "false", + }); + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + sinon.stub(loader, "setTimer"); + sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + + sinon.stub(manager, "onRecipe"); + sinon.stub(manager, "onFinalize"); + sinon.stub(manager.store, "ready").resolves(); + sinon.stub(manager.store, "getAllActiveExperiments").returns([]); + sinon.stub(manager.store, "getAllActiveRollouts").returns([]); + + const telemetrySpy = sinon.stub(manager, "sendValidationFailedTelemetry"); + const targetingSpy = sinon.spy( + EnrollmentsContext.prototype, + "checkTargeting" + ); + + await loader.init(); + ok(targetingSpy.calledOnce, "Should have checked targeting for recipe"); + ok( + !(await targetingSpy.returnValues[0]), + "Targeting should not have matched" + ); + ok(manager.onRecipe.notCalled, "should not call .onRecipe for the recipe"); + ok( + telemetrySpy.notCalled, + "Should not have submitted validation failed telemetry" + ); + + targetingSpy.restore(); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_updateRecipes_rollout_bucketing() { + TelemetryEvents.init(); + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); + + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + const experiment = ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: {}, + }, + ], + }, + ], + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + }); + const rollout = ExperimentFakes.recipe("rollout", { + isRollout: true, + branches: [ + { + slug: "rollout", + ratio: 1, + features: [ + { + featureId: "testFeature", + value: {}, + }, + ], + }, + ], + bucketConfig: { + namespace: "nimbus-test-utils", + randomizationUnit: "normandy_id", + start: 0, + count: 1000, + total: 1000, + }, + }); + + await loader.init(); + await manager.onStartup(); + await manager.store.ready(); + + sinon + .stub(loader.remoteSettingsClient, "get") + .resolves([experiment, rollout]); + + await loader.updateRecipes(); + + Assert.equal( + manager.store.getExperimentForFeature("testFeature")?.slug, + experiment.slug, + "Should enroll in experiment" + ); + Assert.equal( + manager.store.getRolloutForFeature("testFeature")?.slug, + rollout.slug, + "Should enroll in rollout" + ); + + experiment.bucketConfig.count = 0; + rollout.bucketConfig.count = 0; + + await loader.updateRecipes(); + + Assert.equal( + manager.store.getExperimentForFeature("testFeature")?.slug, + experiment.slug, + "Should stay enrolled in experiment -- experiments cannot be resized" + ); + Assert.ok( + !manager.store.getRolloutForFeature("testFeature"), + "Should unenroll from rollout" + ); + + const unenrollmentEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal( + unenrollmentEvents.length, + 1, + "Should be one unenrollment event" + ); + Assert.equal( + unenrollmentEvents[0].extra.experiment, + rollout.slug, + "Experiment slug should match" + ); + Assert.equal( + unenrollmentEvents[0].extra.reason, + "bucketing", + "Reason should match" + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: rollout.slug, + extra: { + reason: "bucketing", + }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + } + ); + + manager.unenroll(experiment.slug); + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_reenroll_rollout_resized() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + await loader.init(); + await manager.onStartup(); + await manager.store.ready(); + + const rollout = ExperimentFakes.recipe("rollout", { + isRollout: true, + }); + rollout.bucketConfig = { + ...rollout.bucketConfig, + start: 0, + count: 1000, + total: 1000, + }; + + sinon.stub(loader.remoteSettingsClient, "get").resolves([rollout]); + + await loader.updateRecipes(); + Assert.equal( + manager.store.getRolloutForFeature("testFeature")?.slug, + rollout.slug, + "Should enroll in rollout" + ); + + rollout.bucketConfig.count = 0; + await loader.updateRecipes(); + + Assert.ok( + !manager.store.getRolloutForFeature("testFeature"), + "Should unenroll from rollout" + ); + + const enrollment = manager.store.get(rollout.slug); + Assert.equal(enrollment.unenrollReason, "bucketing"); + + rollout.bucketConfig.count = 1000; + await loader.updateRecipes(); + + Assert.equal( + manager.store.getRolloutForFeature("testFeature")?.slug, + rollout.slug, + "Should re-enroll in rollout" + ); + + const newEnrollment = manager.store.get(rollout.slug); + Assert.ok( + !Object.is(enrollment, newEnrollment), + "Should have new enrollment object" + ); + Assert.ok( + !("unenrollReason" in newEnrollment), + "New enrollment should not have unenroll reason" + ); + + manager.unenroll(rollout.slug); + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_experiment_reenroll() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + await loader.init(); + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment"); + experiment.bucketConfig = { + ...experiment.bucketConfig, + start: 0, + count: 1000, + total: 1000, + }; + + await manager.enroll(experiment, "test"); + Assert.equal( + manager.store.getExperimentForFeature("testFeature")?.slug, + experiment.slug, + "Should enroll in experiment" + ); + + manager.unenroll(experiment.slug); + Assert.ok( + !manager.store.getExperimentForFeature("testFeature"), + "Should unenroll from experiment" + ); + + sinon.stub(loader.remoteSettingsClient, "get").resolves([experiment]); + + await loader.updateRecipes(); + Assert.ok( + !manager.store.getExperimentForFeature("testFeature"), + "Should not re-enroll in experiment" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); + +add_task(async function test_rollout_reenroll_optout() { + const loader = ExperimentFakes.rsLoader(); + const manager = loader.manager; + + await loader.init(); + await manager.onStartup(); + await manager.store.ready(); + + const rollout = ExperimentFakes.recipe("experiment", { isRollout: true }); + rollout.bucketConfig = { + ...rollout.bucketConfig, + start: 0, + count: 1000, + total: 1000, + }; + + sinon.stub(loader.remoteSettingsClient, "get").resolves([rollout]); + await loader.updateRecipes(); + + Assert.ok( + manager.store.getRolloutForFeature("testFeature"), + "Should enroll in rollout" + ); + + manager.unenroll(rollout.slug, "individual-opt-out"); + + await loader.updateRecipes(); + + Assert.ok( + !manager.store.getRolloutForFeature("testFeature"), + "Should not re-enroll in rollout" + ); + + await assertEmptyStore(manager.store, { cleanup: true }); +}); diff --git a/toolkit/components/nimbus/test/unit/test_SharedDataMap.js b/toolkit/components/nimbus/test/unit/test_SharedDataMap.js new file mode 100644 index 0000000000..6186b41a40 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_SharedDataMap.js @@ -0,0 +1,207 @@ +const { SharedDataMap } = ChromeUtils.importESModule( + "resource://nimbus/lib/SharedDataMap.sys.mjs" +); +const { FileTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/FileTestUtils.sys.mjs" +); +const { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); + +const PATH = FileTestUtils.getTempFile("shared-data-map").path; + +function with_sharedDataMap(test) { + let testTask = async () => { + const sandbox = sinon.createSandbox(); + const instance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: true, + }); + try { + await test({ instance, sandbox }); + } finally { + sandbox.restore(); + } + }; + + // Copy the name of the test function to identify the test + Object.defineProperty(testTask, "name", { value: test.name }); + add_task(testTask); +} + +with_sharedDataMap(async function test_set_notify({ instance, sandbox }) { + await instance.init(); + let updateStub = sandbox.stub(); + + instance.on("parent-store-update:foo", updateStub); + instance.set("foo", "bar"); + + Assert.equal(updateStub.callCount, 1, "Update event sent"); + Assert.equal(updateStub.firstCall.args[1], "bar", "Update event sent value"); +}); + +with_sharedDataMap(async function test_set_child_notify({ instance, sandbox }) { + await instance.init(); + + let updateStub = sandbox.stub(); + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + childInstance.on("child-store-update:foo", updateStub); + let childStoreUpdate = new Promise(resolve => + childInstance.on("child-store-update:foo", resolve) + ); + instance.set("foo", "bar"); + + await childStoreUpdate; + + Assert.equal(updateStub.callCount, 1, "Update event sent"); + Assert.equal(updateStub.firstCall.args[1], "bar", "Update event sent value"); +}); + +with_sharedDataMap(async function test_async({ instance, sandbox }) { + const spy = sandbox.spy(instance._store, "load"); + await instance.init(); + + instance.set("foo", "bar"); + + Assert.equal(spy.callCount, 1, "Should init async"); + Assert.equal(instance.get("foo"), "bar", "It should retrieve a string value"); +}); + +with_sharedDataMap(async function test_saveSoon({ instance, sandbox }) { + await instance.init(); + const stub = sandbox.stub(instance._store, "saveSoon"); + + instance.set("foo", "bar"); + + Assert.equal(stub.callCount, 1, "Should call save soon when setting a value"); +}); + +with_sharedDataMap(async function test_init_safe({ instance, sandbox }) { + let stub = sandbox.stub(instance._store, "load"); + sandbox.replaceGetter(instance._store, "data", () => { + throw new Error("expected xpcshell"); + }); + + try { + await instance.init(); + Assert.ok(stub.calledOnce, "Load should be called"); + } catch (e) { + Assert.ok(false, "Error should be caught in SharedDataMap"); + } +}); + +with_sharedDataMap(async function test_childInit({ instance, sandbox }) { + sandbox.stub(instance, "isParent").get(() => false); + const stubA = sandbox.stub(instance._store, "ensureDataReady"); + const stubB = sandbox.stub(instance._store, "load"); + + await instance.init(); + + Assert.equal( + stubA.callCount, + 0, + "It should not try to initialize sync from child" + ); + Assert.equal( + stubB.callCount, + 0, + "It should not try to initialize async from child" + ); +}); + +with_sharedDataMap(async function test_parentChildSync_synchronously({ + instance: parentInstance, + sandbox, +}) { + await parentInstance.init(); + parentInstance.set("foo", { bar: 1 }); + + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + await parentInstance.ready(); + await childInstance.ready(); + + await TestUtils.waitForCondition( + () => childInstance.get("foo"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("foo"), + parentInstance.get("foo"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_parentChildSync_async({ + instance: parentInstance, + sandbox, +}) { + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + await parentInstance.init(); + parentInstance.set("foo", { bar: 1 }); + + await parentInstance.ready(); + await childInstance.ready(); + + await TestUtils.waitForCondition( + () => childInstance.get("foo"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("foo"), + parentInstance.get("foo"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_earlyChildSync({ + instance: parentInstance, + sandbox, +}) { + const childInstance = new SharedDataMap("xpcshell", { + path: PATH, + isParent: false, + }); + + Assert.equal(childInstance.has("baz"), false, "Should not fail"); + + await parentInstance.init(); + parentInstance.set("baz", { bar: 1 }); + + await TestUtils.waitForCondition( + () => childInstance.get("baz"), + "Wait for child to sync" + ); + + Assert.deepEqual( + childInstance.get("baz"), + parentInstance.get("baz"), + "Parent and child should be in sync" + ); +}); + +with_sharedDataMap(async function test_updateStoreData({ instance, sandbox }) { + await instance.init(); + + Assert.ok(!instance.get("foo"), "No value initially"); + + instance.set("foo", "foo"); + instance.set("bar", "bar"); + instance._removeEntriesByKeys(["bar"]); + + Assert.ok(instance.get("foo"), "We keep one of the values"); + Assert.ok(!instance.get("bar"), "The other value is removed"); +}); diff --git a/toolkit/components/nimbus/test/unit/test_localization.js b/toolkit/components/nimbus/test/unit/test_localization.js new file mode 100644 index 0000000000..1e950941a3 --- /dev/null +++ b/toolkit/components/nimbus/test/unit/test_localization.js @@ -0,0 +1,1401 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { ExperimentAPI, _ExperimentFeature: ExperimentFeature } = + ChromeUtils.importESModule("resource://nimbus/ExperimentAPI.sys.mjs"); +const { ExperimentFakes, ExperimentTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/NimbusTestUtils.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); +const { TelemetryEvents } = ChromeUtils.importESModule( + "resource://normandy/lib/TelemetryEvents.sys.mjs" +); + +const LOCALIZATIONS = { + "en-US": { + foo: "localized foo text", + qux: "localized qux text", + grault: "localized grault text", + waldo: "localized waldo text", + }, +}; + +const DEEPLY_NESTED_VALUE = { + foo: { + $l10n: { + id: "foo", + comment: "foo comment", + text: "original foo text", + }, + }, + bar: { + qux: { + $l10n: { + id: "qux", + comment: "qux comment", + text: "original qux text", + }, + }, + quux: { + grault: { + $l10n: { + id: "grault", + comment: "grault comment", + text: "orginal grault text", + }, + }, + garply: "original garply text", + }, + corge: "original corge text", + }, + baz: "original baz text", + waldo: [ + { + $l10n: { + id: "waldo", + comment: "waldo comment", + text: "original waldo text", + }, + }, + ], +}; + +const LOCALIZED_DEEPLY_NESTED_VALUE = { + foo: "localized foo text", + bar: { + qux: "localized qux text", + quux: { + grault: "localized grault text", + garply: "original garply text", + }, + corge: "original corge text", + }, + baz: "original baz text", + waldo: ["localized waldo text"], +}; + +const FEATURE_ID = "testfeature1"; +const TEST_PREF_BRANCH = "testfeature1."; +const FEATURE = new ExperimentFeature(FEATURE_ID, { + isEarlyStartup: false, + variables: { + foo: { + type: "string", + fallbackPref: `${TEST_PREF_BRANCH}foo`, + }, + bar: { + type: "json", + fallbackPref: `${TEST_PREF_BRANCH}bar`, + }, + baz: { + type: "string", + fallbackPref: `${TEST_PREF_BRANCH}baz`, + }, + waldo: { + type: "json", + fallbackPref: `${TEST_PREF_BRANCH}waldo`, + }, + }, +}); + +/** + * Remove the experiment store. + */ +async function cleanupStore(store) { + // We need to call finalize first to ensure that any pending saves from + // JSONFile.saveSoon overwrite files on disk. + await store._store.finalize(); + await IOUtils.remove(store._store.path); +} + +function resetTelemetry() { + Services.fog.testResetFOG(); + Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + /* clear = */ true + ); +} + +add_setup(function setup() { + do_get_profile(); + + Services.fog.initializeFOG(); + TelemetryEvents.init(); + + registerCleanupFunction(ExperimentTestUtils.addTestFeatures(FEATURE)); + registerCleanupFunction(resetTelemetry); +}); + +add_task(async function test_schema() { + const recipe = ExperimentFakes.recipe("foo"); + + info("Testing recipe without a localizations entry"); + await ExperimentTestUtils.validateExperiment(recipe); + + info("Testing recipe with a 'null' localizations entry"); + await ExperimentTestUtils.validateExperiment({ + ...recipe, + localizations: null, + }); + + info("Testing recipe with a valid localizations entry"); + await ExperimentTestUtils.validateExperiment({ + ...recipe, + localizations: LOCALIZATIONS, + }); + + info("Testing recipe with an invalid localizations entry"); + await Assert.rejects( + ExperimentTestUtils.validateExperiment({ + ...recipe, + localizations: [], + }), + /Experiment foo not valid/ + ); +}); + +add_task(function test_substituteLocalizations() { + Assert.equal( + ExperimentFeature.substituteLocalizations("string", LOCALIZATIONS["en-US"]), + "string", + "String values should not be subsituted" + ); + + Assert.equal( + ExperimentFeature.substituteLocalizations( + { + $l10n: { + id: "foo", + comment: "foo comment", + text: "original foo text", + }, + }, + LOCALIZATIONS["en-US"] + ), + "localized foo text", + "$l10n objects should be substituted" + ); + + Assert.deepEqual( + ExperimentFeature.substituteLocalizations( + DEEPLY_NESTED_VALUE, + LOCALIZATIONS["en-US"] + ), + LOCALIZED_DEEPLY_NESTED_VALUE, + "Supports nested substitutions" + ); + + Assert.throws( + () => + ExperimentFeature.substituteLocalizations( + { + foo: { + $l10n: { + id: "BOGUS", + comment: "A variable with a missing id", + text: "Original text", + }, + }, + }, + LOCALIZATIONS["en-US"] + ), + ex => ex.reason === "l10n-missing-entry" + ); +}); + +add_task(async function test_getLocalizedValue() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + }, + ], + localizations: LOCALIZATIONS, + }); + + const { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(experiment); + await enrollmentPromise; + + const enrollment = manager.store.getExperimentForFeature(FEATURE_ID); + + Assert.deepEqual( + FEATURE._getLocalizedValue(enrollment), + LOCALIZED_DEEPLY_NESTED_VALUE, + "_getLocalizedValue() for all values" + ); + + Assert.deepEqual( + FEATURE._getLocalizedValue(enrollment, "foo"), + LOCALIZED_DEEPLY_NESTED_VALUE.foo, + "_getLocalizedValue() with a top-level localized variable" + ); + + Assert.deepEqual( + FEATURE._getLocalizedValue(enrollment, "bar"), + LOCALIZED_DEEPLY_NESTED_VALUE.bar, + "_getLocalizedValue() with a nested localization" + ); + + await doExperimentCleanup(); + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_getLocalizedValue_unenroll_missingEntry() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + bar: { + $l10n: { + id: "BOGUS", + comment: "Bogus localization", + text: "Original text", + }, + }, + }, + }, + ], + }, + ], + localizations: LOCALIZATIONS, + }); + + await ExperimentFakes.enrollmentHelper(experiment).enrollmentPromise; + + const enrollment = manager.store.getExperimentForFeature(FEATURE_ID); + + Assert.deepEqual( + FEATURE._getLocalizedValue(enrollment), + undefined, + "_getLocalizedValue() with a bogus localization" + ); + + Assert.equal( + manager.store.getExperimentForFeature(FEATURE_ID), + null, + "Experiment should be unenrolled" + ); + + const gleanEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal(gleanEvents.length, 1, "Should be one unenrollment event"); + Assert.equal( + gleanEvents[0].extra.reason, + "l10n-missing-entry", + "Reason should match" + ); + Assert.equal( + gleanEvents[0].extra.experiment, + "experiment", + "Slug should match" + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "experiment", + extra: { reason: "l10n-missing-entry" }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_getLocalizedValue_unenroll_missingEntry() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + bar: { + $l10n: { + id: "BOGUS", + comment: "Bogus localization", + text: "Original text", + }, + }, + }, + }, + ], + }, + ], + localizations: { + "en-CA": {}, + }, + }); + + await ExperimentFakes.enrollmentHelper(experiment).enrollmentPromise; + + const enrollment = manager.store.getExperimentForFeature(FEATURE_ID); + + Assert.deepEqual( + FEATURE._getLocalizedValue(enrollment), + undefined, + "_getLocalizedValue() with a bogus localization" + ); + + Assert.equal( + manager.store.getExperimentForFeature(FEATURE_ID), + null, + "Experiment should be unenrolled" + ); + + const gleanEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal(gleanEvents.length, 1, "Should be one unenrollment event"); + Assert.equal( + gleanEvents[0].extra.reason, + "l10n-missing-locale", + "Reason should match" + ); + Assert.equal( + gleanEvents[0].extra.experiment, + "experiment", + "Slug should match" + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "experiment", + extra: { reason: "l10n-missing-locale" }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_getVariables() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + const experiment = ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + }, + ], + localizations: LOCALIZATIONS, + }); + + const { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(experiment); + await enrollmentPromise; + + Assert.deepEqual( + FEATURE.getAllVariables(), + LOCALIZED_DEEPLY_NESTED_VALUE, + "getAllVariables() returns subsituted values" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + LOCALIZED_DEEPLY_NESTED_VALUE.foo, + "getVariable() returns a top-level substituted value" + ); + + Assert.deepEqual( + FEATURE.getVariable("bar"), + LOCALIZED_DEEPLY_NESTED_VALUE.bar, + "getVariable() returns a nested substitution" + ); + + Assert.deepEqual( + FEATURE.getVariable("baz"), + DEEPLY_NESTED_VALUE.baz, + "getVariable() returns non-localized variables unmodified" + ); + + Assert.deepEqual( + FEATURE.getVariable("waldo"), + LOCALIZED_DEEPLY_NESTED_VALUE.waldo, + "getVariable() returns substitutions inside arrays" + ); + + await doExperimentCleanup(); + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_getVariables_fallback() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + Services.prefs.setStringPref( + FEATURE.manifest.variables.foo.fallbackPref, + "fallback-foo-pref-value" + ); + Services.prefs.setStringPref( + FEATURE.manifest.variables.baz.fallbackPref, + "fallback-baz-pref-value" + ); + + const recipes = { + experiment: ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + foo: DEEPLY_NESTED_VALUE.foo, + }, + }, + ], + }, + ], + localizations: { + "en-US": { + foo: LOCALIZATIONS["en-US"].foo, + }, + }, + }), + + rollout: ExperimentFakes.recipe("rollout", { + isRollout: true, + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + bar: DEEPLY_NESTED_VALUE.bar, + }, + }, + ], + }, + ], + localizations: { + "en-US": { + qux: LOCALIZATIONS["en-US"].qux, + grault: LOCALIZATIONS["en-US"].grault, + }, + }, + }), + }; + + const cleanup = {}; + + Assert.deepEqual( + FEATURE.getAllVariables({ defaultValues: { waldo: ["default-value"] } }), + { + foo: "fallback-foo-pref-value", + bar: null, + baz: "fallback-baz-pref-value", + waldo: ["default-value"], + }, + "getAllVariables() returns only values from prefs and defaults" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + "fallback-foo-pref-value", + "variable foo returned from prefs" + ); + Assert.equal( + FEATURE.getVariable("bar"), + undefined, + "variable bar returned from rollout" + ); + Assert.equal( + FEATURE.getVariable("baz"), + "fallback-baz-pref-value", + "variable baz returned from prefs" + ); + + // Enroll in the rollout. + { + const { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(recipes.rollout); + await enrollmentPromise; + + cleanup.rollout = doExperimentCleanup; + } + + Assert.deepEqual( + FEATURE.getAllVariables({ defaultValues: { waldo: ["default-value"] } }), + { + foo: "fallback-foo-pref-value", + bar: LOCALIZED_DEEPLY_NESTED_VALUE.bar, + baz: "fallback-baz-pref-value", + waldo: ["default-value"], + }, + "getAllVariables() returns subsituted values from the rollout" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + "fallback-foo-pref-value", + "variable foo returned from prefs" + ); + Assert.deepEqual( + FEATURE.getVariable("bar"), + LOCALIZED_DEEPLY_NESTED_VALUE.bar, + "variable bar returned from rollout" + ); + Assert.equal( + FEATURE.getVariable("baz"), + "fallback-baz-pref-value", + "variable baz returned from prefs" + ); + + // Enroll in the experiment. + { + const { enrollmentPromise, doExperimentCleanup } = + ExperimentFakes.enrollmentHelper(recipes.experiment); + await enrollmentPromise; + + cleanup.experiment = doExperimentCleanup; + } + + Assert.deepEqual( + FEATURE.getAllVariables({ defaultValues: { waldo: ["default-value"] } }), + { + foo: LOCALIZED_DEEPLY_NESTED_VALUE.foo, + bar: null, + baz: "fallback-baz-pref-value", + waldo: ["default-value"], + }, + "getAllVariables() returns subsituted values from the experiment" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + LOCALIZED_DEEPLY_NESTED_VALUE.foo, + "variable foo returned from experiment" + ); + Assert.deepEqual( + FEATURE.getVariable("bar"), + LOCALIZED_DEEPLY_NESTED_VALUE.bar, + "variable bar returned from rollout" + ); + Assert.equal( + FEATURE.getVariable("baz"), + "fallback-baz-pref-value", + "variable baz returned from prefs" + ); + + // Unenroll from the rollout so we are only enrolled in an experiment. + await cleanup.rollout(); + + Assert.deepEqual( + FEATURE.getAllVariables({ defaultValues: { waldo: ["default-value"] } }), + { + foo: LOCALIZED_DEEPLY_NESTED_VALUE.foo, + bar: null, + baz: "fallback-baz-pref-value", + waldo: ["default-value"], + }, + "getAllVariables() returns substituted values from the experiment" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + LOCALIZED_DEEPLY_NESTED_VALUE.foo, + "variable foo returned from experiment" + ); + Assert.equal( + FEATURE.getVariable("bar"), + undefined, + "variable bar is not set" + ); + Assert.equal( + FEATURE.getVariable("baz"), + "fallback-baz-pref-value", + "variable baz returned from prefs" + ); + + // Unenroll from experiment. We are enrolled in nothing. + await cleanup.experiment(); + + Assert.deepEqual( + FEATURE.getAllVariables({ defaultValues: { waldo: ["default-value"] } }), + { + foo: "fallback-foo-pref-value", + bar: null, + baz: "fallback-baz-pref-value", + waldo: ["default-value"], + }, + "getAllVariables() returns only values from prefs and defaults" + ); + + Assert.equal( + FEATURE.getVariable("foo"), + "fallback-foo-pref-value", + "variable foo returned from prefs" + ); + Assert.equal( + FEATURE.getVariable("bar"), + undefined, + "variable bar returned from rollout" + ); + Assert.equal( + FEATURE.getVariable("baz"), + "fallback-baz-pref-value", + "variable baz returned from prefs" + ); + + Services.prefs.clearUserPref(FEATURE.manifest.variables.foo.fallbackPref); + Services.prefs.clearUserPref(FEATURE.manifest.variables.baz.fallbackPref); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_getVariables_fallback_unenroll() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + + await manager.onStartup(); + await manager.store.ready(); + + Services.prefs.setStringPref( + FEATURE.manifest.variables.foo.fallbackPref, + "fallback-foo-pref-value" + ); + Services.prefs.setStringPref( + FEATURE.manifest.variables.bar.fallbackPref, + `"fallback-bar-pref-value"` + ); + Services.prefs.setStringPref( + FEATURE.manifest.variables.baz.fallbackPref, + "fallback-baz-pref-value" + ); + Services.prefs.setStringPref( + FEATURE.manifest.variables.waldo.fallbackPref, + JSON.stringify(["fallback-waldo-pref-value"]) + ); + + const recipes = [ + ExperimentFakes.recipe("experiment", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + foo: DEEPLY_NESTED_VALUE.foo, + }, + }, + ], + }, + ], + localizations: {}, + }), + + ExperimentFakes.recipe("rollout", { + isRollout: true, + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: { + bar: DEEPLY_NESTED_VALUE.bar, + }, + }, + ], + }, + ], + localizations: { + "en-US": {}, + }, + }), + ]; + + for (const recipe of recipes) { + await ExperimentFakes.enrollmentHelper(recipe).enrollmentPromise; + } + + Assert.deepEqual(FEATURE.getAllVariables(), { + foo: "fallback-foo-pref-value", + bar: "fallback-bar-pref-value", + baz: "fallback-baz-pref-value", + waldo: ["fallback-waldo-pref-value"], + }); + + Assert.equal( + manager.store.getExperimentForFeature(FEATURE_ID), + null, + "Experiment should be unenrolled" + ); + + Assert.equal( + manager.store.getRolloutForFeature(FEATURE_ID), + null, + "Rollout should be unenrolled" + ); + + const gleanEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal(gleanEvents.length, 2, "Should be two unenrollment events"); + Assert.equal( + gleanEvents[0].extra.reason, + "l10n-missing-locale", + "Reason should match" + ); + Assert.equal( + gleanEvents[0].extra.experiment, + "experiment", + "Slug should match" + ); + Assert.equal( + gleanEvents[1].extra.reason, + "l10n-missing-entry", + "Reason should match" + ); + Assert.equal(gleanEvents[1].extra.experiment, "rollout", "Slug should match"); + + TelemetryTestUtils.assertEvents( + [ + { + value: "experiment", + extra: { reason: "l10n-missing-locale" }, + }, + { + value: "rollout", + extra: { reason: "l10n-missing-entry" }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + } + ); + + Services.prefs.clearUserPref(FEATURE.manifest.variables.foo.fallbackPref); + Services.prefs.clearUserPref(FEATURE.manifest.variables.bar.fallbackPref); + Services.prefs.clearUserPref(FEATURE.manifest.variables.baz.fallbackPref); + Services.prefs.clearUserPref(FEATURE.manifest.variables.waldo.fallbackPref); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_updateRecipes() { + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + + loader.manager = manager; + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.stub(manager, "onRecipe"); + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + ratio: 1, + }, + ], + localizations: LOCALIZATIONS, + }); + + await loader.init(); + + await manager.onStartup(); + await manager.store.ready(); + + sandbox.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + await loader.updateRecipes(); + + Assert.ok(manager.onRecipe.calledOnce, "Enrolled"); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +async function test_updateRecipes_missingLocale({ + featureValidationOptOut = false, + validationEnabled = true, +} = {}) { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + + loader.manager = manager; + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.stub(manager, "onRecipe"); + sandbox.spy(manager, "onFinalize"); + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + ratio: 1, + }, + ], + localizations: {}, + featureValidationOptOut, + }); + + await loader.init(); + + await manager.onStartup(); + await manager.store.ready(); + + sandbox.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + await loader.updateRecipes(); + + Assert.ok(!manager.onRecipe.called, "Did not enroll in the recipe"); + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: ["foo"], + missingL10nIds: new Map(), + locale: "en-US", + validationEnabled, + }), + "should call .onFinalize with missing locale" + ); + + const gleanEvents = Glean.nimbusEvents.validationFailed.testGetValue(); + Assert.equal(gleanEvents.length, 1, "Should be one validationFailed event"); + Assert.equal( + gleanEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + gleanEvents[0].extra.reason, + "l10n-missing-locale", + "Reason should match" + ); + Assert.equal(gleanEvents[0].extra.locale, "en-US", "Locale should match"); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + }, + ], + { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +} + +add_task(test_updateRecipes_missingLocale); + +add_task(async function test_updateRecipes_missingEntry() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + + loader.manager = manager; + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.stub(manager, "onRecipe"); + sandbox.spy(manager, "onFinalize"); + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + ratio: 1, + }, + ], + localizations: { + "en-US": {}, + }, + }); + + await loader.init(); + + await manager.onStartup(); + await manager.store.ready(); + + sandbox.stub(loader.remoteSettingsClient, "get").resolves([recipe]); + await loader.updateRecipes(); + + Assert.ok(!manager.onRecipe.called, "Did not enroll in the recipe"); + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map([["foo", ["foo", "qux", "grault", "waldo"]]]), + locale: "en-US", + validationEnabled: true, + }), + "should call .onFinalize with missing locale" + ); + + const gleanEvents = Glean.nimbusEvents.validationFailed.testGetValue(); + Assert.equal(gleanEvents.length, 1, "Should be one validationFailed event"); + Assert.equal( + gleanEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + gleanEvents[0].extra.reason, + "l10n-missing-entry", + "Reason should match" + ); + Assert.equal( + gleanEvents[0].extra.l10n_ids, + "foo,qux,grault,waldo", + "Missing IDs should match" + ); + Assert.equal(gleanEvents[0].extra.locale, "en-US", "Locale should match"); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + extra: { + reason: "l10n-missing-entry", + locale: "en-US", + l10n_ids: "foo,qux,grault,waldo", + }, + }, + ], + { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_updateRecipes_validationDisabled_pref() { + resetTelemetry(); + + Services.prefs.setBoolPref("nimbus.validation.enabled", false); + + await test_updateRecipes_missingLocale({ validationEnabled: false }); + + Services.prefs.clearUserPref("nimbus.validation.enabled"); +}); + +add_task(async function test_updateRecipes_validationDisabled_flag() { + resetTelemetry(); + + await test_updateRecipes_missingLocale({ featureValidationOptOut: true }); +}); + +add_task(async function test_updateRecipes_unenroll_missingEntry() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + + loader.manager = manager; + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.spy(manager, "onRecipe"); + sandbox.spy(manager, "onFinalize"); + sandbox.spy(manager, "unenroll"); + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + ratio: 1, + }, + ], + localizations: LOCALIZATIONS, + }); + + await loader.init(); + + await manager.onStartup(); + await manager.store.ready(); + + await ExperimentFakes.enrollmentHelper(recipe, { + source: "rs-loader", + }).enrollmentPromise; + Assert.ok( + !!manager.store.getExperimentForFeature(FEATURE_ID), + "Should be enrolled in the experiment" + ); + + const badRecipe = { ...recipe, localizations: { "en-US": {} } }; + + sandbox.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + + await loader.updateRecipes(); + + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: [], + missingL10nIds: new Map([ + [recipe.slug, ["foo", "qux", "grault", "waldo"]], + ]), + locale: "en-US", + validationEnabled: true, + }), + "should call .onFinalize with missing l10n entry" + ); + + Assert.ok(manager.unenroll.calledWith(recipe.slug, "l10n-missing-entry")); + + Assert.equal( + manager.store.getExperimentForFeature(FEATURE_ID), + null, + "Should no longer be enrolled in the experiment" + ); + + const unenrollEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal(unenrollEvents.length, 1, "Should be one unenroll event"); + Assert.equal( + unenrollEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + unenrollEvents[0].extra.reason, + "l10n-missing-entry", + "Reason should match" + ); + + const validationFailedEvents = + Glean.nimbusEvents.validationFailed.testGetValue(); + Assert.equal( + validationFailedEvents.length, + 1, + "Should be one validation failed event" + ); + Assert.equal( + validationFailedEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + validationFailedEvents[0].extra.reason, + "l10n-missing-entry", + "Reason should match" + ); + Assert.equal( + validationFailedEvents[0].extra.l10n_ids, + "foo,qux,grault,waldo", + "Missing IDs should match" + ); + Assert.equal( + validationFailedEvents[0].extra.locale, + "en-US", + "Locale should match" + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + extra: { + reason: "l10n-missing-entry", + }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + }, + { clear: false } + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + extra: { + reason: "l10n-missing-entry", + l10n_ids: "foo,qux,grault,waldo", + locale: "en-US", + }, + }, + ], + { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +}); + +add_task(async function test_updateRecipes_unenroll_missingLocale() { + resetTelemetry(); + + const sandbox = sinon.createSandbox(); + const manager = ExperimentFakes.manager(); + const loader = ExperimentFakes.rsLoader(); + + loader.manager = manager; + sandbox.stub(ExperimentAPI, "_manager").get(() => manager); + sandbox.stub(ExperimentAPI, "_store").get(() => manager.store); + sandbox.spy(manager, "onRecipe"); + sandbox.spy(manager, "onFinalize"); + sandbox.spy(manager, "unenroll"); + + const recipe = ExperimentFakes.recipe("foo", { + branches: [ + { + slug: "control", + features: [ + { + featureId: FEATURE_ID, + value: DEEPLY_NESTED_VALUE, + }, + ], + ratio: 1, + }, + ], + localizations: LOCALIZATIONS, + }); + + await loader.init(); + + await manager.onStartup(); + await manager.store.ready(); + + await ExperimentFakes.enrollmentHelper(recipe, { + source: "rs-loader", + }).enrollmentPromise; + Assert.ok( + !!manager.store.getExperimentForFeature(FEATURE_ID), + "Should be enrolled in the experiment" + ); + + const badRecipe = { + ...recipe, + localizations: {}, + }; + + sandbox.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]); + + await loader.updateRecipes(); + + Assert.ok( + onFinalizeCalled(manager.onFinalize, "rs-loader", { + recipeMismatches: [], + invalidRecipes: [], + invalidBranches: new Map(), + invalidFeatures: new Map(), + missingLocale: ["foo"], + missingL10nIds: new Map(), + locale: "en-US", + validationEnabled: true, + }), + "should call .onFinalize with missing locale" + ); + + Assert.ok(manager.unenroll.calledWith(recipe.slug, "l10n-missing-locale")); + + Assert.equal( + manager.store.getExperimentForFeature(FEATURE_ID), + null, + "Should no longer be enrolled in the experiment" + ); + + const unenrollEvents = Glean.nimbusEvents.unenrollment.testGetValue(); + Assert.equal(unenrollEvents.length, 1, "Should be one unenroll event"); + Assert.equal( + unenrollEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + unenrollEvents[0].extra.reason, + "l10n-missing-locale", + "Reason should match" + ); + + const validationFailedEvents = + Glean.nimbusEvents.validationFailed.testGetValue(); + Assert.equal( + validationFailedEvents.length, + 1, + "Should be one validation failed event" + ); + Assert.equal( + validationFailedEvents[0].extra.experiment, + "foo", + "Experiment slug should match" + ); + Assert.equal( + validationFailedEvents[0].extra.reason, + "l10n-missing-locale", + "Reason should match" + ); + Assert.equal( + validationFailedEvents[0].extra.locale, + "en-US", + "Locale should match" + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + extra: { + reason: "l10n-missing-locale", + }, + }, + ], + { + category: "normandy", + method: "unenroll", + object: "nimbus_experiment", + }, + { clear: false } + ); + + TelemetryTestUtils.assertEvents( + [ + { + value: "foo", + extra: { + reason: "l10n-missing-locale", + locale: "en-US", + }, + }, + ], + { + category: "normandy", + method: "validationFailed", + object: "nimbus_experiment", + } + ); + + await cleanupStore(manager.store); + sandbox.reset(); +}); diff --git a/toolkit/components/nimbus/test/unit/xpcshell.ini b/toolkit/components/nimbus/test/unit/xpcshell.ini new file mode 100644 index 0000000000..cbb26895ce --- /dev/null +++ b/toolkit/components/nimbus/test/unit/xpcshell.ini @@ -0,0 +1,28 @@ +[DEFAULT] +head = head.js +tags = nimbus +firefox-appdir = browser +support-files = + reference_aboutwelcome_experiment_content.json +skip-if = + toolkit == "android" + appname == "thunderbird" +run-sequentially = very high failure rate in parallel + +[test_ExperimentAPI.js] +[test_ExperimentAPI_ExperimentFeature.js] +[test_ExperimentAPI_ExperimentFeature_getAllVariables.js] +[test_ExperimentAPI_ExperimentFeature_getVariable.js] +[test_ExperimentAPI_NimbusFeatures.js] +[test_ExperimentManager_context.js] +[test_ExperimentManager_enroll.js] +[test_ExperimentManager_generateTestIds.js] +[test_ExperimentManager_lifecycle.js] +[test_ExperimentManager_prefs.js] +[test_ExperimentManager_unenroll.js] +[test_ExperimentStore.js] +[test_NimbusTestUtils.js] +[test_RemoteSettingsExperimentLoader.js] +[test_RemoteSettingsExperimentLoader_updateRecipes.js] +[test_SharedDataMap.js] +[test_localization.js] |