1 files changed, 929 insertions, 0 deletions
diff --git a/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js
new file mode 100644
index 0000000000..9fe8bafef6
--- /dev/null
+++ b/toolkit/components/nimbus/test/unit/test_RemoteSettingsExperimentLoader_updateRecipes.js
@@ -0,0 +1,929 @@
+"use strict";
+
+const { ExperimentFakes } = ChromeUtils.import(
+  "resource://testing-common/NimbusTestUtils.jsm"
+);
+const { FirstStartup } = ChromeUtils.importESModule(
+  "resource://gre/modules/FirstStartup.sys.mjs"
+);
+const { NimbusFeatures } = ChromeUtils.import(
+  "resource://nimbus/ExperimentAPI.jsm"
+);
+const { PanelTestProvider } = ChromeUtils.import(
+  "resource://activity-stream/lib/PanelTestProvider.jsm"
+);
+const { TelemetryTestUtils } = ChromeUtils.importESModule(
+  "resource://testing-common/TelemetryTestUtils.sys.mjs"
+);
+const { TelemetryEvents } = ChromeUtils.import(
+  "resource://normandy/lib/TelemetryEvents.jsm"
+);
+
+add_setup(async function setup() {
+  do_get_profile();
+  Services.fog.initializeFOG();
+});
+
+add_task(async function test_updateRecipes_activeExperiments() {
+  const manager = ExperimentFakes.manager();
+  const sandbox = sinon.createSandbox();
+  const recipe = ExperimentFakes.recipe("foo");
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+  const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo", {
+    targeting: `"${recipe.slug}" in activeExperiments`,
+  });
+  const onRecipe = sandbox.stub(manager, "onRecipe");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]);
+  sandbox.stub(manager.store, "ready").resolves();
+  sandbox.stub(manager.store, "getAllActive").returns([recipe]);
+
+  await loader.init();
+
+  ok(onRecipe.calledOnce, "Should match active experiments");
+});
+
+add_task(async function test_updateRecipes_isFirstRun() {
+  const manager = ExperimentFakes.manager();
+  const sandbox = sinon.createSandbox();
+  const recipe = ExperimentFakes.recipe("foo");
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+  const PASS_FILTER_RECIPE = { ...recipe, targeting: "isFirstStartup" };
+  const onRecipe = sandbox.stub(manager, "onRecipe");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]);
+  sandbox.stub(manager.store, "ready").resolves();
+  sandbox.stub(manager.store, "getAllActive").returns([recipe]);
+
+  // Pretend to be in the first startup
+  FirstStartup._state = FirstStartup.IN_PROGRESS;
+  await loader.init();
+
+  Assert.ok(onRecipe.calledOnce, "Should match first run");
+});
+
+add_task(async function test_updateRecipes_invalidFeatureId() {
+  const manager = ExperimentFakes.manager();
+  const sandbox = sinon.createSandbox();
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+
+  const badRecipe = ExperimentFakes.recipe("foo", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "invalid-feature-id",
+            value: { hello: "world" },
+          },
+        ],
+      },
+      {
+        slug: "treatment",
+        ratio: 1,
+        features: [
+          {
+            featureId: "invalid-feature-id",
+            value: { hello: "goodbye" },
+          },
+        ],
+      },
+    ],
+  });
+
+  const onRecipe = sandbox.stub(manager, "onRecipe");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]);
+  sandbox.stub(manager.store, "ready").resolves();
+  sandbox.stub(manager.store, "getAllActive").returns([]);
+
+  await loader.init();
+  ok(onRecipe.notCalled, "No recipes");
+});
+
+add_task(async function test_updateRecipes_invalidFeatureValue() {
+  const manager = ExperimentFakes.manager();
+  const sandbox = sinon.createSandbox();
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+
+  const badRecipe = ExperimentFakes.recipe("foo", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "spotlight",
+            value: {
+              id: "test-spotlight-invalid-1",
+            },
+          },
+        ],
+      },
+      {
+        slug: "treatment",
+        ratio: 1,
+        features: [
+          {
+            featureId: "spotlight",
+            value: {
+              id: "test-spotlight-invalid-2",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  const onRecipe = sandbox.stub(manager, "onRecipe");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]);
+  sandbox.stub(manager.store, "ready").resolves();
+  sandbox.stub(manager.store, "getAllActive").returns([]);
+
+  await loader.init();
+  ok(onRecipe.notCalled, "No recipes");
+});
+
+add_task(async function test_updateRecipes_invalidRecipe() {
+  const manager = ExperimentFakes.manager();
+  const sandbox = sinon.createSandbox();
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+
+  const badRecipe = ExperimentFakes.recipe("foo");
+  delete badRecipe.slug;
+
+  const onRecipe = sandbox.stub(manager, "onRecipe");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([badRecipe]);
+  sandbox.stub(manager.store, "ready").resolves();
+  sandbox.stub(manager.store, "getAllActive").returns([]);
+
+  await loader.init();
+  ok(onRecipe.notCalled, "No recipes");
+});
+
+add_task(async function test_updateRecipes_invalidRecipeAfterUpdate() {
+  Services.fog.testResetFOG();
+
+  const manager = ExperimentFakes.manager();
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+
+  const recipe = ExperimentFakes.recipe("foo");
+  const badRecipe = { ...recipe };
+  delete badRecipe.branches;
+
+  sinon.stub(loader, "setTimer");
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager, "onFinalize");
+
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+  sinon.stub(manager.store, "ready").resolves();
+  sinon.spy(loader, "updateRecipes");
+
+  await loader.init();
+
+  ok(loader.updateRecipes.calledOnce, "should call .updateRecipes");
+  equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once");
+  ok(
+    loader.manager.onRecipe.calledWith(recipe, "rs-loader"),
+    "should call .onRecipe with argument data"
+  );
+  equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once");
+  ok(
+    loader.manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with no mismatches or invalid recipes"
+  );
+
+  info("Replacing recipe with an invalid one");
+
+  loader.remoteSettingsClient.get.resolves([badRecipe]);
+
+  await loader.updateRecipes("timer");
+  equal(
+    loader.manager.onRecipe.callCount,
+    1,
+    "should not have called .onRecipe again"
+  );
+  equal(
+    loader.manager.onFinalize.callCount,
+    2,
+    "should have called .onFinalize again"
+  );
+
+  ok(
+    loader.manager.onFinalize.secondCall.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: ["foo"],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with an invalid recipe"
+  );
+});
+
+add_task(async function test_updateRecipes_invalidBranchAfterUpdate() {
+  const message = await PanelTestProvider.getMessages().then(msgs =>
+    msgs.find(m => m.id === "SPOTLIGHT_MESSAGE_93")
+  );
+
+  const manager = ExperimentFakes.manager();
+  const loader = ExperimentFakes.rsLoader();
+  loader.manager = manager;
+
+  const recipe = ExperimentFakes.recipe("foo", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "spotlight",
+            value: { ...message },
+          },
+        ],
+      },
+      {
+        slug: "treatment",
+        ratio: 1,
+        features: [
+          {
+            featureId: "spotlight",
+            value: { ...message },
+          },
+        ],
+      },
+    ],
+  });
+
+  const badRecipe = {
+    ...recipe,
+    branches: [
+      { ...recipe.branches[0] },
+      {
+        ...recipe.branches[1],
+        features: [
+          {
+            ...recipe.branches[1].features[0],
+            value: { ...message },
+          },
+        ],
+      },
+    ],
+  };
+  delete badRecipe.branches[1].features[0].value.template;
+
+  sinon.stub(loader, "setTimer");
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager, "onFinalize");
+
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+  sinon.stub(manager.store, "ready").resolves();
+  sinon.spy(loader, "updateRecipes");
+
+  await loader.init();
+
+  ok(loader.updateRecipes.calledOnce, "should call .updateRecipes");
+  equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once");
+  ok(
+    loader.manager.onRecipe.calledWith(recipe, "rs-loader"),
+    "should call .onRecipe with argument data"
+  );
+  equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once");
+  ok(
+    loader.manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with no mismatches or invalid recipes"
+  );
+
+  info("Replacing recipe with an invalid one");
+
+  loader.remoteSettingsClient.get.resolves([badRecipe]);
+
+  await loader.updateRecipes("timer");
+  equal(
+    loader.manager.onRecipe.callCount,
+    1,
+    "should not have called .onRecipe again"
+  );
+  equal(
+    loader.manager.onFinalize.callCount,
+    2,
+    "should have called .onFinalize again"
+  );
+
+  ok(
+    loader.manager.onFinalize.secondCall.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map([["foo", [badRecipe.branches[0].slug]]]),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with an invalid branch"
+  );
+});
+
+add_task(async function test_updateRecipes_simpleFeatureInvalidAfterUpdate() {
+  const loader = ExperimentFakes.rsLoader();
+  const manager = loader.manager;
+
+  const recipe = ExperimentFakes.recipe("foo");
+  const badRecipe = ExperimentFakes.recipe("foo", {
+    branches: [
+      {
+        ...recipe.branches[0],
+        features: [
+          {
+            featureId: "testFeature",
+            value: { testInt: "abc123", enabled: true },
+          },
+        ],
+      },
+      {
+        ...recipe.branches[1],
+        features: [
+          {
+            featureId: "testFeature",
+            value: { testInt: 456, enabled: true },
+          },
+        ],
+      },
+    ],
+  });
+
+  const EXPECTED_SCHEMA = {
+    $schema: "https://json-schema.org/draft/2019-09/schema",
+    title: "testFeature",
+    description: NimbusFeatures.testFeature.manifest.description,
+    type: "object",
+    properties: {
+      testInt: {
+        type: "integer",
+      },
+      enabled: {
+        type: "boolean",
+      },
+      testSetString: {
+        type: "string",
+      },
+    },
+    additionalProperties: true,
+  };
+
+  sinon.spy(loader, "updateRecipes");
+  sinon.spy(loader, "_generateVariablesOnlySchema");
+  sinon.stub(loader, "setTimer");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+  sinon.stub(manager, "onFinalize");
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager.store, "ready").resolves();
+
+  await loader.init();
+  ok(manager.onRecipe.calledOnce, "should call .updateRecipes");
+  equal(loader.manager.onRecipe.callCount, 1, "should call .onRecipe once");
+  ok(
+    loader.manager.onRecipe.calledWith(recipe, "rs-loader"),
+    "should call .onRecipe with argument data"
+  );
+  equal(loader.manager.onFinalize.callCount, 1, "should call .onFinalize once");
+  ok(
+    loader.manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with nomismatches or invalid recipes"
+  );
+
+  ok(
+    loader._generateVariablesOnlySchema.calledOnce,
+    "Should have generated a schema for testFeature"
+  );
+
+  Assert.deepEqual(
+    loader._generateVariablesOnlySchema.returnValues[0],
+    EXPECTED_SCHEMA,
+    "should have generated a schema with three fields"
+  );
+
+  info("Replacing recipe with an invalid one");
+
+  loader.remoteSettingsClient.get.resolves([badRecipe]);
+
+  await loader.updateRecipes("timer");
+  equal(
+    manager.onRecipe.callCount,
+    1,
+    "should not have called .onRecipe again"
+  );
+  equal(
+    manager.onFinalize.callCount,
+    2,
+    "should have called .onFinalize again"
+  );
+
+  ok(
+    loader.manager.onFinalize.secondCall.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map([["foo", [badRecipe.branches[0].slug]]]),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "should call .onFinalize with an invalid branch"
+  );
+});
+
+add_task(async function test_updateRecipes_validationTelemetry() {
+  TelemetryEvents.init();
+
+  Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    /* clear = */ true
+  );
+
+  const invalidRecipe = ExperimentFakes.recipe("invalid-recipe");
+  delete invalidRecipe.channel;
+
+  const invalidBranch = ExperimentFakes.recipe("invalid-branch");
+  invalidBranch.branches[0].features[0].value.testInt = "hello";
+  invalidBranch.branches[1].features[0].value.testInt = "world";
+
+  const invalidFeature = ExperimentFakes.recipe("invalid-feature", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "unknown-feature",
+            value: { foo: "bar" },
+          },
+          {
+            featureId: "second-unknown-feature",
+            value: { baz: "qux" },
+          },
+        ],
+      },
+    ],
+  });
+
+  const TEST_CASES = [
+    {
+      recipe: invalidRecipe,
+      reason: "invalid-recipe",
+      events: [{}],
+      callCount: 1,
+    },
+    {
+      recipe: invalidBranch,
+      reason: "invalid-branch",
+      events: invalidBranch.branches.map(branch => ({ branch: branch.slug })),
+      callCount: 2,
+    },
+    {
+      recipe: invalidFeature,
+      reason: "invalid-feature",
+      events: invalidFeature.branches[0].features.map(feature => ({
+        feature: feature.featureId,
+      })),
+      callCount: 2,
+    },
+  ];
+
+  const LEGACY_FILTER = {
+    category: "normandy",
+    method: "validationFailed",
+    object: "nimbus_experiment",
+  };
+
+  for (const { recipe, reason, events, callCount } of TEST_CASES) {
+    info(`Testing validation failed telemetry for reason = "${reason}" ...`);
+    const loader = ExperimentFakes.rsLoader();
+    const manager = loader.manager;
+
+    sinon.stub(loader, "setTimer");
+    sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+    sinon.stub(manager, "onRecipe");
+    sinon.stub(manager.store, "ready").resolves();
+    sinon.stub(manager.store, "getAllActive").returns([]);
+    sinon.stub(manager.store, "getAllRollouts").returns([]);
+
+    const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry");
+
+    await loader.init();
+
+    Assert.equal(
+      telemetrySpy.callCount,
+      callCount,
+      `Should call sendValidationFailedTelemetry ${callCount} times for reason ${reason}`
+    );
+
+    const gleanEvents = Glean.nimbusEvents.validationFailed
+      .testGetValue()
+      .map(event => {
+        event = { ...event };
+        // We do not care about the timestamp.
+        delete event.timestamp;
+        return event;
+      });
+
+    const expectedGleanEvents = events.map(event => ({
+      category: "nimbus_events",
+      name: "validation_failed",
+      extra: {
+        experiment: recipe.slug,
+        reason,
+        ...event,
+      },
+    }));
+
+    Assert.deepEqual(
+      gleanEvents,
+      expectedGleanEvents,
+      "Glean telemetry matches"
+    );
+
+    const expectedLegacyEvents = events.map(event => ({
+      ...LEGACY_FILTER,
+      value: recipe.slug,
+      extra: {
+        reason,
+        ...event,
+      },
+      LEGACY_FILTER,
+    }));
+
+    TelemetryTestUtils.assertEvents(expectedLegacyEvents, LEGACY_FILTER, {
+      clear: true,
+    });
+
+    Services.fog.testResetFOG();
+  }
+});
+
+add_task(async function test_updateRecipes_validationDisabled() {
+  Services.prefs.setBoolPref("nimbus.validation.enabled", false);
+
+  const invalidRecipe = ExperimentFakes.recipe("invalid-recipe");
+  delete invalidRecipe.channel;
+
+  const invalidBranch = ExperimentFakes.recipe("invalid-branch");
+  invalidBranch.branches[0].features[0].value.testInt = "hello";
+  invalidBranch.branches[1].features[0].value.testInt = "world";
+
+  const invalidFeature = ExperimentFakes.recipe("invalid-feature", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "unknown-feature",
+            value: { foo: "bar" },
+          },
+          {
+            featureId: "second-unknown-feature",
+            value: { baz: "qux" },
+          },
+        ],
+      },
+    ],
+  });
+
+  for (const recipe of [invalidRecipe, invalidBranch, invalidFeature]) {
+    const loader = ExperimentFakes.rsLoader();
+    const manager = loader.manager;
+
+    sinon.stub(loader, "setTimer");
+    sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+    sinon.stub(manager, "onRecipe");
+    sinon.stub(manager.store, "ready").resolves();
+    sinon.stub(manager.store, "getAllActive").returns([]);
+    sinon.stub(manager.store, "getAllRollouts").returns([]);
+
+    const finalizeStub = sinon.stub(manager, "onFinalize");
+    const telemetrySpy = sinon.spy(manager, "sendValidationFailedTelemetry");
+
+    await loader.init();
+
+    Assert.equal(
+      telemetrySpy.callCount,
+      0,
+      "Should not send validation failed telemetry"
+    );
+    Assert.ok(
+      finalizeStub.calledWith("rs-loader", {
+        recipeMismatches: [],
+        invalidRecipes: [],
+        invalidBranches: new Map(),
+        invalidFeatures: new Map(),
+        validationEnabled: false,
+      }),
+      "should call .onFinalize with no validation issues"
+    );
+  }
+  Services.prefs.clearUserPref("nimbus.validation.enabled");
+});
+
+add_task(async function test_updateRecipes_appId() {
+  const loader = ExperimentFakes.rsLoader();
+  const manager = loader.manager;
+
+  const recipe = ExperimentFakes.recipe("background-task-recipe", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "backgroundTaskMessage",
+            value: {},
+          },
+        ],
+      },
+    ],
+  });
+
+  sinon.stub(loader, "setTimer");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager, "onFinalize");
+  sinon.stub(manager.store, "ready").resolves();
+
+  info("Testing updateRecipes() with the default application ID");
+  await loader.init();
+
+  Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called");
+  Assert.ok(
+    manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "Should call .onFinalize with no validation issues"
+  );
+
+  info("Testing updateRecipes() with a custom application ID");
+
+  Services.prefs.setStringPref(
+    "nimbus.appId",
+    "firefox-desktop-background-task"
+  );
+
+  await loader.updateRecipes();
+  Assert.ok(
+    manager.onRecipe.calledWith(recipe, "rs-loader"),
+    `.onRecipe called with ${recipe.slug}`
+  );
+
+  Assert.ok(
+    manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "Should call .onFinalize with no validation issues"
+  );
+
+  Services.prefs.clearUserPref("nimbus.appId");
+});
+
+add_task(async function test_updateRecipes_withPropNotInManifest() {
+  // Need to randomize the slug so subsequent test runs don't skip enrollment
+  // due to a conflicting slug
+  const PASS_FILTER_RECIPE = ExperimentFakes.recipe("foo" + Math.random(), {
+    arguments: {},
+    branches: [
+      {
+        features: [
+          {
+            enabled: true,
+            featureId: "testFeature",
+            value: {
+              enabled: true,
+              testInt: 5,
+              testSetString: "foo",
+              additionalPropNotInManifest: 7,
+            },
+          },
+        ],
+        ratio: 1,
+        slug: "treatment-2",
+      },
+    ],
+    channel: "nightly",
+    schemaVersion: "1.9.0",
+    targeting: "true",
+  });
+
+  const loader = ExperimentFakes.rsLoader();
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([PASS_FILTER_RECIPE]);
+  sinon.stub(loader.manager, "onRecipe").resolves();
+  sinon.stub(loader.manager, "onFinalize");
+
+  await loader.init();
+
+  ok(
+    loader.manager.onRecipe.calledWith(PASS_FILTER_RECIPE, "rs-loader"),
+    "should call .onRecipe with this recipe"
+  );
+  equal(loader.manager.onRecipe.callCount, 1, "should only call onRecipe once");
+});
+
+add_task(async function test_updateRecipes_recipeAppId() {
+  const loader = ExperimentFakes.rsLoader();
+  const manager = loader.manager;
+
+  const recipe = ExperimentFakes.recipe("mobile-experiment", {
+    appId: "org.mozilla.firefox",
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "mobile-feature",
+            value: {
+              enabled: true,
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  sinon.stub(loader, "setTimer");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager, "onFinalize");
+  sinon.stub(manager.store, "ready").resolves();
+
+  await loader.init();
+
+  Assert.equal(manager.onRecipe.callCount, 0, ".onRecipe was never called");
+  Assert.ok(
+    manager.onFinalize.calledWith("rs-loader", {
+      recipeMismatches: [],
+      invalidRecipes: [],
+      invalidBranches: new Map(),
+      invalidFeatures: new Map(),
+      validationEnabled: true,
+    }),
+    "Should call .onFinalize with no validation issues"
+  );
+});
+
+add_task(async function test_updateRecipes_featureValidationOptOut() {
+  const invalidTestRecipe = ExperimentFakes.recipe("invalid-recipe", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "testFeature",
+            value: {
+              enabled: "true",
+              testInt: false,
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  const message = await PanelTestProvider.getMessages().then(msgs =>
+    msgs.find(m => m.id === "SPOTLIGHT_MESSAGE_93")
+  );
+  delete message.template;
+
+  const invalidMsgRecipe = ExperimentFakes.recipe("invalid-recipe", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "spotlight",
+            value: message,
+          },
+        ],
+      },
+    ],
+  });
+
+  for (const invalidRecipe of [invalidTestRecipe, invalidMsgRecipe]) {
+    const optOutRecipe = {
+      ...invalidMsgRecipe,
+      slug: "optout-recipe",
+      featureValidationOptOut: true,
+    };
+
+    const loader = ExperimentFakes.rsLoader();
+    const manager = loader.manager;
+
+    sinon.stub(loader, "setTimer");
+    sinon
+      .stub(loader.remoteSettingsClient, "get")
+      .resolves([invalidRecipe, optOutRecipe]);
+
+    sinon.stub(manager, "onRecipe");
+    sinon.stub(manager, "onFinalize");
+    sinon.stub(manager.store, "ready").resolves();
+    sinon.stub(manager.store, "getAllActive").returns([]);
+    sinon.stub(manager.store, "getAllRollouts").returns([]);
+
+    await loader.init();
+    ok(
+      manager.onRecipe.calledOnceWith(optOutRecipe, "rs-loader"),
+      "should call .onRecipe for the opt-out recipe"
+    );
+    ok(
+      manager.onFinalize.calledOnceWith("rs-loader", {
+        recipeMismatches: [],
+        invalidRecipes: [],
+        invalidBranches: new Map([[invalidRecipe.slug, ["control"]]]),
+        invalidFeatures: new Map(),
+        validationEnabled: true,
+      }),
+      "should call .onFinalize with only one invalid recipe"
+    );
+  }
+});
+
+add_task(async function test_updateRecipes_invalidFeature_mismatch() {
+  info(
+    "Testing that we do not submit validation telemetry when the targeting does not match"
+  );
+  const recipe = ExperimentFakes.recipe("recipe", {
+    branches: [
+      {
+        slug: "control",
+        ratio: 1,
+        features: [
+          {
+            featureId: "bogus",
+            value: {
+              bogus: "bogus",
+            },
+          },
+        ],
+      },
+    ],
+    targeting: "false",
+  });
+
+  const loader = ExperimentFakes.rsLoader();
+  const manager = loader.manager;
+
+  sinon.stub(loader, "setTimer");
+  sinon.stub(loader.remoteSettingsClient, "get").resolves([recipe]);
+
+  sinon.stub(manager, "onRecipe");
+  sinon.stub(manager, "onFinalize");
+  sinon.stub(manager.store, "ready").resolves();
+  sinon.stub(manager.store, "getAllActive").returns([]);
+  sinon.stub(manager.store, "getAllRollouts").returns([]);
+
+  const telemetrySpy = sinon.stub(manager, "sendValidationFailedTelemetry");
+  const targetingSpy = sinon.spy(loader, "checkTargeting");
+
+  await loader.init();
+  ok(targetingSpy.calledOnce, "Should have checked targeting for recipe");
+  ok(
+    !(await targetingSpy.returnValues[0]),
+    "Targeting should not have matched"
+  );
+  ok(manager.onRecipe.notCalled, "should not call .onRecipe for the recipe");
+  ok(
+    telemetrySpy.notCalled,
+    "Should not have submitted validation failed telemetry"
+  );
+});