diff options
Diffstat (limited to 'toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js')
| -rw-r--r-- | toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js | 790 |
1 files changed, 790 insertions, 0 deletions
diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js new file mode 100644 index 0000000000..8a426c3669 --- /dev/null +++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js @@ -0,0 +1,790 @@ +/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set sts=2 sw=2 et tw=80: */ +"use strict"; + +// This test file verifies various scenarios related to the data migration +// from the JSONFile backend to the IDB backend. + +AddonTestUtils.init(this); + +// Create appInfo before importing any other jsm file, to prevent +// Services.appinfo to be cached before an appInfo.version is +// actually defined (which prevent failures to be triggered when +// the test run in a non nightly build). +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "42" +); + +const { getTrimmedString } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionTelemetry.sys.mjs" +); +const { ExtensionStorage } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorage.sys.mjs" +); +const { ExtensionStorageIDB } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageIDB.sys.mjs" +); +const { TelemetryController } = ChromeUtils.importESModule( + "resource://gre/modules/TelemetryController.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +const { IDB_MIGRATED_PREF_BRANCH, IDB_MIGRATE_RESULT_HISTOGRAM } = + ExtensionStorageIDB; +const CATEGORIES = ["success", "failure"]; +const EVENT_CATEGORY = "extensions.data"; +const EVENT_OBJECT = "storageLocal"; +const EVENT_METHOD = "migrateResult"; +const LEAVE_STORAGE_PREF = "extensions.webextensions.keepStorageOnUninstall"; +const LEAVE_UUID_PREF = "extensions.webextensions.keepUuidOnUninstall"; +const TELEMETRY_EVENTS_FILTER = { + category: "extensions.data", + method: "migrateResult", + object: "storageLocal", +}; + +async function createExtensionJSONFileWithData(extensionId, data) { + await ExtensionStorage.set(extensionId, data); + const jsonFile = await ExtensionStorage.getFile(extensionId); + await jsonFile._save(); + const oldStorageFilename = ExtensionStorage.getStorageFile(extensionId); + equal( + await IOUtils.exists(oldStorageFilename), + true, + "The old json file has been created" + ); + + return { jsonFile, oldStorageFilename }; +} + +function clearMigrationHistogram() { + const histogram = Services.telemetry.getHistogramById( + IDB_MIGRATE_RESULT_HISTOGRAM + ); + histogram.clear(); + equal( + histogram.snapshot().sum, + 0, + `No data recorded for histogram ${IDB_MIGRATE_RESULT_HISTOGRAM}` + ); +} + +function assertMigrationHistogramCount(category, expectedCount) { + const histogram = Services.telemetry.getHistogramById( + IDB_MIGRATE_RESULT_HISTOGRAM + ); + + equal( + histogram.snapshot().values[CATEGORIES.indexOf(category)], + expectedCount, + `Got the expected count on category "${category}" for histogram ${IDB_MIGRATE_RESULT_HISTOGRAM}` + ); +} + +function assertTelemetryEvents(expectedEvents) { + TelemetryTestUtils.assertEvents(expectedEvents, { + category: EVENT_CATEGORY, + method: EVENT_METHOD, + object: EVENT_OBJECT, + }); +} + +add_setup(async function setup() { + Services.prefs.setBoolPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF, true); + + await promiseStartupManager(); + + // Telemetry test setup needed to ensure that the builtin events are defined + // and they can be collected and verified. + await TelemetryController.testSetup(); + + // This is actually only needed on Android, because it does not properly support unified telemetry + // and so, if not enabled explicitly here, it would make these tests to fail when running on a + // non-Nightly build. + const oldCanRecordBase = Services.telemetry.canRecordBase; + Services.telemetry.canRecordBase = true; + registerCleanupFunction(() => { + Services.telemetry.canRecordBase = oldCanRecordBase; + }); + + // Clear any telemetry events collected so far. + Services.telemetry.clearEvents(); +}); + +// Test that for newly installed extension the IDB backend is enabled without +// any data migration. +add_task(async function test_no_migration_for_newly_installed_extensions() { + const EXTENSION_ID = "test-no-data-migration@mochi.test"; + + await createExtensionJSONFileWithData(EXTENSION_ID, { + test_old_data: "test_old_value", + }); + + const extension = ExtensionTestUtils.loadExtension({ + useAddonManager: "temporary", + manifest: { + permissions: ["storage"], + browser_specific_settings: { gecko: { id: EXTENSION_ID } }, + }, + async background() { + const data = await browser.storage.local.get(); + browser.test.assertEq( + Object.keys(data).length, + 0, + "Expect the storage.local store to be empty" + ); + browser.test.sendMessage("test-stored-data:done"); + }, + }); + + await extension.startup(); + equal( + ExtensionStorageIDB.isMigratedExtension(extension), + true, + "The newly installed test extension is marked as migrated" + ); + await extension.awaitMessage("test-stored-data:done"); + await extension.unload(); + + // Verify that no data migration have been needed on the newly installed + // extension, by asserting that no telemetry events has been collected. + await TelemetryTestUtils.assertEvents([], TELEMETRY_EVENTS_FILTER); +}); + +// Test that the data migration is still running for a newly installed extension +// if keepStorageOnUninstall is true. +add_task(async function test_data_migration_on_keep_storage_on_uninstall() { + Services.prefs.setBoolPref(LEAVE_STORAGE_PREF, true); + + // Store some fake data in the storage.local file backend before starting the extension. + const EXTENSION_ID = "new-extension-on-keep-storage-on-uninstall@mochi.test"; + await createExtensionJSONFileWithData(EXTENSION_ID, { + test_key_string: "test_value", + }); + + const extension = ExtensionTestUtils.loadExtension({ + useAddonManager: "temporary", + manifest: { + permissions: ["storage"], + browser_specific_settings: { gecko: { id: EXTENSION_ID } }, + }, + async background() { + const storedData = await browser.storage.local.get(); + browser.test.assertEq( + "test_value", + storedData.test_key_string, + "Got the expected data after the storage.local data migration" + ); + browser.test.sendMessage("storage-local-data-migrated"); + }, + }); + + await extension.startup(); + await extension.awaitMessage("storage-local-data-migrated"); + equal( + ExtensionStorageIDB.isMigratedExtension(extension), + true, + "The newly installed test extension is marked as migrated" + ); + await extension.unload(); + + // Verify that the expected telemetry has been recorded. + await TelemetryTestUtils.assertEvents( + [ + { + method: "migrateResult", + value: EXTENSION_ID, + extra: { + backend: "IndexedDB", + data_migrated: "y", + has_jsonfile: "y", + has_olddata: "y", + }, + }, + ], + TELEMETRY_EVENTS_FILTER + ); + + Services.prefs.clearUserPref(LEAVE_STORAGE_PREF); +}); + +// Test that the old data is migrated successfully to the new storage backend +// and that the original JSONFile has been renamed. +add_task(async function test_storage_local_data_migration() { + const EXTENSION_ID = "extension-to-be-migrated@mozilla.org"; + + // Keep the extension storage and the uuid on uninstall, to verify that no telemetry events + // are being sent for an already migrated extension. + Services.prefs.setBoolPref(LEAVE_STORAGE_PREF, true); + Services.prefs.setBoolPref(LEAVE_UUID_PREF, true); + + const data = { + test_key_string: "test_value1", + test_key_number: 1000, + test_nested_data: { + nested_key: true, + }, + }; + + // Store some fake data in the storage.local file backend before starting the extension. + const { oldStorageFilename } = await createExtensionJSONFileWithData( + EXTENSION_ID, + data + ); + + async function background() { + const storedData = await browser.storage.local.get(); + + browser.test.assertEq( + "test_value1", + storedData.test_key_string, + "Got the expected data after the storage.local data migration" + ); + browser.test.assertEq( + 1000, + storedData.test_key_number, + "Got the expected data after the storage.local data migration" + ); + browser.test.assertEq( + true, + storedData.test_nested_data.nested_key, + "Got the expected data after the storage.local data migration" + ); + + browser.test.sendMessage("storage-local-data-migrated"); + } + + clearMigrationHistogram(); + + let extensionDefinition = { + useAddonManager: "temporary", + manifest: { + permissions: ["storage"], + browser_specific_settings: { + gecko: { + id: EXTENSION_ID, + }, + }, + }, + }; + + let extension = ExtensionTestUtils.loadExtension(extensionDefinition); + + // Install the extension while the storage.local IDB backend is disabled. + Services.prefs.setBoolPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF, false); + await extension.startup(); + + ok( + !ExtensionStorageIDB.isMigratedExtension(extension), + "The test extension should be using the JSONFile backend" + ); + + // Enabled the storage.local IDB backend and upgrade the extension. + Services.prefs.setBoolPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF, true); + await extension.upgrade({ + ...extensionDefinition, + background, + }); + + await extension.awaitMessage("storage-local-data-migrated"); + + ok( + ExtensionStorageIDB.isMigratedExtension(extension), + "The test extension should be using the IndexedDB backend" + ); + + const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal( + extension.extension + ); + + const idbConn = await ExtensionStorageIDB.open(storagePrincipal); + + equal( + await idbConn.isEmpty(extension.extension), + false, + "Data stored in the ExtensionStorageIDB backend as expected" + ); + + equal( + await IOUtils.exists(oldStorageFilename), + false, + "The old json storage file name should not exist anymore" + ); + + equal( + await IOUtils.exists(`${oldStorageFilename}.migrated`), + true, + "The old json storage file name should have been renamed as .migrated" + ); + + equal( + Services.prefs.getBoolPref( + `${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, + false + ), + true, + `Got the ${IDB_MIGRATED_PREF_BRANCH} preference set to true as expected` + ); + + assertMigrationHistogramCount("success", 1); + assertMigrationHistogramCount("failure", 0); + + assertTelemetryEvents([ + { + method: "migrateResult", + value: EXTENSION_ID, + extra: { + backend: "IndexedDB", + data_migrated: "y", + has_jsonfile: "y", + has_olddata: "y", + }, + }, + ]); + + equal( + Services.prefs.getBoolPref( + `${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, + false + ), + true, + `${IDB_MIGRATED_PREF_BRANCH} should still be true on keepStorageOnUninstall=true` + ); + + // Upgrade the extension and check that no telemetry events are being sent + // for an already migrated extension. + await extension.upgrade({ + ...extensionDefinition, + background, + }); + + await extension.awaitMessage("storage-local-data-migrated"); + + // The histogram values are unmodified. + assertMigrationHistogramCount("success", 1); + assertMigrationHistogramCount("failure", 0); + + // No new telemetry events recorded for the extension. + const snapshot = Services.telemetry.snapshotEvents( + Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS, + true + ); + const filterByCategory = ([timestamp, category]) => + category === EVENT_CATEGORY; + + ok( + !snapshot.parent || snapshot.parent.filter(filterByCategory).length === 0, + "No telemetry events should be recorded for an already migrated extension" + ); + + Services.prefs.setBoolPref(LEAVE_STORAGE_PREF, false); + Services.prefs.setBoolPref(LEAVE_UUID_PREF, false); + + await extension.unload(); + + equal( + Services.prefs.getPrefType(`${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`), + Services.prefs.PREF_INVALID, + `Got the ${IDB_MIGRATED_PREF_BRANCH} preference has been cleared on addon uninstall` + ); +}); + +// Test that the extensionId included in the telemetry event is being trimmed down to 80 chars +// as expected. +add_task(async function test_extensionId_trimmed_in_telemetry_event() { + // Generated extensionId in email-like format, longer than 80 chars. + const EXTENSION_ID = `long.extension.id@${Array(80).fill("a").join("")}`; + + const data = { test_key_string: "test_value" }; + + // Store some fake data in the storage.local file backend before starting the extension. + await createExtensionJSONFileWithData(EXTENSION_ID, data); + + async function background() { + const storedData = await browser.storage.local.get("test_key_string"); + + browser.test.assertEq( + "test_value", + storedData.test_key_string, + "Got the expected data after the storage.local data migration" + ); + + browser.test.sendMessage("storage-local-data-migrated"); + } + + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + browser_specific_settings: { + gecko: { + id: EXTENSION_ID, + }, + }, + }, + background, + // We don't want the (default) startupReason ADDON_INSTALL because + // that automatically sets the migrated pref and skips migration. + startupReason: "APP_STARTUP", + }); + + await extension.startup(); + + await extension.awaitMessage("storage-local-data-migrated"); + + const expectedTrimmedExtensionId = getTrimmedString(EXTENSION_ID); + + equal( + expectedTrimmedExtensionId.length, + 80, + "The trimmed version of the extensionId should be 80 chars long" + ); + + assertTelemetryEvents([ + { + method: "migrateResult", + value: expectedTrimmedExtensionId, + extra: { + backend: "IndexedDB", + data_migrated: "y", + has_jsonfile: "y", + has_olddata: "y", + }, + }, + ]); + + await extension.unload(); +}); + +// Test that if the old JSONFile data file is corrupted and the old data +// can't be successfully migrated to the new storage backend, then: +// - the new storage backend for that extension is still initialized and enabled +// - any new data is being stored in the new backend +// - the old file is being renamed (with the `.corrupted` suffix that JSONFile.sys.mjs +// adds when it fails to load the data file) and still available on disk. +add_task(async function test_storage_local_corrupted_data_migration() { + const EXTENSION_ID = "extension-corrupted-data-migration@mozilla.org"; + + const invalidData = `{"test_key_string": "test_value1"`; + const oldStorageFilename = ExtensionStorage.getStorageFile(EXTENSION_ID); + + await IOUtils.makeDirectory( + PathUtils.join(PathUtils.profileDir, "browser-extension-data", EXTENSION_ID) + ); + + // Write the json file with some invalid data. + await IOUtils.writeUTF8(oldStorageFilename, invalidData, { flush: true }); + equal( + await IOUtils.readUTF8(oldStorageFilename), + invalidData, + "The old json file has been overwritten with invalid data" + ); + + async function background() { + const storedData = await browser.storage.local.get(); + + browser.test.assertEq( + Object.keys(storedData).length, + 0, + "No data should be found on invalid data migration" + ); + + await browser.storage.local.set({ + test_key_string_on_IDBBackend: "expected-value", + }); + + browser.test.sendMessage("storage-local-data-migrated-and-set"); + } + + clearMigrationHistogram(); + + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + browser_specific_settings: { + gecko: { + id: EXTENSION_ID, + }, + }, + }, + background, + // We don't want the (default) startupReason ADDON_INSTALL because + // that automatically sets the migrated pref and skips migration. + startupReason: "APP_STARTUP", + }); + + await extension.startup(); + + await extension.awaitMessage("storage-local-data-migrated-and-set"); + + const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal( + extension.extension + ); + + const idbConn = await ExtensionStorageIDB.open(storagePrincipal); + + equal( + await idbConn.isEmpty(extension.extension), + false, + "Data stored in the ExtensionStorageIDB backend as expected" + ); + + equal( + await IOUtils.exists(`${oldStorageFilename}.corrupt`), + true, + "The old json storage should still be available if failed to be read" + ); + + // The extension is still migrated successfully to the new backend if the file from the + // original json file was corrupted. + + equal( + Services.prefs.getBoolPref( + `${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, + false + ), + true, + `Got the ${IDB_MIGRATED_PREF_BRANCH} preference set to true as expected` + ); + + assertMigrationHistogramCount("success", 1); + assertMigrationHistogramCount("failure", 0); + + assertTelemetryEvents([ + { + method: "migrateResult", + value: EXTENSION_ID, + extra: { + backend: "IndexedDB", + data_migrated: "y", + has_jsonfile: "y", + has_olddata: "n", + }, + }, + ]); + + await extension.unload(); +}); + +// Test that if the data migration fails to store the old data into the IndexedDB backend +// then the expected telemetry histogram is being updated. +add_task(async function test_storage_local_data_migration_failure() { + const EXTENSION_ID = "extension-data-migration-failure@mozilla.org"; + + // Create the file under the expected directory tree. + const { jsonFile, oldStorageFilename } = + await createExtensionJSONFileWithData(EXTENSION_ID, {}); + + // Store a fake invalid value which is going to fail to be saved into IndexedDB + // (because it can't be cloned and it is going to raise a DataCloneError), which + // will trigger a data migration failure that we expect to increment the related + // telemetry histogram. + jsonFile.data.set("fake_invalid_key", function () {}); + + async function background() { + await browser.storage.local.set({ + test_key_string_on_JSONFileBackend: "expected-value", + }); + browser.test.sendMessage("storage-local-data-migrated-and-set"); + } + + clearMigrationHistogram(); + + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + browser_specific_settings: { + gecko: { + id: EXTENSION_ID, + }, + }, + }, + background, + // We don't want the (default) startupReason ADDON_INSTALL because + // that automatically sets the migrated pref and skips migration. + startupReason: "APP_STARTUP", + }); + + await extension.startup(); + + await extension.awaitMessage("storage-local-data-migrated-and-set"); + + const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal( + extension.extension + ); + + const idbConn = await ExtensionStorageIDB.open(storagePrincipal); + equal( + await idbConn.isEmpty(extension.extension), + true, + "No data stored in the ExtensionStorageIDB backend as expected" + ); + equal( + await IOUtils.exists(oldStorageFilename), + true, + "The old json storage should still be available if failed to be read" + ); + + await extension.unload(); + + assertTelemetryEvents([ + { + method: "migrateResult", + value: EXTENSION_ID, + extra: { + backend: "JSONFile", + data_migrated: "n", + error_name: "DataCloneError", + has_jsonfile: "y", + has_olddata: "y", + }, + }, + ]); + + assertMigrationHistogramCount("success", 0); + assertMigrationHistogramCount("failure", 1); +}); + +add_task(async function test_migration_aborted_on_shutdown() { + const EXTENSION_ID = "test-migration-aborted-on-shutdown@mochi.test"; + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + browser_specific_settings: { + gecko: { + id: EXTENSION_ID, + }, + }, + }, + // We don't want the (default) startupReason ADDON_INSTALL because + // that automatically sets the migrated pref and skips migration. + startupReason: "APP_STARTUP", + }); + + await extension.startup(); + + equal( + extension.extension.hasShutdown, + false, + "The extension is still running" + ); + + await extension.unload(); + equal(extension.extension.hasShutdown, true, "The extension has shutdown"); + + // Trigger a data migration after the extension has been unloaded. + const result = await ExtensionStorageIDB.selectBackend({ + extension: extension.extension, + }); + Assert.deepEqual( + result, + { backendEnabled: false }, + "Expect migration to have been aborted" + ); + TelemetryTestUtils.assertEvents( + [ + { + value: EXTENSION_ID, + extra: { + backend: "JSONFile", + error_name: "DataMigrationAbortedError", + }, + }, + ], + TELEMETRY_EVENTS_FILTER + ); +}); + +add_task(async function test_storage_local_data_migration_clear_pref() { + Services.prefs.clearUserPref(LEAVE_STORAGE_PREF); + Services.prefs.clearUserPref(LEAVE_UUID_PREF); + Services.prefs.clearUserPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF); + await promiseShutdownManager(); + await TelemetryController.testShutdown(); +}); + +add_task(async function setup_quota_manager_testing_prefs() { + Services.prefs.setBoolPref("dom.quotaManager.testing", true); + Services.prefs.setIntPref( + "dom.quotaManager.temporaryStorage.fixedLimit", + 100 + ); + await promiseQuotaManagerServiceReset(); +}); + +add_task( + // TODO: temporarily disabled because it currently perma-fails on + // android builds (Bug 1564871) + { skip_if: () => AppConstants.platform === "android" }, + // eslint-disable-next-line no-use-before-define + test_quota_exceeded_while_migrating_data +); +async function test_quota_exceeded_while_migrating_data() { + const EXT_ID = "test-data-migration-stuck@mochi.test"; + const dataSize = 1000 * 1024; + + await createExtensionJSONFileWithData(EXT_ID, { + data: new Array(dataSize).fill("x").join(""), + }); + + const extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + browser_specific_settings: { gecko: { id: EXT_ID } }, + }, + background() { + browser.test.onMessage.addListener(async (msg, dataSize) => { + if (msg !== "verify-stored-data") { + return; + } + const res = await browser.storage.local.get(); + browser.test.assertEq( + res.data && res.data.length, + dataSize, + "Got the expected data" + ); + browser.test.sendMessage("verify-stored-data:done"); + }); + + browser.test.sendMessage("bg-page:ready"); + }, + // We don't want the (default) startupReason ADDON_INSTALL because + // that automatically sets the migrated pref and skips migration. + startupReason: "APP_STARTUP", + }); + + await extension.startup(); + await extension.awaitMessage("bg-page:ready"); + + extension.sendMessage("verify-stored-data", dataSize); + await extension.awaitMessage("verify-stored-data:done"); + + await ok( + !ExtensionStorageIDB.isMigratedExtension(extension), + "The extension falls back to the JSONFile backend because of the migration failure" + ); + await extension.unload(); + + TelemetryTestUtils.assertEvents( + [ + { + value: EXT_ID, + extra: { + backend: "JSONFile", + error_name: "QuotaExceededError", + }, + }, + ], + TELEMETRY_EVENTS_FILTER + ); + + Services.prefs.clearUserPref("dom.quotaManager.temporaryStorage.fixedLimit"); + await promiseQuotaManagerServiceClear(); + Services.prefs.clearUserPref("dom.quotaManager.testing"); +} |