From 2aa4a82499d4becd2284cdb482213d541b8804dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 16:29:10 +0200 Subject: Adding upstream version 86.0.1. Signed-off-by: Daniel Baumann --- services/settings/test/moz.build | 7 + services/settings/test/unit/moz.build | 7 + .../test/unit/test_attachments_downloader.js | 573 ++++++++ .../65650a0f-7c22-4c10-9744-2d67e301f5f4.pem | 26 + .../dump-collection/filename-of-dump.txt | 1 + .../dump-collection/filename-of-dump.txt.meta.json | 10 + .../filename-without-content.txt.meta.json | 9 + .../dump-collection/filename-without-meta.txt | 1 + .../settings/test/unit/test_remote_settings.js | 1417 ++++++++++++++++++++ .../test/unit/test_remote_settings_jexl_filters.js | 217 +++ .../test/unit/test_remote_settings_poll.js | 1135 ++++++++++++++++ .../test/unit/test_remote_settings_signatures.js | 827 ++++++++++++ .../collection_signing_ee.pem.certspec | 5 + .../collection_signing_int.pem.certspec | 4 + .../collection_signing_root.pem.certspec | 4 + .../unit/test_remote_settings_signatures/moz.build | 14 + .../test/unit/test_remote_settings_worker.js | 143 ++ .../settings/test/unit/test_shutdown_handling.js | 143 ++ services/settings/test/unit/xpcshell.ini | 16 + 19 files changed, 4559 insertions(+) create mode 100644 services/settings/test/moz.build create mode 100644 services/settings/test/unit/moz.build create mode 100644 services/settings/test/unit/test_attachments_downloader.js create mode 100644 services/settings/test/unit/test_attachments_downloader/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem create mode 100644 services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt create mode 100644 services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json create mode 100644 services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.txt.meta.json create mode 100644 services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt create mode 100644 services/settings/test/unit/test_remote_settings.js create mode 100644 services/settings/test/unit/test_remote_settings_jexl_filters.js create mode 100644 services/settings/test/unit/test_remote_settings_poll.js create mode 100644 services/settings/test/unit/test_remote_settings_signatures.js create mode 100644 services/settings/test/unit/test_remote_settings_signatures/collection_signing_ee.pem.certspec create mode 100644 services/settings/test/unit/test_remote_settings_signatures/collection_signing_int.pem.certspec create mode 100644 services/settings/test/unit/test_remote_settings_signatures/collection_signing_root.pem.certspec create mode 100644 services/settings/test/unit/test_remote_settings_signatures/moz.build create mode 100644 services/settings/test/unit/test_remote_settings_worker.js create mode 100644 services/settings/test/unit/test_shutdown_handling.js create mode 100644 services/settings/test/unit/xpcshell.ini (limited to 'services/settings/test') diff --git a/services/settings/test/moz.build b/services/settings/test/moz.build new file mode 100644 index 0000000000..18bbf1376d --- /dev/null +++ b/services/settings/test/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TEST_DIRS += ["unit"] diff --git a/services/settings/test/unit/moz.build b/services/settings/test/unit/moz.build new file mode 100644 index 0000000000..27bf4c80e7 --- /dev/null +++ b/services/settings/test/unit/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TEST_DIRS += ["test_remote_settings_signatures"] diff --git a/services/settings/test/unit/test_attachments_downloader.js b/services/settings/test/unit/test_attachments_downloader.js new file mode 100644 index 0000000000..d21b11a053 --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader.js @@ -0,0 +1,573 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { UptakeTelemetry } = ChromeUtils.import( + "resource://services-common/uptake-telemetry.js" +); +const { Downloader } = ChromeUtils.import( + "resource://services-settings/Attachments.jsm" +); +const { TelemetryTestUtils } = ChromeUtils.import( + "resource://testing-common/TelemetryTestUtils.jsm" +); +const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm"); + +const RECORD = { + id: "1f3a0802-648d-11ea-bd79-876a8b69c377", + attachment: { + hash: "f41ed47d0f43325c9f089d03415c972ce1d3f1ecab6e4d6260665baf3db3ccee", + size: 1597, + filename: "test_file.pem", + location: + "main-workspace/some-collection/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem", + mimetype: "application/x-pem-file", + }, +}; + +const RECORD_OF_DUMP = { + id: "filename-of-dump.txt", + attachment: { + filename: "filename-of-dump.txt", + hash: "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b", + size: 25, + }, + last_modified: 1234567, + some_key: "some metadata", +}; + +let downloader; +let server; + +function pathFromURL(url) { + const uri = Services.io.newURI(url); + const file = uri.QueryInterface(Ci.nsIFileURL).file; + return file.path; +} + +const PROFILE_URL = + "file://" + + OS.Path.split(OS.Constants.Path.localProfileDir).components.join("/"); + +function run_test() { + server = new HttpServer(); + server.start(-1); + registerCleanupFunction(() => server.stop(() => {})); + + server.registerDirectory( + "/cdn/main-workspace/some-collection/", + do_get_file("test_attachments_downloader") + ); + + server.registerPathHandler("/v1/", (request, response) => { + response.write( + JSON.stringify({ + capabilities: { + attachments: { + base_url: `http://localhost:${server.identity.primaryPort}/cdn/`, + }, + }, + }) + ); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setStatusLine(null, 200, "OK"); + }); + + Services.prefs.setCharPref( + "services.settings.server", + `http://localhost:${server.identity.primaryPort}/v1` + ); + + run_next_test(); +} + +async function clear_state() { + downloader = new Downloader("main", "some-collection"); + await downloader.delete(RECORD); +} + +add_task(clear_state); + +add_task(async function test_download_writes_file_in_profile() { + const fileURL = await downloader.download(RECORD); + const localFilePath = pathFromURL(fileURL); + + Assert.equal( + fileURL, + PROFILE_URL + "/settings/main/some-collection/test_file.pem" + ); + Assert.ok(await OS.File.exists(localFilePath)); + const stat = await OS.File.stat(localFilePath); + Assert.equal(stat.size, 1597); +}); +add_task(clear_state); + +add_task(async function test_download_as_bytes() { + const bytes = await downloader.downloadAsBytes(RECORD); + + // See *.pem file in tests data. + Assert.ok(bytes.byteLength > 1500, `Wrong bytes size: ${bytes.byteLength}`); +}); +add_task(clear_state); + +add_task(async function test_file_is_redownloaded_if_size_does_not_match() { + const fileURL = await downloader.download(RECORD); + const localFilePath = pathFromURL(fileURL); + await OS.File.writeAtomic(localFilePath, "bad-content", { + encoding: "utf-8", + }); + let stat = await OS.File.stat(localFilePath); + Assert.notEqual(stat.size, 1597); + + await downloader.download(RECORD); + + stat = await OS.File.stat(localFilePath); + Assert.equal(stat.size, 1597); +}); +add_task(clear_state); + +add_task(async function test_file_is_redownloaded_if_corrupted() { + const fileURL = await downloader.download(RECORD); + const localFilePath = pathFromURL(fileURL); + const byteArray = await OS.File.read(localFilePath); + byteArray[0] = 42; + await OS.File.writeAtomic(localFilePath, byteArray); + let content = await OS.File.read(localFilePath, { encoding: "utf-8" }); + Assert.notEqual(content.slice(0, 5), "-----"); + + await downloader.download(RECORD); + + content = await OS.File.read(localFilePath, { encoding: "utf-8" }); + Assert.equal(content.slice(0, 5), "-----"); +}); +add_task(clear_state); + +add_task(async function test_download_is_retried_3_times_if_download_fails() { + const record = { + attachment: { + ...RECORD.attachment, + location: "404-error.pem", + }, + }; + + let called = 0; + const _fetchAttachment = downloader._fetchAttachment; + downloader._fetchAttachment = async url => { + called++; + return _fetchAttachment(url); + }; + + let error; + try { + await downloader.download(record); + } catch (e) { + error = e; + } + + Assert.equal(called, 4); // 1 + 3 retries + Assert.ok(error instanceof Downloader.DownloadError); +}); +add_task(clear_state); + +add_task(async function test_download_is_retried_3_times_if_content_fails() { + const record = { + attachment: { + ...RECORD.attachment, + hash: "always-wrong", + }, + }; + let called = 0; + downloader._fetchAttachment = async () => { + called++; + return new ArrayBuffer(); + }; + + let error; + try { + await downloader.download(record); + } catch (e) { + error = e; + } + + Assert.equal(called, 4); // 1 + 3 retries + Assert.ok(error instanceof Downloader.BadContentError); +}); +add_task(clear_state); + +add_task(async function test_delete_removes_local_file() { + const fileURL = await downloader.download(RECORD); + const localFilePath = pathFromURL(fileURL); + Assert.ok(await OS.File.exists(localFilePath)); + + downloader.delete(RECORD); + + Assert.ok(!(await OS.File.exists(localFilePath))); + Assert.ok( + !(await OS.File.exists( + OS.Path.join(OS.Constants.Path.localProfileDir, ...downloader.folders) + )) + ); +}); +add_task(clear_state); + +add_task(async function test_delete_all() { + const client = RemoteSettings("some-collection"); + await client.db.create(RECORD); + const fileURL = await downloader.download(RECORD); + const localFilePath = pathFromURL(fileURL); + Assert.ok(await OS.File.exists(localFilePath)); + + await client.attachments.deleteAll(); + + Assert.ok(!(await OS.File.exists(localFilePath))); +}); +add_task(clear_state); + +add_task(async function test_downloader_is_accessible_via_client() { + const client = RemoteSettings("some-collection"); + + const fileURL = await client.attachments.download(RECORD); + + Assert.equal( + fileURL, + [ + PROFILE_URL, + "settings", + client.bucketName, + client.collectionName, + RECORD.attachment.filename, + ].join("/") + ); +}); +add_task(clear_state); + +add_task(async function test_downloader_reports_download_errors() { + await withFakeChannel("nightly", async () => { + const client = RemoteSettings("some-collection"); + + const record = { + attachment: { + ...RECORD.attachment, + location: "404-error.pem", + }, + }; + + try { + await client.attachments.download(record, { retry: 0 }); + } catch (e) {} + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.DOWNLOAD_ERROR, + { + source: client.identifier, + }, + ], + ]); + }); +}); +add_task(clear_state); + +add_task(async function test_downloader_reports_offline_error() { + const backupOffline = Services.io.offline; + Services.io.offline = true; + + await withFakeChannel("nightly", async () => { + try { + const client = RemoteSettings("some-collection"); + const record = { + attachment: { + ...RECORD.attachment, + location: "will-try-and-fail.pem", + }, + }; + try { + await client.attachments.download(record, { retry: 0 }); + } catch (e) {} + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR, + { + source: client.identifier, + }, + ], + ]); + } finally { + Services.io.offline = backupOffline; + } + }); +}); +add_task(clear_state); + +// Common code for test_download_cache_hit and test_download_cache_corruption. +async function doTestDownloadCacheImpl({ simulateCorruption }) { + let readCount = 0; + let writeCount = 0; + const cacheImpl = { + async get(attachmentId) { + Assert.equal(attachmentId, RECORD.id, "expected attachmentId"); + ++readCount; + if (simulateCorruption) { + throw new Error("Simulation of corrupted cache (read)"); + } + }, + async set(attachmentId, attachment) { + Assert.equal(attachmentId, RECORD.id, "expected attachmentId"); + Assert.deepEqual(attachment.record, RECORD, "expected record"); + ++writeCount; + if (simulateCorruption) { + throw new Error("Simulation of corrupted cache (write)"); + } + }, + async delete(attachmentId) {}, + }; + Object.defineProperty(downloader, "cacheImpl", { value: cacheImpl }); + + let downloadResult = await downloader.download(RECORD, { + useCache: true, + }); + Assert.equal(downloadResult._source, "remote_match", "expected source"); + Assert.equal(downloadResult.buffer.byteLength, 1597, "expected result"); + Assert.equal(readCount, 1, "expected cache read attempts"); + Assert.equal(writeCount, 1, "expected cache write attempts"); +} + +add_task(async function test_download_cache_hit() { + await doTestDownloadCacheImpl({ simulateCorruption: false }); +}); +add_task(clear_state); + +// Verify that the downloader works despite a broken cache implementation. +add_task(async function test_download_cache_corruption() { + await doTestDownloadCacheImpl({ simulateCorruption: true }); +}); +add_task(clear_state); + +add_task(async function test_download_cached() { + const client = RemoteSettings("main", "some-collection"); + const attachmentId = "dummy filename"; + const badRecord = { + attachment: { + ...RECORD.attachment, + hash: "non-matching hash", + location: "non-existing-location-should-fail.bin", + }, + }; + async function downloadWithCache(record, options) { + options = { ...options, useCache: true }; + return client.attachments.download(record, options); + } + function checkInfo(downloadResult, expectedSource, msg) { + Assert.deepEqual( + downloadResult.record, + RECORD, + `${msg} : expected identical record` + ); + // Simple check: assume that content is identical if the size matches. + Assert.equal( + downloadResult.buffer.byteLength, + RECORD.attachment.size, + `${msg} : expected buffer` + ); + Assert.equal( + downloadResult._source, + expectedSource, + `${msg} : expected source of the result` + ); + } + + await Assert.rejects( + downloadWithCache(null, { attachmentId }), + /DownloadError: Could not download dummy filename/, + "Download without record or cache should fail." + ); + + // Populate cache. + const info1 = await downloadWithCache(RECORD, { attachmentId }); + checkInfo(info1, "remote_match", "first time download"); + + await Assert.rejects( + downloadWithCache(null, { attachmentId }), + /DownloadError: Could not download dummy filename/, + "Download without record still fails even if there is a cache." + ); + + await Assert.rejects( + downloadWithCache(badRecord, { attachmentId }), + /DownloadError: Could not download .*non-existing-location-should-fail.bin/, + "Download with non-matching record still fails even if there is a cache." + ); + + // Download from cache. + const info2 = await downloadWithCache(RECORD, { attachmentId }); + checkInfo(info2, "cache_match", "download matching record from cache"); + + const info3 = await downloadWithCache(RECORD, { + attachmentId, + fallbackToCache: true, + }); + checkInfo(info3, "cache_match", "fallbackToCache accepts matching record"); + + const info4 = await downloadWithCache(null, { + attachmentId, + fallbackToCache: true, + }); + checkInfo(info4, "cache_fallback", "fallbackToCache accepts null record"); + + const info5 = await downloadWithCache(badRecord, { + attachmentId, + fallbackToCache: true, + }); + checkInfo(info5, "cache_fallback", "fallbackToCache ignores bad record"); + + // Bye bye cache. + await client.attachments.deleteCached(attachmentId); + await Assert.rejects( + downloadWithCache(null, { attachmentId, fallbackToCache: true }), + /DownloadError: Could not download dummy filename/, + "Download without cache should fail again." + ); + await Assert.rejects( + downloadWithCache(badRecord, { attachmentId, fallbackToCache: true }), + /DownloadError: Could not download .*non-existing-location-should-fail.bin/, + "Download should fail to fall back to a download of a non-existing record" + ); +}); +add_task(clear_state); + +add_task(async function test_download_from_dump() { + const bucketNamePref = "services.testing.custom-bucket-name-in-this-test"; + Services.prefs.setCharPref(bucketNamePref, "dump-bucket"); + const client = RemoteSettings("dump-collection", { bucketNamePref }); + + // Temporarily replace the resource:-URL with another resource:-URL. + const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL; + Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test"; + const resProto = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + resProto.setSubstitution( + "rs-downloader-test", + Services.io.newFileURI(do_get_file("test_attachments_downloader")) + ); + + function checkInfo(result, expectedSource, expectedRecord = RECORD_OF_DUMP) { + Assert.equal( + new TextDecoder().decode(new Uint8Array(result.buffer)), + "This would be a RS dump.\n", + "expected content from dump" + ); + Assert.deepEqual(result.record, expectedRecord, "expected record for dump"); + Assert.equal(result._source, expectedSource, "expected source of dump"); + } + + // If record matches, should happen before network request. + const dump1 = await client.attachments.download(RECORD_OF_DUMP, { + // Note: attachmentId not set, so should fall back to record.id. + useCache: true, + fallbackToDump: true, + }); + checkInfo(dump1, "dump_match"); + + // If no record given, should try network first, but then fall back to dump. + const dump2 = await client.attachments.download(null, { + attachmentId: RECORD_OF_DUMP.id, + useCache: true, + fallbackToDump: true, + }); + checkInfo(dump2, "dump_fallback"); + + // Fill the cache with the same data as the dump for the next part. + await client.db.saveAttachment(RECORD_OF_DUMP.id, { + record: RECORD_OF_DUMP, + blob: new Blob([dump1.buffer]), + }); + // The dump should take precedence over the cache. + const dump3 = await client.attachments.download(RECORD_OF_DUMP, { + useCache: true, + fallbackToCache: true, + fallbackToDump: true, + }); + checkInfo(dump3, "dump_match"); + + // When the record is not given, the dump takes precedence over the cache + // as a fallback (when the cache and dump are identical). + const dump4 = await client.attachments.download(null, { + attachmentId: RECORD_OF_DUMP.id, + useCache: true, + fallbackToCache: true, + fallbackToDump: true, + }); + checkInfo(dump4, "dump_fallback"); + + // Store a record in the cache that is newer than the dump. + const RECORD_NEWER_THAN_DUMP = { + ...RECORD_OF_DUMP, + last_modified: RECORD_OF_DUMP.last_modified + 1, + }; + await client.db.saveAttachment(RECORD_OF_DUMP.id, { + record: RECORD_NEWER_THAN_DUMP, + blob: new Blob([dump1.buffer]), + }); + + // When the record is not given, use the cache if it has a more recent record. + const dump5 = await client.attachments.download(null, { + attachmentId: RECORD_OF_DUMP.id, + useCache: true, + fallbackToCache: true, + fallbackToDump: true, + }); + checkInfo(dump5, "cache_fallback", RECORD_NEWER_THAN_DUMP); + + // When a record is given, use whichever that has the matching last_modified. + const dump6 = await client.attachments.download(RECORD_OF_DUMP, { + useCache: true, + fallbackToCache: true, + fallbackToDump: true, + }); + checkInfo(dump6, "dump_match"); + const dump7 = await client.attachments.download(RECORD_NEWER_THAN_DUMP, { + useCache: true, + fallbackToCache: true, + fallbackToDump: true, + }); + checkInfo(dump7, "cache_match", RECORD_NEWER_THAN_DUMP); + + await client.attachments.deleteCached(RECORD_OF_DUMP.id); + + await Assert.rejects( + client.attachments.download(null, { + attachmentId: "filename-without-meta.txt", + useCache: true, + fallbackToDump: true, + }), + /DownloadError: Could not download filename-without-meta.txt/, + "Cannot download dump that lacks a .meta.json file" + ); + + await Assert.rejects( + client.attachments.download(null, { + attachmentId: "filename-without-content.txt", + useCache: true, + fallbackToDump: true, + }), + /Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-content\.txt(?!\.meta\.json)/, + "Cannot download dump that is missing, despite the existing .meta.json" + ); + + // Restore, just in case. + Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL; + resProto.setSubstitution("rs-downloader-test", null); +}); +// Not really needed because the last test doesn't modify the main collection, +// but added for consistency with other tests tasks around here. +add_task(clear_state); diff --git a/services/settings/test/unit/test_attachments_downloader/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem b/services/settings/test/unit/test_attachments_downloader/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem new file mode 100644 index 0000000000..502e8c9ce0 --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEbjCCA1agAwIBAgIQBg3WwdBnkBtUdfz/wp4xNzANBgkqhkiG9w0BAQsFADBa +MQswCQYDVQQGEwJJRTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJl +clRydXN0MSIwIAYDVQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTE1 +MTAxNDEyMDAwMFoXDTIwMTAxNDEyMDAwMFowbzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBDbG91ZEZs +YXJlLCBJbmMuMSAwHgYDVQQDExdDbG91ZEZsYXJlIEluYyBSU0EgQ0EtMTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJGiNOIE4s0M4wdhDeV9aMfAYY9l +yG9cfGQqt7a5UgrRA81bi4istCyhzfzRWUW+NAmf6X2HEnA3xLI1M+pH/xEbk9pw +jc8/1CPy9jUjBwb89zt5PWh2I1KxZVg/Bnx2yYdVcKTUMKt0GLDXfZXN+RYZHJQo +lDlzjH5xV0IpDMv/FsMEZWcfx1JorBf08bRnRVkl9RY00y2ujVr+492ze+zYQ9s7 +HcidpR+7ret3jzLSvojsaA5+fOaCG0ctVJcLfnkQ5lWR95ByBdO1NapfqZ1+kmCL +3baVSeUpYQriBwznxfLuGs8POo4QdviYVtSPBWjOEfb+o1c6Mbo8p4noFzUCAwEA +AaOCARkwggEVMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDQG +CCsGAQUFBwEBBCgwJjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu +Y29tMDoGA1UdHwQzMDEwL6AtoCuGKWh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9P +bW5pcm9vdDIwMjUuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIB +FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBSRBYrfTCLG +bYuUTBZFfu5vAvu3wDAfBgNVHSMEGDAWgBTlnVkwgkdYzKz6CFQ2hns6tQRN8DAN +BgkqhkiG9w0BAQsFAAOCAQEAVJle3ar9NSnTrLAhgfkcpClIY6/kabDIEa8cOnu1 +SOXf4vbtZakSmmIbFbmYDUGIU5XwwVdF/FKNzNBRf9G4EL/S0NXytBKj4A34UGQA +InaV+DgVLzCifN9cAHi8EFEAfbglUvPvLPFXF0bwffElYm7QBSiHYSZmfOKLCyiv +3zlQsf7ozNBAxfbmnRMRSUBcIhRwnaFoFgDs7yU6R1Yk4pO7eMgWpdPGhymDTIvv +RnauKStzKsAli9i5hQ4nTDITUpMAmeJoXodgwRkC3Civw32UR2rxObIyxPpbfODb +sZKNGO9K5Sjj6turB1zwbd2wI8MhtUCY9tGmSYhe7G6Bkw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt new file mode 100644 index 0000000000..77d7b4154f --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt @@ -0,0 +1 @@ +This would be a RS dump. diff --git a/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json new file mode 100644 index 0000000000..de7681940d --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json @@ -0,0 +1,10 @@ +{ + "id": "filename-of-dump.txt", + "attachment": { + "filename": "filename-of-dump.txt", + "hash": "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b", + "size": 25 + }, + "last_modified": 1234567, + "some_key": "some metadata" +} diff --git a/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.txt.meta.json b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.txt.meta.json new file mode 100644 index 0000000000..33fd28a710 --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.txt.meta.json @@ -0,0 +1,9 @@ +{ + "fyi": "This .meta.json file describes an attachment, but that attachment is missing.", + "attachment": { + "filename": "filename-without-content.txt", + "hash": "...", + "size": "..." + } +} + diff --git a/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt new file mode 100644 index 0000000000..5fbfd11c9e --- /dev/null +++ b/services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt @@ -0,0 +1 @@ +The filename-without-meta.txt.meta.json file is missing. diff --git a/services/settings/test/unit/test_remote_settings.js b/services/settings/test/unit/test_remote_settings.js new file mode 100644 index 0000000000..20ef181b5e --- /dev/null +++ b/services/settings/test/unit/test_remote_settings.js @@ -0,0 +1,1417 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +const { Constructor: CC } = Components; + +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +const { ObjectUtils } = ChromeUtils.import( + "resource://gre/modules/ObjectUtils.jsm" +); +const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm"); + +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm"); +const { UptakeTelemetry } = ChromeUtils.import( + "resource://services-common/uptake-telemetry.js" +); +const { TelemetryTestUtils } = ChromeUtils.import( + "resource://testing-common/TelemetryTestUtils.jsm" +); + +const IS_ANDROID = AppConstants.platform == "android"; + +const BinaryInputStream = CC( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); + +let server; +let client; +let clientWithDump; + +async function clear_state() { + client.verifySignature = false; + clientWithDump.verifySignature = false; + + // Clear local DB. + await client.db.clear(); + // Reset event listeners. + client._listeners.set("sync", []); + + await clientWithDump.db.clear(); + + Services.prefs.clearUserPref("services.settings.default_bucket"); + + // Clear events snapshot. + TelemetryTestUtils.assertEvents([], {}, { process: "dummy" }); +} + +function run_test() { + // Set up an HTTP Server + server = new HttpServer(); + server.start(-1); + + // Point the blocklist clients to use this local HTTP server. + Services.prefs.setCharPref( + "services.settings.server", + `http://localhost:${server.identity.primaryPort}/v1` + ); + + Services.prefs.setCharPref("services.settings.loglevel", "debug"); + + client = RemoteSettings("password-fields"); + clientWithDump = RemoteSettings("language-dictionaries"); + + server.registerPathHandler("/v1/", handleResponse); + server.registerPathHandler( + "/v1/buckets/monitor/collections/changes/records", + handleResponse + ); + server.registerPathHandler( + "/v1/buckets/main/collections/password-fields/changeset", + handleResponse + ); + server.registerPathHandler( + "/v1/buckets/main/collections/language-dictionaries/changeset", + handleResponse + ); + server.registerPathHandler( + "/v1/buckets/main/collections/with-local-fields/changeset", + handleResponse + ); + server.registerPathHandler("/fake-x5u", handleResponse); + + run_next_test(); + + registerCleanupFunction(function() { + server.stop(() => {}); + }); +} +add_task(clear_state); + +add_task(async function test_records_obtained_from_server_are_stored_in_db() { + // Test an empty db populates + await client.maybeSync(2000); + + // Open the collection, verify it's been populated: + // Our test data has a single record; it should be in the local collection + const list = await client.get(); + equal(list.length, 1); + + const timestamp = await client.db.getLastModified(); + equal(timestamp, 3000, "timestamp was stored"); + + const { signature } = await client.db.getMetadata(); + equal(signature.signature, "abcdef", "metadata was stored"); +}); +add_task(clear_state); + +add_task( + async function test_records_from_dump_are_listed_as_created_in_event() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + let received; + clientWithDump.on("sync", ({ data }) => (received = data)); + // Use a timestamp superior to latest record in dump. + const timestamp = 5000000000000; // Fri Jun 11 2128 + + await clientWithDump.maybeSync(timestamp); + + const list = await clientWithDump.get(); + ok(list.length > 20, `The dump was loaded (${list.length} records)`); + equal(received.created[0].id, "xx", "Record from the sync come first."); + + const createdById = received.created.reduce((acc, r) => { + acc[r.id] = r; + return acc; + }, {}); + + ok( + !(received.deleted[0].id in createdById), + "Deleted records are not listed as created" + ); + equal( + createdById[received.updated[0].new.id], + received.updated[0].new, + "The records that were updated should appear as created in their newest form." + ); + + equal( + received.created.length, + list.length, + "The list of created records contains the dump" + ); + equal(received.current.length, received.created.length); + } +); +add_task(clear_state); + +add_task(async function test_throws_when_network_is_offline() { + const backupOffline = Services.io.offline; + try { + Services.io.offline = true; + const startHistogram = getUptakeTelemetrySnapshot( + clientWithDump.identifier + ); + let error; + try { + await clientWithDump.maybeSync(2000); + } catch (e) { + error = e; + } + equal(error.name, "NetworkOfflineError"); + + const endHistogram = getUptakeTelemetrySnapshot(clientWithDump.identifier); + const expectedIncrements = { + [UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); + } finally { + Services.io.offline = backupOffline; + } +}); +add_task(clear_state); + +add_task(async function test_sync_event_is_sent_even_if_up_to_date() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + // First, determine what is the dump timestamp. Sync will load it. + // Use a timestamp inferior to latest record in dump. + await clientWithDump._importJSONDump(); + const uptodateTimestamp = await clientWithDump.db.getLastModified(); + await clear_state(); + + // Now, simulate that server data wasn't changed since dump was released. + const startHistogram = getUptakeTelemetrySnapshot(clientWithDump.identifier); + let received; + clientWithDump.on("sync", ({ data }) => (received = data)); + + await clientWithDump.maybeSync(uptodateTimestamp); + + ok(received.current.length > 0, "Dump records are listed as created"); + equal(received.current.length, received.created.length); + + const endHistogram = getUptakeTelemetrySnapshot(clientWithDump.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.UP_TO_DATE]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_records_can_have_local_fields() { + const c = RemoteSettings("with-local-fields", { localFields: ["accepted"] }); + c.verifySignature = false; + + await c.maybeSync(2000); + + await c.db.update({ + id: "c74279ce-fb0a-42a6-ae11-386b567a6119", + accepted: true, + }); + await c.maybeSync(3000); // Does not fail. +}); +add_task(clear_state); + +add_task( + async function test_records_changes_are_overwritten_by_server_changes() { + // Create some local conflicting data, and make sure it syncs without error. + await client.db.create({ + website: "", + id: "9d500963-d80e-3a91-6e74-66f3811b99cc", + }); + + await client.maybeSync(2000); + + const data = await client.get(); + equal(data[0].website, "https://some-website.com"); + } +); +add_task(clear_state); + +add_task( + async function test_get_loads_default_records_from_a_local_dump_when_database_is_empty() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + + // When collection has a dump in services/settings/dumps/{bucket}/{collection}.json + const data = await clientWithDump.get(); + notEqual(data.length, 0); + // No synchronization happened (responses are not mocked). + } +); +add_task(clear_state); + +add_task(async function test_get_does_not_load_dump_when_pref_is_false() { + Services.prefs.setBoolPref("services.settings.load_dump", false); + + const data = await clientWithDump.get(); + + equal(data.map(r => r.id).join(", "), "pt-BR, xx"); // No dump, 2 pulled from test server. + Services.prefs.clearUserPref("services.settings.load_dump"); +}); +add_task(clear_state); + +add_task(async function test_get_loads_dump_only_once_if_called_in_parallel() { + const backup = clientWithDump._importJSONDump; + let callCount = 0; + clientWithDump._importJSONDump = async () => { + callCount++; + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + await new Promise(resolve => setTimeout(resolve, 100)); + return 42; + }; + await Promise.all([clientWithDump.get(), clientWithDump.get()]); + equal(callCount, 1, "JSON dump was called more than once"); + clientWithDump._importJSONDump = backup; +}); +add_task(clear_state); + +add_task(async function test_get_falls_back_to_dump_if_db_fails() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + const backup = clientWithDump.db.getLastModified; + clientWithDump.db.getLastModified = () => { + throw new Error("Unknown error"); + }; + + const records = await clientWithDump.get({ dumpFallback: true }); + ok(records.length > 0, "dump content is returned"); + + // If fallback is disabled, error is thrown. + let error; + try { + await clientWithDump.get({ dumpFallback: false }); + } catch (e) { + error = e; + } + equal(error.message, "Unknown error"); + + clientWithDump.db.getLastModified = backup; +}); +add_task(clear_state); + +add_task(async function test_get_sorts_results_if_specified() { + await client.db.importChanges( + {}, + 42, + [ + { + field: 12, + id: "9d500963-d80e-3a91-6e74-66f3811b99cc", + }, + { + field: 7, + id: "d83444a4-f348-4cd8-8228-842cb927db9f", + }, + ], + { clear: true } + ); + + const records = await client.get({ order: "field" }); + ok( + records[0].field < records[records.length - 1].field, + "records are sorted" + ); +}); +add_task(clear_state); + +add_task(async function test_get_falls_back_sorts_results() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + const backup = clientWithDump.db.getLastModified; + clientWithDump.db.getLastModified = () => { + throw new Error("Unknown error"); + }; + + const records = await clientWithDump.get({ + dumpFallback: true, + order: "-id", + }); + + ok(records[0].id > records[records.length - 1].id, "records are sorted"); + + clientWithDump.db.getLastModified = backup; +}); +add_task(clear_state); + +add_task(async function test_get_falls_back_to_dump_if_db_fails_later() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + const backup = clientWithDump.db.list; + clientWithDump.db.list = () => { + throw new Error("Unknown error"); + }; + + const records = await clientWithDump.get({ dumpFallback: true }); + ok(records.length > 0, "dump content is returned"); + + // If fallback is disabled, error is thrown. + let error; + try { + await clientWithDump.get({ dumpFallback: false }); + } catch (e) { + error = e; + } + equal(error.message, "Unknown error"); + + clientWithDump.db.list = backup; +}); +add_task(clear_state); + +add_task(async function test_get_does_not_sync_if_empty_dump_is_provided() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + + const clientWithEmptyDump = RemoteSettings("example"); + Assert.ok(!(await Utils.hasLocalData(clientWithEmptyDump))); + + const data = await clientWithEmptyDump.get(); + + equal(data.length, 0); + Assert.ok(await Utils.hasLocalData(clientWithEmptyDump)); +}); +add_task(clear_state); + +add_task(async function test_get_synchronization_can_be_disabled() { + const data = await client.get({ syncIfEmpty: false }); + + equal(data.length, 0); +}); +add_task(clear_state); + +add_task( + async function test_get_triggers_synchronization_when_database_is_empty() { + // The "password-fields" collection has no local dump, and no local data. + // Therefore a synchronization will happen. + const data = await client.get(); + + // Data comes from mocked HTTP response (see below). + equal(data.length, 1); + equal(data[0].selector, "#webpage[field-pwd]"); + } +); +add_task(clear_state); + +add_task(async function test_get_ignores_synchronization_errors() { + // The monitor endpoint won't contain any information about this collection. + let data = await RemoteSettings("some-unknown-key").get(); + equal(data.length, 0); + // The sync endpoints are not mocked, this fails internally. + data = await RemoteSettings("no-mocked-responses").get(); + equal(data.length, 0); +}); +add_task(clear_state); + +add_task(async function test_get_verify_signature_no_sync() { + // No signature in metadata, and no sync if empty. + let error; + try { + await client.get({ verifySignature: true, syncIfEmpty: false }); + } catch (e) { + error = e; + } + equal(error.message, "Missing signature (main/password-fields)"); +}); +add_task(clear_state); + +add_task(async function test_get_can_verify_signature_pulled() { + // Populate the local DB (only records, eg. loaded from dump previously) + await client._importJSONDump(); + + let calledSignature; + client._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + calledSignature = signature; + return true; + }, + }; + client.verifySignature = true; + + // No metadata in local DB, but gets pulled and then verifies. + ok(ObjectUtils.isEmpty(await client.db.getMetadata()), "Metadata is empty"); + + await client.get({ verifySignature: true }); + + ok( + !ObjectUtils.isEmpty(await client.db.getMetadata()), + "Metadata was pulled" + ); + ok(calledSignature.endsWith("some-sig"), "Signature was verified"); +}); +add_task(clear_state); + +add_task(async function test_get_can_verify_signature() { + // Populate the local DB (record and metadata) + await client.maybeSync(2000); + + // It validates signature that was stored in local DB. + let calledSignature; + client._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + calledSignature = signature; + return JSON.parse(serialized).data.length == 1; + }, + }; + ok(await Utils.hasLocalData(client), "Local data was populated"); + await client.get({ verifySignature: true }); + + ok(calledSignature.endsWith("abcdef"), "Signature was verified"); + + // It throws when signature does not verify. + await client.db.delete("9d500963-d80e-3a91-6e74-66f3811b99cc"); + let error = null; + try { + await client.get({ verifySignature: true }); + } catch (e) { + error = e; + } + equal(error.message, "Invalid content signature (main/password-fields)"); +}); +add_task(clear_state); + +add_task(async function test_get_does_not_verify_signature_if_load_dump() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + + let called; + clientWithDump._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + called = true; + return true; + }, + }; + + // When dump is loaded, signature is not verified. + const records = await clientWithDump.get({ verifySignature: true }); + ok(records.length > 0, "dump is loaded"); + ok(!called, "signature is missing but not verified"); + + // If metadata is missing locally, it is not fetched if `syncIfEmpty` is disabled. + let error; + try { + await clientWithDump.get({ verifySignature: true, syncIfEmpty: false }); + } catch (e) { + error = e; + } + ok(!called, "signer was not called"); + equal( + error.message, + "Missing signature (main/language-dictionaries)", + "signature is missing locally" + ); + + // If metadata is missing locally, it is fetched by default (`syncIfEmpty: true`) + await clientWithDump.get({ verifySignature: true }); + const metadata = await clientWithDump.db.getMetadata(); + ok(Object.keys(metadata).length > 0, "metadata was fetched"); + ok(called, "signature was verified for the data that was in dump"); +}); +add_task(clear_state); + +add_task( + async function test_get_does_verify_signature_if_json_loaded_in_parallel() { + const backup = clientWithDump._verifier; + let callCount = 0; + clientWithDump._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + callCount++; + return true; + }, + }; + await Promise.all([ + clientWithDump.get({ verifySignature: true }), + clientWithDump.get({ verifySignature: true }), + ]); + equal(callCount, 0, "No need to verify signatures if JSON dump is loaded"); + clientWithDump._verifier = backup; + } +); +add_task(clear_state); + +add_task(async function test_sync_runs_once_only() { + const backup = Utils.log.warn; + const messages = []; + Utils.log.warn = m => { + messages.push(m); + }; + + await Promise.all([client.maybeSync(2000), client.maybeSync(2000)]); + + ok( + messages.includes("main/password-fields sync already running"), + "warning is shown about sync already running" + ); + Utils.log.warn = backup; +}); +add_task(clear_state); + +add_task( + async function test_sync_pulls_metadata_if_missing_with_dump_is_up_to_date() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + + let called; + clientWithDump._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + called = true; + return true; + }, + }; + // When dump is loaded, signature is not verified. + const records = await clientWithDump.get({ verifySignature: true }); + ok(records.length > 0, "dump is loaded"); + ok(!called, "signature is missing but not verified"); + + // Synchronize the collection (local data is up-to-date). + // Signature verification is disabled (see `clear_state()`), so we don't bother with + // fetching metadata. + const uptodateTimestamp = await clientWithDump.db.getLastModified(); + await clientWithDump.maybeSync(uptodateTimestamp); + let metadata = await clientWithDump.db.getMetadata(); + ok(!metadata, "metadata was not fetched"); + + // Synchronize again the collection (up-to-date, since collection last modified still > 42) + clientWithDump.verifySignature = true; + await clientWithDump.maybeSync(42); + + // With signature verification, metadata was fetched. + metadata = await clientWithDump.db.getMetadata(); + ok(Object.keys(metadata).length > 0, "metadata was fetched"); + ok(called, "signature was verified for the data that was in dump"); + + // Metadata is present, signature will now verified. + called = false; + await clientWithDump.get({ verifySignature: true }); + ok(called, "local signature is verified"); + } +); +add_task(clear_state); + +add_task(async function test_sync_event_provides_information_about_records() { + let eventData; + client.on("sync", ({ data }) => (eventData = data)); + + await client.maybeSync(2000); + equal(eventData.current.length, 1); + + await client.maybeSync(3001); + equal(eventData.current.length, 2); + equal(eventData.created.length, 1); + equal(eventData.created[0].website, "https://www.other.org/signin"); + equal(eventData.updated.length, 1); + equal(eventData.updated[0].old.website, "https://some-website.com"); + equal(eventData.updated[0].new.website, "https://some-website.com/login"); + equal(eventData.deleted.length, 0); + + await client.maybeSync(4001); + equal(eventData.current.length, 1); + equal(eventData.created.length, 0); + equal(eventData.updated.length, 0); + equal(eventData.deleted.length, 1); + equal(eventData.deleted[0].website, "https://www.other.org/signin"); +}); +add_task(clear_state); + +add_task(async function test_inspect_method() { + // Synchronize the `password-fields` collection in order to have + // some local data when .inspect() is called. + await client.maybeSync(2000); + + const inspected = await RemoteSettings.inspect(); + + // Assertion for global attributes. + const { + mainBucket, + serverURL, + defaultSigner, + collections, + serverTimestamp, + } = inspected; + const rsSigner = "remote-settings.content-signature.mozilla.org"; + equal(mainBucket, "main"); + equal(serverURL, `http://localhost:${server.identity.primaryPort}/v1`); + equal(defaultSigner, rsSigner); + equal(serverTimestamp, '"5000"'); + + // A collection is listed in .inspect() if it has local data or if there + // is a JSON dump for it. + // "password-fields" has no dump but was synchronized above and thus has local data. + let col = collections.pop(); + equal(col.collection, "password-fields"); + equal(col.serverTimestamp, 3000); + equal(col.localTimestamp, 3000); + + if (!IS_ANDROID) { + // "language-dictionaries" has a local dump (not on Android) + col = collections.pop(); + equal(col.collection, "language-dictionaries"); + equal(col.serverTimestamp, 4000); + ok(!col.localTimestamp); // not synchronized. + } +}); +add_task(clear_state); + +add_task(async function test_clearAll_method() { + // Make sure we have some local data. + await client.maybeSync(2000); + await clientWithDump.maybeSync(2000); + + await RemoteSettings.clearAll(); + + ok(!(await Utils.hasLocalData(client)), "Local data was deleted"); + ok(!(await Utils.hasLocalData(clientWithDump)), "Local data was deleted"); + ok( + !Services.prefs.prefHasUserValue(client.lastCheckTimePref), + "Pref was cleaned" + ); + + // Synchronization is not broken after resuming. + await client.maybeSync(2000); + await clientWithDump.maybeSync(2000); + ok(await Utils.hasLocalData(client), "Local data was populated"); + ok(await Utils.hasLocalData(clientWithDump), "Local data was populated"); +}); +add_task(clear_state); + +add_task(async function test_listeners_are_not_deduplicated() { + let count = 0; + const plus1 = () => { + count += 1; + }; + + client.on("sync", plus1); + client.on("sync", plus1); + client.on("sync", plus1); + + await client.maybeSync(2000); + + equal(count, 3); +}); +add_task(clear_state); + +add_task(async function test_listeners_can_be_removed() { + let count = 0; + const onSync = () => { + count += 1; + }; + + client.on("sync", onSync); + client.off("sync", onSync); + + await client.maybeSync(2000); + + equal(count, 0); +}); +add_task(clear_state); + +add_task(async function test_all_listeners_are_executed_if_one_fails() { + let count = 0; + client.on("sync", () => { + count += 1; + }); + client.on("sync", () => { + throw new Error("boom"); + }); + client.on("sync", () => { + count += 2; + }); + + let error; + try { + await client.maybeSync(2000); + } catch (e) { + error = e; + } + + equal(count, 3); + equal(error.message, "boom"); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_up_to_date() { + await client.maybeSync(2000); + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + await client.maybeSync(3000); + + // No Telemetry was sent. + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.UP_TO_DATE]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_if_sync_succeeds() { + // We test each client because Telemetry requires preleminary declarations. + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + await client.maybeSync(2000); + + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.SUCCESS]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task( + async function test_synchronization_duration_is_reported_in_uptake_status() { + await withFakeChannel("nightly", async () => { + await client.maybeSync(2000); + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.SUCCESS, + { + source: client.identifier, + duration: v => v > 0, + trigger: "manual", + }, + ], + ]); + }); + } +); +add_task(clear_state); + +add_task(async function test_telemetry_reports_if_application_fails() { + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + client.on("sync", () => { + throw new Error("boom"); + }); + + try { + await client.maybeSync(2000); + } catch (e) {} + + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.APPLY_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_if_sync_fails() { + await client.db.importChanges({}, 9999); + + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + try { + await client.maybeSync(10000); + } catch (e) {} + + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.SERVER_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_if_parsing_fails() { + await client.db.importChanges({}, 10000); + + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + try { + await client.maybeSync(10001); + } catch (e) {} + + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.PARSE_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_if_fetching_signature_fails() { + await client.db.importChanges({}, 11000); + + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + try { + await client.maybeSync(11001); + } catch (e) {} + + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.SERVER_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_unknown_errors() { + const backup = client.db.list; + client.db.list = () => { + throw new Error("Internal"); + }; + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + try { + await client.maybeSync(2000); + } catch (e) {} + + client.db.list = backup; + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.UNKNOWN_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_indexeddb_as_custom_1() { + const backup = client.db.getLastModified; + const msg = + "IndexedDB getLastModified() The operation failed for reasons unrelated to the database itself"; + client.db.getLastModified = () => { + throw new Error(msg); + }; + const startHistogram = getUptakeTelemetrySnapshot(client.identifier); + + try { + await client.maybeSync(2000); + } catch (e) {} + + client.db.getLastModified = backup; + const endHistogram = getUptakeTelemetrySnapshot(client.identifier); + const expectedIncrements = { [UptakeTelemetry.STATUS.CUSTOM_1_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_telemetry_reports_error_name_as_event_nightly() { + const backup = client.db.list; + client.db.list = () => { + const e = new Error("Some unknown error"); + e.name = "ThrownError"; + throw e; + }; + + await withFakeChannel("nightly", async () => { + try { + await client.maybeSync(2000); + } catch (e) {} + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.UNKNOWN_ERROR, + { + source: client.identifier, + trigger: "manual", + duration: v => v >= 0, + errorName: "ThrownError", + }, + ], + ]); + }); + + client.db.list = backup; +}); +add_task(clear_state); + +add_task(async function test_bucketname_changes_when_bucket_pref_changes() { + equal(client.bucketName, "main"); + + Services.prefs.setCharPref( + "services.settings.default_bucket", + "main-preview" + ); + + equal(client.bucketName, "main-preview"); +}); +add_task(clear_state); + +add_task( + async function test_get_loads_default_records_from_a_local_dump_if_preview_collection() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + Services.prefs.setCharPref( + "services.settings.default_bucket", + "main-preview" + ); + // When collection has a dump in services/settings/dumps/{bucket}/{collection}.json + const data = await clientWithDump.get(); + notEqual(data.length, 0); + // No synchronization happened (responses are not mocked). + } +); +add_task(clear_state); + +add_task( + async function test_inspect_changes_the_list_when_bucket_pref_is_changed() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + // Register a client only listed in -preview... + RemoteSettings("crash-rate"); + + const { collections: before } = await RemoteSettings.inspect(); + + // These two collections are listed in the main bucket in monitor/changes (one with dump, one registered). + deepEqual(before.map(c => c.collection).sort(), [ + "language-dictionaries", + "password-fields", + ]); + + // Switch to main-preview bucket. + Services.prefs.setCharPref( + "services.settings.default_bucket", + "main-preview" + ); + const { collections: after, mainBucket } = await RemoteSettings.inspect(); + + // These two collections are listed in the main bucket in monitor/changes (both are registered). + deepEqual(after.map(c => c.collection).sort(), [ + "crash-rate", + "password-fields", + ]); + equal(mainBucket, "main-preview"); + } +); +add_task(clear_state); + +function handleResponse(request, response) { + try { + const sample = getSampleResponse(request, server.identity.primaryPort); + if (!sample) { + do_throw( + `unexpected ${request.method} request for ${request.path}?${request.queryString}` + ); + } + + response.setStatusLine( + null, + sample.status.status, + sample.status.statusText + ); + // send the headers + for (let headerLine of sample.sampleHeaders) { + let headerElements = headerLine.split(":"); + response.setHeader(headerElements[0], headerElements[1].trimLeft()); + } + response.setHeader("Date", new Date().toUTCString()); + + const body = + typeof sample.responseBody == "string" + ? sample.responseBody + : JSON.stringify(sample.responseBody); + response.write(body); + response.finish(); + } catch (e) { + info(e); + } +} + +function getSampleResponse(req, port) { + const responses = { + OPTIONS: { + sampleHeaders: [ + "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page", + "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS", + "Access-Control-Allow-Origin: *", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + ], + status: { status: 200, statusText: "OK" }, + responseBody: null, + }, + "GET:/v1/": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + ], + status: { status: 200, statusText: "OK" }, + responseBody: { + settings: { + batch_max_requests: 25, + }, + url: `http://localhost:${port}/v1/`, + documentation: "https://kinto.readthedocs.org/", + version: "1.5.1", + commit: "cbc6f58", + hello: "kinto", + }, + }, + "GET:/v1/buckets/monitor/collections/changes/records": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + `Date: ${new Date().toUTCString()}`, + 'Etag: "5000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: { + data: [ + { + id: "4676f0c7-9757-4796-a0e8-b40a5a37a9c9", + bucket: "main", + collection: "unknown-locally", + last_modified: 5000, + }, + { + id: "4676f0c7-9757-4796-a0e8-b40a5a37a9c9", + bucket: "main", + collection: "language-dictionaries", + last_modified: 4000, + }, + { + id: "0af8da0b-3e03-48fb-8d0d-2d8e4cb7514d", + bucket: "main", + collection: "password-fields", + last_modified: 3000, + }, + { + id: "4acda969-3bd3-4074-a678-ff311eeb076e", + bucket: "main-preview", + collection: "password-fields", + last_modified: 2000, + }, + { + id: "58697bd1-315f-4185-9bee-3371befc2585", + bucket: "main-preview", + collection: "crash-rate", + last_modified: 1000, + }, + ], + }, + }, + "GET:/fake-x5u": { + sampleHeaders: ["Content-Type: application/octet-stream"], + status: { status: 200, statusText: "OK" }, + responseBody: `-----BEGIN CERTIFICATE----- +MIIGYTCCBEmgAwIBAgIBATANBgkqhkiG9w0BAQwFADB9MQswCQYDVQQGEwJVU +ZARKjbu1TuYQHf0fs+GwID8zeLc2zJL7UzcHFwwQ6Nda9OJN4uPAuC/BKaIpxCLL +26b24/tRam4SJjqpiq20lynhUrmTtt6hbG3E1Hpy3bmkt2DYnuMFwEx2gfXNcnbT +wNuvFqc= +-----END CERTIFICATE-----`, + }, + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=2000": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + 'Etag: "3000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: { + timestamp: 3000, + metadata: { + id: "password-fields", + last_modified: 1234, + signature: { + signature: "abcdef", + x5u: `http://localhost:${port}/fake-x5u`, + }, + }, + changes: [ + { + id: "9d500963-d80e-3a91-6e74-66f3811b99cc", + last_modified: 3000, + website: "https://some-website.com", + selector: "#user[password]", + }, + ], + }, + }, + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=3001&_since=%223000%22": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + 'Etag: "4000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: { + metadata: { + signature: {}, + }, + timestamp: 4000, + changes: [ + { + id: "aabad965-e556-ffe7-4191-074f5dee3df3", + last_modified: 4000, + website: "https://www.other.org/signin", + selector: "#signinpassword", + }, + { + id: "9d500963-d80e-3a91-6e74-66f3811b99cc", + last_modified: 3500, + website: "https://some-website.com/login", + selector: "input#user[password]", + }, + ], + }, + }, + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=4001&_since=%224000%22": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + 'Etag: "5000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: { + metadata: { + signature: {}, + }, + timestamp: 5000, + changes: [ + { + id: "aabad965-e556-ffe7-4191-074f5dee3df3", + deleted: true, + }, + ], + }, + }, + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=10000&_since=%229999%22": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + ], + status: { status: 503, statusText: "Service Unavailable" }, + responseBody: { + code: 503, + errno: 999, + error: "Service Unavailable", + }, + }, + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=10001&_since=%2210000%22": { + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + 'Etag: "10001"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: " ({ + id: `record-${i}`, + ...record, + })), + { + clear: true, + } + ); +} + +function run_test() { + client = RemoteSettings("some-key"); + + run_next_test(); +} + +add_task(async function test_returns_all_without_target() { + await createRecords([ + { + passwordSelector: "#pass-signin", + }, + { + filter_expression: null, + }, + { + filter_expression: "", + }, + ]); + + const list = await client.get(); + equal(list.length, 3); +}); + +add_task(async function test_filters_can_be_disabled() { + const c = RemoteSettings("no-jexl", { filterFunc: null }); + await c.db.importChanges({}, 42, [ + { + id: "abc", + filter_expression: "1 == 2", + }, + ]); + + const list = await c.get(); + equal(list.length, 1); +}); + +add_task(async function test_returns_entries_where_jexl_is_true() { + await createRecords([ + { + willMatch: true, + filter_expression: "1", + }, + { + willMatch: true, + filter_expression: "[42]", + }, + { + willMatch: true, + filter_expression: "1 == 2 || 1 == 1", + }, + { + willMatch: true, + filter_expression: 'env.appinfo.ID == "xpcshell@tests.mozilla.org"', + }, + { + willMatch: false, + filter_expression: "env.version == undefined", + }, + { + willMatch: true, + filter_expression: "env.unknown == undefined", + }, + { + willMatch: false, + filter_expression: "1 == 2", + }, + ]); + + const list = await client.get(); + equal(list.length, 5); + ok(list.every(e => e.willMatch)); +}); + +add_task(async function test_ignores_entries_where_jexl_is_invalid() { + await createRecords([ + { + filter_expression: "true === true", // JavaScript Error: "Invalid expression token: =" + }, + { + filter_expression: "Objects.keys({}) == []", // Token ( (openParen) unexpected in expression + }, + ]); + + const list = await client.get(); + equal(list.length, 0); +}); + +add_task(async function test_support_of_date_filters() { + await createRecords([ + { + willMatch: true, + filter_expression: '"1982-05-08"|date < "2016-03-22"|date', + }, + { + willMatch: false, + filter_expression: '"2000-01-01"|date < "1970-01-01"|date', + }, + ]); + + const list = await client.get(); + equal(list.length, 1); + ok(list.every(e => e.willMatch)); +}); + +add_task(async function test_support_of_preferences_filters() { + await createRecords([ + { + willMatch: true, + filter_expression: '"services.settings.last_etag"|preferenceValue == 42', + }, + { + willMatch: true, + filter_expression: + '"services.settings.default_bucket"|preferenceExists == true', + }, + { + willMatch: true, + filter_expression: + '"services.settings.default_bucket"|preferenceIsUserSet == false', + }, + { + willMatch: true, + filter_expression: + '"services.settings.last_etag"|preferenceIsUserSet == true', + }, + ]); + + // Set a pref for the user. + Services.prefs.setIntPref("services.settings.last_etag", 42); + + const list = await client.get(); + equal(list.length, 4); + ok(list.every(e => e.willMatch)); +}); + +add_task(async function test_support_of_intersect_operator() { + await createRecords([ + { + willMatch: true, + filter_expression: '{foo: 1, bar: 2}|keys intersect ["foo"]', + }, + { + willMatch: true, + filter_expression: '(["a", "b"] intersect ["a", 1, 4]) == "a"', + }, + { + willMatch: false, + filter_expression: '(["a", "b"] intersect [3, 1, 4]) == "c"', + }, + { + willMatch: true, + filter_expression: ` + [1, 2, 3] + intersect + [3, 4, 5] + `, + }, + ]); + + const list = await client.get(); + equal(list.length, 3); + ok(list.every(e => e.willMatch)); +}); + +add_task(async function test_support_of_samples() { + await createRecords([ + { + willMatch: true, + filter_expression: '"always-true"|stableSample(1)', + }, + { + willMatch: false, + filter_expression: '"always-false"|stableSample(0)', + }, + { + willMatch: true, + filter_expression: '"turns-to-true-0"|stableSample(0.5)', + }, + { + willMatch: false, + filter_expression: '"turns-to-false-1"|stableSample(0.5)', + }, + { + willMatch: true, + filter_expression: '"turns-to-true-0"|bucketSample(0, 50, 100)', + }, + { + willMatch: false, + filter_expression: '"turns-to-false-1"|bucketSample(0, 50, 100)', + }, + ]); + + const list = await client.get(); + equal(list.length, 3); + ok(list.every(e => e.willMatch)); +}); diff --git a/services/settings/test/unit/test_remote_settings_poll.js b/services/settings/test/unit/test_remote_settings_poll.js new file mode 100644 index 0000000000..ef49470e8c --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_poll.js @@ -0,0 +1,1135 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm"); + +const { UptakeTelemetry } = ChromeUtils.import( + "resource://services-common/uptake-telemetry.js" +); +const { RemoteSettingsClient } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsClient.jsm" +); +const { pushBroadcastService } = ChromeUtils.import( + "resource://gre/modules/PushBroadcastService.jsm" +); +const { + RemoteSettings, + remoteSettingsBroadcastHandler, + BROADCAST_ID, +} = ChromeUtils.import("resource://services-settings/remote-settings.js"); +const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm"); +const { TelemetryTestUtils } = ChromeUtils.import( + "resource://testing-common/TelemetryTestUtils.jsm" +); + +const IS_ANDROID = AppConstants.platform == "android"; + +const PREF_SETTINGS_SERVER = "services.settings.server"; +const PREF_SETTINGS_SERVER_BACKOFF = "services.settings.server.backoff"; +const PREF_LAST_UPDATE = "services.settings.last_update_seconds"; +const PREF_LAST_ETAG = "services.settings.last_etag"; +const PREF_CLOCK_SKEW_SECONDS = "services.settings.clock_skew_seconds"; + +// Telemetry report result. +const TELEMETRY_HISTOGRAM_POLL_KEY = "settings-changes-monitoring"; +const TELEMETRY_HISTOGRAM_SYNC_KEY = "settings-sync"; +const CHANGES_PATH = "/v1" + Utils.CHANGES_PATH; + +var server; + +async function clear_state() { + // set up prefs so the kinto updater talks to the test server + Services.prefs.setCharPref( + PREF_SETTINGS_SERVER, + `http://localhost:${server.identity.primaryPort}/v1` + ); + + // set some initial values so we can check these are updated appropriately + Services.prefs.setIntPref(PREF_LAST_UPDATE, 0); + Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0); + Services.prefs.clearUserPref(PREF_LAST_ETAG); + + // Clear events snapshot. + TelemetryTestUtils.assertEvents([], {}, { process: "dummy" }); +} + +function serveChangesEntries(serverTime, entries) { + return (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date(serverTime).toUTCString()); + if (entries.length) { + const latest = entries[0].last_modified; + response.setHeader("ETag", `"${latest}"`); + response.setHeader("Last-Modified", new Date(latest).toGMTString()); + } + response.write(JSON.stringify({ data: entries })); + }; +} + +function run_test() { + // Set up an HTTP Server + server = new HttpServer(); + server.start(-1); + + run_next_test(); + + registerCleanupFunction(function() { + server.stop(function() {}); + }); +} + +add_task(clear_state); + +add_task(async function test_an_event_is_sent_on_start() { + server.registerPathHandler(CHANGES_PATH, (request, response) => { + response.write(JSON.stringify({ data: [] })); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("ETag", '"42"'); + response.setHeader("Date", new Date().toUTCString()); + response.setStatusLine(null, 200, "OK"); + }); + let notificationObserved = null; + const observer = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, "remote-settings:changes-poll-start"); + notificationObserved = JSON.parse(aData); + }, + }; + Services.obs.addObserver(observer, "remote-settings:changes-poll-start"); + + await RemoteSettings.pollChanges({ expectedTimestamp: 13 }); + + Assert.equal( + notificationObserved.expectedTimestamp, + 13, + "start notification should have been observed" + ); +}); +add_task(clear_state); + +add_task(async function test_offline_is_reported_if_relevant() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + const offlineBackup = Services.io.offline; + try { + Services.io.offline = true; + + await RemoteSettings.pollChanges(); + + const endHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + const expectedIncrements = { + [UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); + } finally { + Services.io.offline = offlineBackup; + } +}); +add_task(clear_state); + +add_task(async function test_check_success() { + const startPollHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + const startSyncHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_SYNC_KEY + ); + const serverTime = 8000; + + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(serverTime, [ + { + id: "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a", + last_modified: 1100, + host: "localhost", + bucket: "some-other-bucket", + collection: "test-collection", + }, + { + id: "254cbb9e-6888-4d9f-8e60-58b74faa8778", + last_modified: 1000, + host: "localhost", + bucket: "test-bucket", + collection: "test-collection", + }, + ]) + ); + + // add a test kinto client that will respond to lastModified information + // for a collection called 'test-collection'. + // Let's use a bucket that is not the default one (`test-bucket`). + Services.prefs.setCharPref("services.settings.test_bucket", "test-bucket"); + const c = RemoteSettings("test-collection", { + bucketNamePref: "services.settings.test_bucket", + }); + let maybeSyncCalled = false; + c.maybeSync = () => { + maybeSyncCalled = true; + }; + + // Ensure that the remote-settings:changes-poll-end notification works + let notificationObserved = false; + const observer = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, "remote-settings:changes-poll-end"); + notificationObserved = true; + }, + }; + Services.obs.addObserver(observer, "remote-settings:changes-poll-end"); + + await RemoteSettings.pollChanges(); + + // It didn't fail, hence we are sure that the unknown collection ``some-other-bucket/test-collection`` + // was ignored, otherwise it would have tried to reach the network. + + Assert.ok(maybeSyncCalled, "maybeSync was called"); + Assert.ok(notificationObserved, "a notification should have been observed"); + // Last timestamp was saved. An ETag header value is a quoted string. + Assert.equal(Services.prefs.getCharPref(PREF_LAST_ETAG), '"1100"'); + // check the last_update is updated + Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000); + // ensure that we've accumulated the correct telemetry + const endPollHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + const expectedIncrements = { + [UptakeTelemetry.STATUS.SUCCESS]: 1, + }; + checkUptakeTelemetry( + startPollHistogram, + endPollHistogram, + expectedIncrements + ); + const endSyncHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_SYNC_KEY + ); + checkUptakeTelemetry( + startSyncHistogram, + endSyncHistogram, + expectedIncrements + ); +}); +add_task(clear_state); + +add_task(async function test_update_timer_interface() { + const remoteSettings = Cc["@mozilla.org/services/settings;1"].getService( + Ci.nsITimerCallback + ); + + const serverTime = 8000; + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(serverTime, [ + { + id: "028261ad-16d4-40c2-a96a-66f72914d125", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "whatever-collection", + }, + ]) + ); + + await new Promise(resolve => { + const e = "remote-settings:changes-poll-end"; + const changesPolledObserver = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, e); + resolve(); + }, + }; + Services.obs.addObserver(changesPolledObserver, e); + remoteSettings.notify(null); + }); + + // Everything went fine. + Assert.equal(Services.prefs.getCharPref(PREF_LAST_ETAG), '"42"'); + Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000); +}); +add_task(clear_state); + +add_task(async function test_check_up_to_date() { + // Simulate a poll with up-to-date collection. + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + const serverTime = 4000; + function server304(request, response) { + if ( + request.hasHeader("if-none-match") && + request.getHeader("if-none-match") == '"1100"' + ) { + response.setHeader("Date", new Date(serverTime).toUTCString()); + response.setStatusLine(null, 304, "Service Not Modified"); + } + } + server.registerPathHandler(CHANGES_PATH, server304); + + Services.prefs.setCharPref(PREF_LAST_ETAG, '"1100"'); + + // Ensure that the remote-settings:changes-poll-end notification is sent. + let notificationObserved = false; + const observer = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, "remote-settings:changes-poll-end"); + notificationObserved = true; + }, + }; + Services.obs.addObserver(observer, "remote-settings:changes-poll-end"); + + // If server has no change, a 304 is received, maybeSync() is not called. + let maybeSyncCalled = false; + const c = RemoteSettings("test-collection", { + bucketName: "test-bucket", + }); + c.maybeSync = () => { + maybeSyncCalled = true; + }; + + await RemoteSettings.pollChanges(); + + Assert.ok(notificationObserved, "a notification should have been observed"); + Assert.ok(!maybeSyncCalled, "maybeSync should not be called"); + // Last update is overwritten + Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000); + + // ensure that we've accumulated the correct telemetry + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.UP_TO_DATE]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_expected_timestamp() { + function withCacheBust(request, response) { + const entries = [ + { + id: "695c2407-de79-4408-91c7-70720dd59d78", + last_modified: 1100, + host: "localhost", + bucket: "main", + collection: "with-cache-busting", + }, + ]; + if (request.queryString == `_expected=${encodeURIComponent('"42"')}`) { + response.write( + JSON.stringify({ + data: entries, + }) + ); + } + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("ETag", '"1100"'); + response.setHeader("Date", new Date().toUTCString()); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, withCacheBust); + + const c = RemoteSettings("with-cache-busting"); + let maybeSyncCalled = false; + c.maybeSync = () => { + maybeSyncCalled = true; + }; + + await RemoteSettings.pollChanges({ expectedTimestamp: '"42"' }); + + Assert.ok(maybeSyncCalled, "maybeSync was called"); +}); +add_task(clear_state); + +add_task(async function test_client_last_check_is_saved() { + server.registerPathHandler(CHANGES_PATH, (request, response) => { + response.write( + JSON.stringify({ + data: [ + { + id: "695c2407-de79-4408-91c7-70720dd59d78", + last_modified: 1100, + host: "localhost", + bucket: "main", + collection: "models-recipes", + }, + ], + }) + ); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("ETag", '"42"'); + response.setHeader("Date", new Date().toUTCString()); + response.setStatusLine(null, 200, "OK"); + }); + + const c = RemoteSettings("models-recipes"); + c.maybeSync = () => {}; + + equal( + c.lastCheckTimePref, + "services.settings.main.models-recipes.last_check" + ); + Services.prefs.setIntPref(c.lastCheckTimePref, 0); + + await RemoteSettings.pollChanges({ expectedTimestamp: '"42"' }); + + notEqual(Services.prefs.getIntPref(c.lastCheckTimePref), 0); +}); +add_task(clear_state); + +add_task(async function test_age_of_data_is_reported_in_uptake_status() { + await withFakeChannel("nightly", async () => { + const serverTime = 1552323900000; + const recordsTimestamp = serverTime - 3600 * 1000; + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(serverTime, [ + { + id: "b6ba7fab-a40a-4d03-a4af-6b627f3c5b36", + last_modified: recordsTimestamp, + host: "localhost", + bucket: "main", + collection: "some-entry", + }, + ]) + ); + + await RemoteSettings.pollChanges(); + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.SUCCESS, + { + source: TELEMETRY_HISTOGRAM_POLL_KEY, + age: "3600", + trigger: "manual", + }, + ], + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.SUCCESS, + { + source: TELEMETRY_HISTOGRAM_SYNC_KEY, + duration: () => true, + trigger: "manual", + timestamp: `"${recordsTimestamp}"`, + }, + ], + ]); + }); +}); +add_task(clear_state); + +add_task( + async function test_synchronization_duration_is_reported_in_uptake_status() { + await withFakeChannel("nightly", async () => { + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(10000, [ + { + id: "b6ba7fab-a40a-4d03-a4af-6b627f3c5b36", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "some-entry", + }, + ]) + ); + const c = RemoteSettings("some-entry"); + // Simulate a synchronization that lasts 1 sec. + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + c.maybeSync = () => new Promise(resolve => setTimeout(resolve, 1000)); + + await RemoteSettings.pollChanges(); + + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + "success", + { + source: TELEMETRY_HISTOGRAM_POLL_KEY, + age: () => true, + trigger: "manual", + }, + ], + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + "success", + { + source: TELEMETRY_HISTOGRAM_SYNC_KEY, + duration: v => v >= 1000, + trigger: "manual", + }, + ], + ]); + }); + } +); +add_task(clear_state); + +add_task(async function test_success_with_partial_list() { + function partialList(request, response) { + const entries = [ + { + id: "028261ad-16d4-40c2-a96a-66f72914d125", + last_modified: 43, + host: "localhost", + bucket: "main", + collection: "cid-1", + }, + { + id: "98a34576-bcd6-423f-abc2-1d290b776ed8", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "poll-test-collection", + }, + ]; + if (request.queryString == `_since=${encodeURIComponent('"42"')}`) { + response.write( + JSON.stringify({ + data: entries.slice(0, 1), + }) + ); + response.setHeader("ETag", '"43"'); + } else { + response.write( + JSON.stringify({ + data: entries, + }) + ); + response.setHeader("ETag", '"42"'); + } + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date().toUTCString()); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, partialList); + + const c = RemoteSettings("poll-test-collection"); + let maybeSyncCount = 0; + c.maybeSync = () => { + maybeSyncCount++; + }; + + await RemoteSettings.pollChanges(); + await RemoteSettings.pollChanges(); + + // On the second call, the server does not mention the poll-test-collection + // and maybeSync() is not called. + Assert.equal(maybeSyncCount, 1, "maybeSync should not be called twice"); +}); +add_task(clear_state); + +add_task(async function test_full_polling() { + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(10000, [ + { + id: "b6ba7fab-a40a-4d03-a4af-6b627f3c5b36", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "poll-test-collection", + }, + ]) + ); + + const c = RemoteSettings("poll-test-collection"); + let maybeSyncCount = 0; + c.maybeSync = () => { + maybeSyncCount++; + }; + + await RemoteSettings.pollChanges(); + await RemoteSettings.pollChanges({ full: true }); + + // Since the second call is full, clients are called + Assert.equal(maybeSyncCount, 2, "maybeSync should be called twice"); +}); +add_task(clear_state); + +add_task(async function test_server_bad_json() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + function simulateBadJSON(request, response) { + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.write(""); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, simulateBadJSON); + + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + Assert.ok(/JSON.parse: unexpected character/.test(error.message)); + + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.PARSE_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_server_bad_content_type() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + function simulateBadContentType(request, response) { + response.setHeader("Content-Type", "text/html"); + response.write(""); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, simulateBadContentType); + + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + Assert.ok(/Unexpected content-type/.test(error.message)); + + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.CONTENT_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_server_404_response() { + function simulateDummy404(request, response) { + response.setHeader("Content-Type", "text/html; charset=UTF-8"); + response.write(""); + response.setStatusLine(null, 404, "OK"); + } + server.registerPathHandler(CHANGES_PATH, simulateDummy404); + + await RemoteSettings.pollChanges(); // Does not fail when running from tests. +}); +add_task(clear_state); + +add_task(async function test_server_error() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + // Simulate a server error. + function simulateErrorResponse(request, response) { + response.setHeader("Date", new Date(3000).toUTCString()); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.write( + JSON.stringify({ + code: 503, + errno: 999, + error: "Service Unavailable", + }) + ); + response.setStatusLine(null, 503, "Service Unavailable"); + } + server.registerPathHandler(CHANGES_PATH, simulateErrorResponse); + + let notificationObserved = false; + const observer = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, "remote-settings:changes-poll-end"); + notificationObserved = true; + }, + }; + Services.obs.addObserver(observer, "remote-settings:changes-poll-end"); + Services.prefs.setIntPref(PREF_LAST_UPDATE, 42); + + // pollChanges() fails with adequate error and no notification. + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + + Assert.ok( + !notificationObserved, + "a notification should not have been observed" + ); + Assert.ok(/Polling for changes failed/.test(error.message)); + // When an error occurs, last update was not overwritten. + Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 42); + // ensure that we've accumulated the correct telemetry + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.SERVER_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_server_error_5xx() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + function simulateErrorResponse(request, response) { + response.setHeader("Date", new Date(3000).toUTCString()); + response.setHeader("Content-Type", "text/html; charset=UTF-8"); + response.write(""); + response.setStatusLine(null, 504, "Gateway Timeout"); + } + server.registerPathHandler(CHANGES_PATH, simulateErrorResponse); + + try { + await RemoteSettings.pollChanges(); + } catch (e) {} + + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.SERVER_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_client_error() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_SYNC_KEY + ); + + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(10000, [ + { + id: "b6ba7fab-a40a-4d03-a4af-6b627f3c5b36", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "some-entry", + }, + ]) + ); + const c = RemoteSettings("some-entry"); + c.maybeSync = () => { + throw new Error("boom"); + }; + + let notificationObserved = false; + const observer = { + observe(aSubject, aTopic, aData) { + Services.obs.removeObserver(this, "remote-settings:changes-poll-end"); + notificationObserved = true; + }, + }; + Services.obs.addObserver(observer, "remote-settings:changes-poll-end"); + Services.prefs.setIntPref(PREF_LAST_ETAG, 42); + + // pollChanges() fails with adequate error and no notification. + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + + Assert.ok( + !notificationObserved, + "a notification should not have been observed" + ); + Assert.ok(/boom/.test(error.message), "original client error is thrown"); + // When an error occurs, last etag was not overwritten. + Assert.equal(Services.prefs.getIntPref(PREF_LAST_ETAG), 42); + // ensure that we've accumulated the correct telemetry + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_SYNC_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.SYNC_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_check_clockskew_is_updated() { + const serverTime = 2000; + + function serverResponse(request, response) { + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date(serverTime).toUTCString()); + response.write(JSON.stringify({ data: [] })); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, serverResponse); + + let startTime = Date.now(); + + await RemoteSettings.pollChanges(); + + // How does the clock difference look? + let endTime = Date.now(); + let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS); + // we previously set the serverTime to 2 (seconds past epoch) + Assert.ok( + clockDifference <= endTime / 1000 && + clockDifference >= Math.floor(startTime / 1000) - serverTime / 1000 + ); + + // check negative clock skew times + // set to a time in the future + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(Date.now() + 10000, []) + ); + + await RemoteSettings.pollChanges(); + + clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS); + // we previously set the serverTime to Date.now() + 10000 ms past epoch + Assert.ok(clockDifference <= 0 && clockDifference >= -10); +}); +add_task(clear_state); + +add_task(async function test_check_clockskew_takes_age_into_account() { + const currentTime = Date.now(); + const skewSeconds = 5; + const ageCDNSeconds = 3600; + const serverTime = currentTime - skewSeconds * 1000 - ageCDNSeconds * 1000; + + function serverResponse(request, response) { + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date(serverTime).toUTCString()); + response.setHeader("Age", `${ageCDNSeconds}`); + response.write(JSON.stringify({ data: [] })); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, serverResponse); + + await RemoteSettings.pollChanges(); + + const clockSkew = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS); + Assert.ok(clockSkew >= skewSeconds, `clockSkew is ${clockSkew}`); +}); +add_task(clear_state); + +add_task(async function test_backoff() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + function simulateBackoffResponse(request, response) { + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Backoff", "10"); + response.write(JSON.stringify({ data: [] })); + response.setStatusLine(null, 200, "OK"); + } + server.registerPathHandler(CHANGES_PATH, simulateBackoffResponse); + + // First will work. + await RemoteSettings.pollChanges(); + // Second will fail because we haven't waited. + try { + await RemoteSettings.pollChanges(); + // The previous line should have thrown an error. + Assert.ok(false); + } catch (e) { + Assert.ok( + /Server is asking clients to back off; retry in \d+s./.test(e.message) + ); + } + + // Once backoff time has expired, polling for changes can start again. + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(12000, [ + { + id: "6a733d4a-601e-11e8-837a-0f85257529a1", + last_modified: 1300, + host: "localhost", + bucket: "some-bucket", + collection: "some-collection", + }, + ]) + ); + Services.prefs.setCharPref( + PREF_SETTINGS_SERVER_BACKOFF, + `${Date.now() - 1000}` + ); + + await RemoteSettings.pollChanges(); + + // Backoff tracking preference was cleared. + Assert.ok(!Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)); + + // Ensure that we've accumulated the correct telemetry + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.SUCCESS]: 1, + [UptakeTelemetry.STATUS.UP_TO_DATE]: 1, + [UptakeTelemetry.STATUS.BACKOFF]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_network_error() { + const startHistogram = getUptakeTelemetrySnapshot( + TELEMETRY_HISTOGRAM_POLL_KEY + ); + + // Simulate a network error (to check telemetry report). + Services.prefs.setCharPref(PREF_SETTINGS_SERVER, "http://localhost:42/v1"); + try { + await RemoteSettings.pollChanges(); + } catch (e) {} + + // ensure that we've accumulated the correct telemetry + const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_POLL_KEY); + const expectedIncrements = { + [UptakeTelemetry.STATUS.NETWORK_ERROR]: 1, + }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); +}); +add_task(clear_state); + +add_task(async function test_syncs_clients_with_local_database() { + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(42000, [ + { + id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844", + last_modified: 10000, + host: "localhost", + bucket: "main", + collection: "some-unknown", + }, + { + id: "39f57e4e-6023-11e8-8b74-77c8dedfb389", + last_modified: 9000, + host: "localhost", + bucket: "blocklists", + collection: "addons", + }, + { + id: "9a594c1a-601f-11e8-9c8a-33b2239d9113", + last_modified: 8000, + host: "localhost", + bucket: "main", + collection: "recipes", + }, + ]) + ); + + // This simulates what remote-settings would do when initializing a local database. + // We don't want to instantiate a client using the RemoteSettings() API + // since we want to test «unknown» clients that have a local database. + new RemoteSettingsClient("addons", { + bucketNamePref: "services.blocklist.bucket", // bucketName = "blocklists" + }).db.importChanges({}, 42); + new RemoteSettingsClient("recipes", { + bucketNamePref: "services.settings.default_bucket", // bucketName = "main" + }).db.importChanges({}, 43); + + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + + // The `main/some-unknown` should be skipped because it has no local database. + // The `blocklists/addons` should be skipped because it is not the main bucket. + // The `recipes` has a local database, and should cause a network error because the test + // does not setup the server to receive the requests of `maybeSync()`. + Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync"); + Assert.equal(error.details.collection, "recipes"); +}); +add_task(clear_state); + +add_task(async function test_syncs_clients_with_local_dump() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + server.registerPathHandler( + CHANGES_PATH, + serveChangesEntries(42000, [ + { + id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844", + last_modified: 10000, + host: "localhost", + bucket: "main", + collection: "some-unknown", + }, + { + id: "39f57e4e-6023-11e8-8b74-77c8dedfb389", + last_modified: 9000, + host: "localhost", + bucket: "blocklists", + collection: "addons", + }, + { + id: "9a594c1a-601f-11e8-9c8a-33b2239d9113", + last_modified: 8000, + host: "localhost", + bucket: "main", + collection: "example", + }, + ]) + ); + + let error; + try { + await RemoteSettings.pollChanges(); + } catch (e) { + error = e; + } + + // The `main/some-unknown` should be skipped because it has no dump. + // The `blocklists/addons` should be skipped because it is not the main bucket. + // The `example` has a dump, and should cause a network error because the test + // does not setup the server to receive the requests of `maybeSync()`. + Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync"); + Assert.equal(error.details.collection, "example"); +}); +add_task(clear_state); + +add_task(async function test_adding_client_resets_polling() { + function serve200or304(request, response) { + const entries = [ + { + id: "aa71e6cc-9f37-447a-b6e0-c025e8eabd03", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "a-collection", + }, + ]; + if (request.queryString == `_since=${encodeURIComponent('"42"')}`) { + response.write( + JSON.stringify({ + data: entries.slice(0, 1), + }) + ); + response.setHeader("ETag", '"42"'); + response.setStatusLine(null, 304, "Not Modified"); + } else { + response.write( + JSON.stringify({ + data: entries, + }) + ); + response.setHeader("ETag", '"42"'); + response.setStatusLine(null, 200, "OK"); + } + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date().toUTCString()); + } + server.registerPathHandler(CHANGES_PATH, serve200or304); + + // Poll once, without any client for "a-collection" + await RemoteSettings.pollChanges(); + + // Register a new client. + let maybeSyncCalled = false; + const c = RemoteSettings("a-collection"); + c.maybeSync = () => { + maybeSyncCalled = true; + }; + + // Poll again. + await RemoteSettings.pollChanges(); + + // The new client was called, even if the server data didn't change. + Assert.ok(maybeSyncCalled); + + // Poll again. This time maybeSync() won't be called. + maybeSyncCalled = false; + await RemoteSettings.pollChanges(); + Assert.ok(!maybeSyncCalled); +}); +add_task(clear_state); + +add_task( + async function test_broadcast_handler_passes_version_and_trigger_values() { + // The polling will use the broadcast version as cache busting query param. + let passedQueryString; + function serveCacheBusted(request, response) { + passedQueryString = request.queryString; + const entries = [ + { + id: "b6ba7fab-a40a-4d03-a4af-6b627f3c5b36", + last_modified: 42, + host: "localhost", + bucket: "main", + collection: "from-broadcast", + }, + ]; + response.write( + JSON.stringify({ + data: entries, + }) + ); + response.setHeader("ETag", '"42"'); + response.setStatusLine(null, 200, "OK"); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date().toUTCString()); + } + server.registerPathHandler(CHANGES_PATH, serveCacheBusted); + + let passedTrigger; + const c = RemoteSettings("from-broadcast"); + c.maybeSync = (last_modified, { trigger }) => { + passedTrigger = trigger; + }; + + const version = "1337"; + + let context = { phase: pushBroadcastService.PHASES.HELLO }; + await remoteSettingsBroadcastHandler.receivedBroadcastMessage( + version, + BROADCAST_ID, + context + ); + Assert.equal(passedTrigger, "startup"); + Assert.equal(passedQueryString, `_expected=${version}`); + + clear_state(); + + context = { phase: pushBroadcastService.PHASES.REGISTER }; + await remoteSettingsBroadcastHandler.receivedBroadcastMessage( + version, + BROADCAST_ID, + context + ); + Assert.equal(passedTrigger, "startup"); + + clear_state(); + + context = { phase: pushBroadcastService.PHASES.BROADCAST }; + await remoteSettingsBroadcastHandler.receivedBroadcastMessage( + version, + BROADCAST_ID, + context + ); + Assert.equal(passedTrigger, "broadcast"); + } +); +add_task(clear_state); diff --git a/services/settings/test/unit/test_remote_settings_signatures.js b/services/settings/test/unit/test_remote_settings_signatures.js new file mode 100644 index 0000000000..711bb7cf87 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_signatures.js @@ -0,0 +1,827 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +"use strict"; + +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { RemoteSettingsClient } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsClient.jsm" +); +const { UptakeTelemetry } = ChromeUtils.import( + "resource://services-common/uptake-telemetry.js" +); +const { TelemetryTestUtils } = ChromeUtils.import( + "resource://testing-common/TelemetryTestUtils.jsm" +); + +const PREF_SETTINGS_SERVER = "services.settings.server"; +const PREF_SIGNATURE_ROOT = "security.content.signature.root_hash"; +const SIGNER_NAME = "onecrl.content-signature.mozilla.org"; + +const CERT_DIR = "test_remote_settings_signatures/"; +const CHAIN_FILES = [ + "collection_signing_ee.pem", + "collection_signing_int.pem", + "collection_signing_root.pem", +]; + +function getFileData(file) { + const stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + stream.init(file, -1, 0, 0); + const data = NetUtil.readInputStreamToString(stream, stream.available()); + stream.close(); + return data; +} + +function setRoot() { + const filename = CERT_DIR + CHAIN_FILES[0]; + + const certFile = do_get_file(filename, false); + const b64cert = getFileData(certFile) + .replace(/-----BEGIN CERTIFICATE-----/, "") + .replace(/-----END CERTIFICATE-----/, "") + .replace(/[\r\n]/g, ""); + const certdb = Cc["@mozilla.org/security/x509certdb;1"].getService( + Ci.nsIX509CertDB + ); + const cert = certdb.constructX509FromBase64(b64cert); + Services.prefs.setCharPref(PREF_SIGNATURE_ROOT, cert.sha256Fingerprint); +} + +function getCertChain() { + const chain = []; + for (let file of CHAIN_FILES) { + chain.push(getFileData(do_get_file(CERT_DIR + file))); + } + return chain.join("\n"); +} + +let server; +let client; + +function run_test() { + // Signature verification is enabled by default. We use a custom signer + // because these tests were originally written for OneCRL. + client = RemoteSettings("signed", { signerName: SIGNER_NAME }); + + Services.prefs.setCharPref("services.settings.loglevel", "debug"); + + // set the content signing root to our test root + setRoot(); + + // Set up an HTTP Server + server = new HttpServer(); + server.start(-1); + + run_next_test(); + + registerCleanupFunction(() => server.stop(() => {})); +} + +add_task(async function test_check_signatures() { + // First, perform a signature verification with known data and signature + // to ensure things are working correctly + let verifier = Cc[ + "@mozilla.org/security/contentsignatureverifier;1" + ].createInstance(Ci.nsIContentSignatureVerifier); + + const emptyData = "[]"; + const emptySignature = + "p384ecdsa=zbugm2FDitsHwk5-IWsas1PpWwY29f0Fg5ZHeqD8fzep7AVl2vfcaHA7LdmCZ28qZLOioGKvco3qT117Q4-HlqFTJM7COHzxGyU2MMJ0ZTnhJrPOC1fP3cVQjU1PTWi9"; + + ok( + await verifier.asyncVerifyContentSignature( + emptyData, + emptySignature, + getCertChain(), + SIGNER_NAME + ) + ); + + const collectionData = + '[{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:43:37Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"97fbf7c4-3ef2-f54f-0029-1ba6540c63ea","issuerName":"MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==","last_modified":2000,"serialNumber":"BAAAAAABA/A35EU="},{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:48:11Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc","issuerName":"MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB","last_modified":3000,"serialNumber":"BAAAAAABI54PryQ="}]'; + const collectionSignature = + "p384ecdsa=f4pA2tYM5jQgWY6YUmhUwQiBLj6QO5sHLD_5MqLePz95qv-7cNCuQoZnPQwxoptDtW8hcWH3kLb0quR7SB-r82gkpR9POVofsnWJRA-ETb0BcIz6VvI3pDT49ZLlNg3p"; + + ok( + await verifier.asyncVerifyContentSignature( + collectionData, + collectionSignature, + getCertChain(), + SIGNER_NAME + ) + ); +}); + +add_task(async function test_check_synchronization_with_signatures() { + const port = server.identity.primaryPort; + + const x5u = `http://localhost:${port}/test_remote_settings_signatures/test_cert_chain.pem`; + + // Telemetry reports. + const TELEMETRY_HISTOGRAM_KEY = client.identifier; + + function registerHandlers(responses) { + function handleResponse(serverTimeMillis, request, response) { + const key = `${request.method}:${request.path}?${request.queryString}`; + const available = responses[key]; + const sampled = available.length > 1 ? available.shift() : available[0]; + if (!sampled) { + do_throw( + `unexpected ${request.method} request for ${request.path}?${request.queryString}` + ); + } + + response.setStatusLine( + null, + sampled.status.status, + sampled.status.statusText + ); + // send the headers + for (let headerLine of sampled.sampleHeaders) { + let headerElements = headerLine.split(":"); + response.setHeader(headerElements[0], headerElements[1].trimLeft()); + } + + // set the server date + response.setHeader("Date", new Date(serverTimeMillis).toUTCString()); + + response.write(sampled.responseBody); + } + + for (let key of Object.keys(responses)) { + const keyParts = key.split(":"); + const valueParts = keyParts[1].split("?"); + const path = valueParts[0]; + + server.registerPathHandler(path, handleResponse.bind(null, 2000)); + } + } + + // set up prefs so the kinto updater talks to the test server + Services.prefs.setCharPref( + PREF_SETTINGS_SERVER, + `http://localhost:${server.identity.primaryPort}/v1` + ); + + // These are records we'll use in the test collections + const RECORD1 = { + details: { + bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145", + created: "2016-01-18T14:43:37Z", + name: "GlobalSign certs", + who: ".", + why: ".", + }, + enabled: true, + id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea", + issuerName: + "MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==", + last_modified: 2000, + serialNumber: "BAAAAAABA/A35EU=", + }; + + const RECORD2 = { + details: { + bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145", + created: "2016-01-18T14:48:11Z", + name: "GlobalSign certs", + who: ".", + why: ".", + }, + enabled: true, + id: "e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc", + issuerName: + "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB", + last_modified: 3000, + serialNumber: "BAAAAAABI54PryQ=", + }; + + const RECORD3 = { + details: { + bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145", + created: "2016-01-18T14:48:11Z", + name: "GlobalSign certs", + who: ".", + why: ".", + }, + enabled: true, + id: "c7c49b69-a4ab-418e-92a9-e1961459aa7f", + issuerName: + "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB", + last_modified: 4000, + serialNumber: "BAAAAAABI54PryQ=", + }; + + const RECORD1_DELETION = { + deleted: true, + enabled: true, + id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea", + last_modified: 3500, + }; + + // Check that a signature on an empty collection is OK + // We need to set up paths on the HTTP server to return specific data from + // specific paths for each test. Here we prepare data for each response. + + // A cert chain response (this the cert chain that contains the signing + // cert, the root and any intermediates in between). This is used in each + // sync. + const RESPONSE_CERT_CHAIN = { + comment: "RESPONSE_CERT_CHAIN", + sampleHeaders: ["Content-Type: text/plain; charset=UTF-8"], + status: { status: 200, statusText: "OK" }, + responseBody: getCertChain(), + }; + + // A server settings response. This is used in each sync. + const RESPONSE_SERVER_SETTINGS = { + comment: "RESPONSE_SERVER_SETTINGS", + sampleHeaders: [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + settings: { + batch_max_requests: 25, + }, + url: `http://localhost:${port}/v1/`, + documentation: "https://kinto.readthedocs.org/", + version: "1.5.1", + commit: "cbc6f58", + hello: "kinto", + }), + }; + + // This is the initial, empty state of the collection. This is only used + // for the first sync. + const RESPONSE_EMPTY_INITIAL = { + comment: "RESPONSE_EMPTY_INITIAL", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "1000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 1000, + metadata: { + signature: { + x5u, + signature: + "vxuAg5rDCB-1pul4a91vqSBQRXJG_j7WOYUTswxRSMltdYmbhLRH8R8brQ9YKuNDF56F-w6pn4HWxb076qgKPwgcEBtUeZAO_RtaHXRkRUUgVzAr86yQL4-aJTbv3D6u", + }, + }, + changes: [], + }), + }; + + // Here, we map request method and path to the available responses + const emptyCollectionResponses = { + "GET:/test_remote_settings_signatures/test_cert_chain.pem?": [ + RESPONSE_CERT_CHAIN, + ], + "GET:/v1/?": [RESPONSE_SERVER_SETTINGS], + "GET:/v1/buckets/main/collections/signed/changeset?_expected=1000": [ + RESPONSE_EMPTY_INITIAL, + ], + }; + + // + // 1. + // - collection: undefined -> [] + // - timestamp: undefined -> 1000 + // + + // .. and use this map to register handlers for each path + registerHandlers(emptyCollectionResponses); + + let startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + + // With all of this set up, we attempt a sync. This will resolve if all is + // well and throw if something goes wrong. + await client.maybeSync(1000); + + equal((await client.get()).length, 0); + + let endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + + // ensure that a success histogram is tracked when a succesful sync occurs. + let expectedIncrements = { [UptakeTelemetry.STATUS.SUCCESS]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); + + // + // 2. + // - collection: [] -> [RECORD2, RECORD1] + // - timestamp: 1000 -> 3000 + // + // Check that some additions (2 records) to the collection have a valid + // signature. + + // This response adds two entries (RECORD1 and RECORD2) to the collection + const RESPONSE_TWO_ADDED = { + comment: "RESPONSE_TWO_ADDED", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "3000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 3000, + metadata: { + signature: { + x5u, + signature: + "dwhJeypadNIyzGj3QdI0KMRTPnHhFPF_j73mNrsPAHKMW46S2Ftf4BzsPMvPMB8h0TjDus13wo_R4l432DHe7tYyMIWXY0PBeMcoe5BREhFIxMxTsh9eGVXBD1e3UwRy", + }, + }, + changes: [RECORD2, RECORD1], + }), + }; + + const twoItemsResponses = { + "GET:/v1/buckets/main/collections/signed/changeset?_expected=3000&_since=%221000%22": [ + RESPONSE_TWO_ADDED, + ], + }; + registerHandlers(twoItemsResponses); + await client.maybeSync(3000); + + equal((await client.get()).length, 2); + + // + // 3. + // - collection: [RECORD2, RECORD1] -> [RECORD2, RECORD3] + // - timestamp: 3000 -> 4000 + // + // Check the collection with one addition and one removal has a valid + // signature + const THREE_ITEMS_SIG = + "MIEmNghKnkz12UodAAIc3q_Y4a3IJJ7GhHF4JYNYmm8avAGyPM9fYU7NzVo94pzjotG7vmtiYuHyIX2rTHTbT587w0LdRWxipgFd_PC1mHiwUyjFYNqBBG-kifYk7kEw"; + + // Remove RECORD1, add RECORD3 + const RESPONSE_ONE_ADDED_ONE_REMOVED = { + comment: "RESPONSE_ONE_ADDED_ONE_REMOVED ", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "4000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: THREE_ITEMS_SIG, + }, + }, + changes: [RECORD3, RECORD1_DELETION], + }), + }; + + const oneAddedOneRemovedResponses = { + "GET:/v1/buckets/main/collections/signed/changeset?_expected=4000&_since=%223000%22": [ + RESPONSE_ONE_ADDED_ONE_REMOVED, + ], + }; + registerHandlers(oneAddedOneRemovedResponses); + await client.maybeSync(4000); + + equal((await client.get()).length, 2); + + // + // 4. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] + // - timestamp: 4000 -> 4100 + // + // Check the signature is still valid with no operation (no changes) + + // Leave the collection unchanged + const RESPONSE_EMPTY_NO_UPDATE = { + comment: "RESPONSE_EMPTY_NO_UPDATE ", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "4000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: THREE_ITEMS_SIG, + }, + }, + changes: [], + }), + }; + + const noOpResponses = { + "GET:/v1/buckets/main/collections/signed/changeset?_expected=4100&_since=%224000%22": [ + RESPONSE_EMPTY_NO_UPDATE, + ], + }; + registerHandlers(noOpResponses); + await client.maybeSync(4100); + + equal((await client.get()).length, 2); + + console.info("---------------------------------------------------------"); + // + // 5. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] + // - timestamp: 4000 -> 5000 + // + // Check the collection is reset when the signature is invalid. + // Client will: + // - Fetch metadata (with bad signature) + // - Perform the sync (fetch empty changes) + // - Refetch the metadata and the whole collection + // - Validate signature successfully, but with no changes to emit. + + const RESPONSE_COMPLETE_INITIAL = { + comment: "RESPONSE_COMPLETE_INITIAL ", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "4000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: THREE_ITEMS_SIG, + }, + }, + changes: [RECORD2, RECORD3], + }), + }; + + const RESPONSE_EMPTY_NO_UPDATE_BAD_SIG = { + ...RESPONSE_EMPTY_NO_UPDATE, + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: "aW52YWxpZCBzaWduYXR1cmUK", + }, + }, + changes: [], + }), + }; + + const badSigGoodSigResponses = { + // The first collection state is the three item collection (since + // there was sync with no updates before) - but, since the signature is wrong, + // another request will be made... + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=%224000%22": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG, + ], + // Subsequent signature returned is a valid one for the three item + // collection. + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ + RESPONSE_COMPLETE_INITIAL, + ], + }; + + registerHandlers(badSigGoodSigResponses); + + startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + + let syncEventSent = false; + client.on("sync", ({ data }) => { + syncEventSent = true; + }); + + await client.maybeSync(5000); + + equal((await client.get()).length, 2); + + endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + + // since we only fixed the signature, and no data was changed, the sync event + // was not sent. + equal(syncEventSent, false); + + // ensure that the failure count is incremented for a succesful sync with an + // (initial) bad signature - only SERVICES_SETTINGS_SYNC_SIG_FAIL should + // increment. + expectedIncrements = { [UptakeTelemetry.STATUS.SIGNATURE_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); + + // + // 6. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] + // - timestamp: 4000 -> 5000 + // + // Check the collection is reset when the signature is invalid. + // Client will: + // - Fetch metadata (with bad signature) + // - Perform the sync (fetch empty changes) + // - Refetch the whole collection and metadata + // - Sync will be no-op since local is equal to server, no changes to emit. + + const badSigGoodOldResponses = { + // The first collection state is the current state (since there's no update + // - but, since the signature is wrong, another request will be made) + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=%224000%22": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG, + ], + // The next request is for the full collection. This will be + // checked against the valid signature and last_modified times will be + // compared. Sync should be a no-op, even though the signature is good, + // because the local collection is newer. + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ + RESPONSE_EMPTY_INITIAL, + ], + }; + + // ensure our collection hasn't been replaced with an older, empty one + equal((await client.get()).length, 2, "collection was restored"); + + registerHandlers(badSigGoodOldResponses); + + syncEventSent = false; + client.on("sync", ({ data }) => { + syncEventSent = true; + }); + + await client.maybeSync(5000); + + // Local data was unchanged, since it was never than the one returned by the server, + // thus the sync event is not sent. + equal(syncEventSent, false, "event was not sent"); + + // + // 7. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] + // - timestamp: 4000 -> 5000 + // + // Check that a tampered local DB will be overwritten and + // sync event contain the appropriate data. + + const RESPONSE_COMPLETE_BAD_SIG = { + ...RESPONSE_EMPTY_NO_UPDATE, + responseBody: JSON.stringify({ + timestamp: 5000, + metadata: { + signature: { + x5u, + signature: "aW52YWxpZCBzaWduYXR1cmUK", + }, + }, + changes: [RECORD2, RECORD3], + }), + }; + + const badLocalContentGoodSigResponses = { + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ + RESPONSE_COMPLETE_BAD_SIG, + RESPONSE_COMPLETE_INITIAL, + ], + }; + + registerHandlers(badLocalContentGoodSigResponses); + + // we create a local state manually here, in order to test that the sync event data + // properly contains created, updated, and deleted records. + // the local DB contains same id as RECORD2 and a fake record. + // the final server collection contains RECORD2 and RECORD3 + const localId = "0602b1b2-12ab-4d3a-b6fb-593244e7b035"; + await client.db.importChanges( + { signature: { x5u, signature: "abc" } }, + null, + [ + { ...RECORD2, last_modified: 1234567890, serialNumber: "abc" }, + { id: localId }, + ], + { + clear: true, + } + ); + + let syncData = null; + client.on("sync", ({ data }) => { + syncData = data; + }); + + // Clear events snapshot. + TelemetryTestUtils.assertEvents([], {}, { process: "dummy" }); + + await withFakeChannel("nightly", async () => { + // Events telemetry is sampled on released, use fake channel. + await client.maybeSync(5000); + + // We should report a corruption_error. + TelemetryTestUtils.assertEvents([ + [ + "uptake.remotecontent.result", + "uptake", + "remotesettings", + UptakeTelemetry.STATUS.CORRUPTION_ERROR, + { + source: client.identifier, + duration: v => v > 0, + trigger: "manual", + }, + ], + ]); + }); + + // The local data was corrupted, and the Telemetry status reflects it. + // But the sync overwrote the bad data and was eventually a success. + // Since local data was replaced, we use records IDs to determine + // what was created and deleted. And bad local data will appear + // in the sync event as deleted. + equal(syncData.current.length, 2); + equal(syncData.created.length, 1); + equal(syncData.created[0].id, RECORD3.id); + equal(syncData.updated.length, 1); + equal(syncData.updated[0].old.serialNumber, "abc"); + equal(syncData.updated[0].new.serialNumber, RECORD2.serialNumber); + equal(syncData.deleted.length, 1); + equal(syncData.deleted[0].id, localId); + + // + // 8. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] (unchanged because of error) + // - timestamp: 4000 -> 6000 + // + // Check that a failing signature throws after retry, and that sync changes + // are not applied. + + const RESPONSE_ONLY_RECORD4_BAD_SIG = { + comment: "Create RECORD4", + sampleHeaders: [ + "Content-Type: application/json; charset=UTF-8", + 'ETag: "6000"', + ], + status: { status: 200, statusText: "OK" }, + responseBody: JSON.stringify({ + timestamp: 6000, + metadata: { + signature: { + x5u, + signature: "aaaaaaaaaaaaaaaaaaaaaaaa", // sig verifier wants proper length or will crash. + }, + }, + changes: [ + { + id: "f765df30-b2f1-42f6-9803-7bd5a07b5098", + last_modified: 6000, + }, + ], + }), + }; + const RESPONSE_EMPTY_NO_UPDATE_BAD_SIG_6000 = { + ...RESPONSE_EMPTY_NO_UPDATE, + responseBody: JSON.stringify({ + timestamp: 6000, + metadata: { + signature: { + x5u, + signature: "aW52YWxpZCBzaWduYXR1cmUK", + }, + }, + changes: [], + }), + }; + const allBadSigResponses = { + "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000&_since=%224000%22": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG_6000, + ], + "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000": [ + RESPONSE_ONLY_RECORD4_BAD_SIG, + ], + }; + + startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + registerHandlers(allBadSigResponses); + await Assert.rejects( + client.maybeSync(6000), + RemoteSettingsClient.InvalidSignatureError, + "Sync failed as expected (bad signature after retry)" + ); + + // Ensure that the failure is reflected in the accumulated telemetry: + endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY); + expectedIncrements = { [UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR]: 1 }; + checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements); + + // When signature fails after retry, the local data present before sync + // should be maintained (if its signature is valid). + ok( + arrayEqual( + (await client.get()).map(r => r.id), + [RECORD3.id, RECORD2.id] + ), + "Local records were not changed" + ); + // And local data should still be valid. + await client.get({ verifySignature: true }); // Not raising. + + // + // 9. + // - collection: [RECORD2, RECORD3] -> [] (cleared) + // - timestamp: 4000 -> 6000 + // + // Check that local data is cleared during sync if signature is not valid. + + await client.db.create({ + id: "c6b19c67-2e0e-4a82-b7f7-1777b05f3e81", + last_modified: 42, + tampered: true, + }); + + await Assert.rejects( + client.maybeSync(6000), + RemoteSettingsClient.InvalidSignatureError, + "Sync failed as expected (bad signature after retry)" + ); + + // Since local data was tampered, it was cleared. + equal((await client.get()).length, 0, "Local database is now empty."); + + // + // 10. + // - collection: [RECORD2, RECORD3] -> [] (cleared) + // - timestamp: 4000 -> 6000 + // + // Check that local data is cleared during sync if signature is not valid. + + await client.db.create({ + id: "c6b19c67-2e0e-4a82-b7f7-1777b05f3e81", + last_modified: 42, + tampered: true, + }); + + await Assert.rejects( + client.maybeSync(6000), + RemoteSettingsClient.InvalidSignatureError, + "Sync failed as expected (bad signature after retry)" + ); + // Since local data was tampered, it was cleared. + equal((await client.get()).length, 0, "Local database is now empty."); + + // + // 11. + // - collection: [RECORD2, RECORD3] -> [RECORD2, RECORD3] + // - timestamp: 4000 -> 6000 + // + // Check that local data is restored if signature was valid before sync. + const sigCalls = []; + let i = 0; + client._verifier = { + async asyncVerifyContentSignature(serialized, signature) { + sigCalls.push(serialized); + console.log(`verify call ${i}`); + return [ + false, // After importing changes. + true, // When checking previous local data. + false, // Still fail after retry. + true, // When checking previous local data again. + ][i++]; + }, + }; + // Create an extra record. It will have a valid signature locally + // thanks to the verifier mock. + await client.db.importChanges( + { + signature: { x5u, signature: "aa" }, + }, + 4000, + [ + { + id: "extraId", + last_modified: 42, + }, + ] + ); + + equal((await client.get()).length, 1); + + // Now sync, but importing changes will have failing signature, + // and so will retry (see `sigResults`). + await Assert.rejects( + client.maybeSync(6000), + RemoteSettingsClient.InvalidSignatureError, + "Sync failed as expected (bad signature after retry)" + ); + equal(i, 4, "sync has retried as expected"); + + // Make sure that we retried on a blank DB. The extra record should + // have been deleted when we validated the signature the second time. + // Since local data was tampered, it was cleared. + ok(/extraId/.test(sigCalls[0]), "extra record when importing changes"); + ok(/extraId/.test(sigCalls[1]), "extra record when checking local"); + ok(!/extraId/.test(sigCalls[2]), "db was flushed before retry"); + ok(/extraId/.test(sigCalls[3]), "when checking local after retry"); +}); diff --git a/services/settings/test/unit/test_remote_settings_signatures/collection_signing_ee.pem.certspec b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_ee.pem.certspec new file mode 100644 index 0000000000..866c357c50 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_ee.pem.certspec @@ -0,0 +1,5 @@ +issuer:collection-signer-int-CA +subject:collection-signer-ee-int-CA +subjectKey:secp384r1 +extension:extKeyUsage:codeSigning +extension:subjectAlternativeName:onecrl.content-signature.mozilla.org diff --git a/services/settings/test/unit/test_remote_settings_signatures/collection_signing_int.pem.certspec b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_int.pem.certspec new file mode 100644 index 0000000000..8ca4815fa5 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_int.pem.certspec @@ -0,0 +1,4 @@ +issuer:collection-signer-ca +subject:collection-signer-int-CA +extension:basicConstraints:cA, +extension:extKeyUsage:codeSigning diff --git a/services/settings/test/unit/test_remote_settings_signatures/collection_signing_root.pem.certspec b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_root.pem.certspec new file mode 100644 index 0000000000..11bd68768b --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_signatures/collection_signing_root.pem.certspec @@ -0,0 +1,4 @@ +issuer:collection-signer-ca +subject:collection-signer-ca +extension:basicConstraints:cA, +extension:extKeyUsage:codeSigning diff --git a/services/settings/test/unit/test_remote_settings_signatures/moz.build b/services/settings/test/unit/test_remote_settings_signatures/moz.build new file mode 100644 index 0000000000..edb8da08c9 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_signatures/moz.build @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +test_certificates = ( + "collection_signing_root.pem", + "collection_signing_int.pem", + "collection_signing_ee.pem", +) + +for test_certificate in test_certificates: + GeneratedTestCertificate(test_certificate) diff --git a/services/settings/test/unit/test_remote_settings_worker.js b/services/settings/test/unit/test_remote_settings_worker.js new file mode 100644 index 0000000000..755015dab3 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_worker.js @@ -0,0 +1,143 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { TestUtils } = ChromeUtils.import( + "resource://testing-common/TestUtils.jsm" +); + +const { RemoteSettingsWorker } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsWorker.jsm" +); +const { RemoteSettingsClient } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsClient.jsm" +); +const { Database } = ChromeUtils.import( + "resource://services-settings/Database.jsm" +); + +XPCOMUtils.defineLazyGlobalGetters(this, ["indexedDB"]); + +const IS_ANDROID = AppConstants.platform == "android"; + +add_task(async function test_canonicaljson() { + const records = [ + { id: "1", title: "title 1" }, + { id: "2", title: "title 2" }, + ]; + const timestamp = 42; + + const serialized = await RemoteSettingsWorker.canonicalStringify( + records, + timestamp + ); + + Assert.equal( + serialized, + '{"data":[{"id":"1","title":"title 1"},{"id":"2","title":"title 2"}],"last_modified":"42"}' + ); +}); + +add_task(async function test_import_json_dump_into_idb() { + if (IS_ANDROID) { + // Skip test: we don't ship remote settings dumps on Android (see package-manifest). + return; + } + const client = new RemoteSettingsClient("language-dictionaries", { + bucketNamePref: "services.settings.default_bucket", + }); + const before = await client.get({ syncIfEmpty: false }); + Assert.equal(before.length, 0); + + await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries"); + + const after = await client.get({ syncIfEmpty: false }); + Assert.ok(after.length > 0); + let lastModifiedStamp = await client.getLastModified(); + + Assert.equal( + lastModifiedStamp, + Math.max(...after.map(record => record.last_modified)), + "Should have correct last modified timestamp" + ); + + // Force a DB close for shutdown so we can delete the DB later. + Database._shutdownHandler(); +}); + +add_task(async function test_throws_error_if_worker_fails() { + let error; + try { + await RemoteSettingsWorker.canonicalStringify(null, 42); + } catch (e) { + error = e; + } + Assert.equal(error.message.endsWith("records is null"), true); +}); + +add_task(async function test_throws_error_if_worker_fails_async() { + if (IS_ANDROID) { + // Skip test: we don't ship dump, so importJSONDump() is no-op. + return; + } + // Delete the Remote Settings database, and try to import a dump. + // This is not supported, and the error thrown asynchronously in the worker + // should be reported to the caller. + await new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase("remote-settings"); + request.onsuccess = event => resolve(); + request.onblocked = event => reject(new Error("Cannot delete DB")); + request.onerror = event => reject(event.target.error); + }); + let error; + try { + await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries"); + } catch (e) { + error = e; + } + Assert.ok(/IndexedDB: Error accessing remote-settings/.test(error.message)); +}); + +add_task(async function test_throws_error_if_worker_crashes() { + // This simulates a crash at the worker level (not within a promise). + let error; + try { + await RemoteSettingsWorker._execute("unknown_method"); + } catch (e) { + error = e; + } + Assert.equal(error.message, "TypeError: Agent[method] is not a function"); +}); + +add_task(async function test_stops_worker_after_timeout() { + // Change the idle time. + Services.prefs.setIntPref( + "services.settings.worker_idle_max_milliseconds", + 1 + ); + // Run a task: + let serialized = await RemoteSettingsWorker.canonicalStringify([], 42); + Assert.equal(serialized, '{"data":[],"last_modified":"42"}', "API works."); + // Check that the worker gets stopped now the task is done: + await TestUtils.waitForCondition(() => !RemoteSettingsWorker.worker); + // Ensure the worker stays alive for 10 minutes instead: + Services.prefs.setIntPref( + "services.settings.worker_idle_max_milliseconds", + 600000 + ); + // Run another task: + serialized = await RemoteSettingsWorker.canonicalStringify([], 42); + Assert.equal( + serialized, + '{"data":[],"last_modified":"42"}', + "API still works." + ); + Assert.ok(RemoteSettingsWorker.worker, "Worker should stay alive a bit."); + + // Clear the pref. + Services.prefs.clearUserPref( + "services.settings.worker_idle_max_milliseconds" + ); +}); diff --git a/services/settings/test/unit/test_shutdown_handling.js b/services/settings/test/unit/test_shutdown_handling.js new file mode 100644 index 0000000000..5134d5665a --- /dev/null +++ b/services/settings/test/unit/test_shutdown_handling.js @@ -0,0 +1,143 @@ +/* Any copyright is dedicated to the Public Domain. +http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +const { TestUtils } = ChromeUtils.import( + "resource://testing-common/TestUtils.jsm" +); + +const { Database } = ChromeUtils.import( + "resource://services-settings/Database.jsm" +); +const { RemoteSettingsWorker } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsWorker.jsm" +); +const { RemoteSettingsClient } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsClient.jsm" +); + +add_task(async function test_shutdown_abort_after_start() { + // Start a forever transaction: + let counter = 0; + let transactionStarted; + let startedPromise = new Promise(r => { + transactionStarted = r; + }); + let promise = Database._executeIDB( + "records", + store => { + // Signal we've started. + transactionStarted(); + function makeRequest() { + if (++counter > 1000) { + Assert.ok( + false, + "We ran 1000 requests and didn't get aborted, what?" + ); + return; + } + dump("Making request " + counter + "\n"); + const request = store + .index("cid") + .openCursor(IDBKeyRange.only("foopydoo/foo")); + request.onsuccess = event => { + makeRequest(); + }; + } + makeRequest(); + }, + { mode: "readonly" } + ); + + // Wait for the transaction to start. + await startedPromise; + + Database._shutdownHandler(); // should abort the readonly transaction. + + let rejection; + await promise.catch(e => { + rejection = e; + }); + ok(rejection, "Promise should have rejected."); + + // Now clear the shutdown flag and rejection error: + Database._cancelShutdown(); + rejection = null; +}); + +add_task(async function test_shutdown_immediate_abort() { + // Now abort directly from the successful request. + let promise = Database._executeIDB( + "records", + store => { + let request = store + .index("cid") + .openCursor(IDBKeyRange.only("foopydoo/foo")); + request.onsuccess = event => { + // Abort immediately. + Database._shutdownHandler(); + request = store + .index("cid") + .openCursor(IDBKeyRange.only("foopydoo/foo")); + Assert.ok(false, "IndexedDB allowed opening a cursor after aborting?!"); + }; + }, + { mode: "readonly" } + ); + + let rejection; + // Wait for the abort + await promise.catch(e => { + rejection = e; + }); + ok(rejection, "Directly aborted promise should also have rejected."); + // Now clear the shutdown flag and rejection error: + Database._cancelShutdown(); +}); + +add_task(async function test_shutdown_worker() { + let client = new RemoteSettingsClient("language-dictionaries", { + bucketNamePref: "services.settings.default_bucket", + }); + const before = await client.get({ syncIfEmpty: false }); + Assert.equal(before.length, 0); + + let records = [{}]; + let importPromise = RemoteSettingsWorker._execute( + "_test_only_import", + ["main", "language-dictionaries", records], + { mustComplete: true } + ); + let stringifyPromise = RemoteSettingsWorker.canonicalStringify( + [], + [], + Date.now() + ); + // Change the idle time so we shut the worker down even though we can't + // set gShutdown from outside of the worker management code. + Services.prefs.setIntPref( + "services.settings.worker_idle_max_milliseconds", + 1 + ); + RemoteSettingsWorker._abortCancelableRequests(); + await Assert.rejects( + stringifyPromise, + /Shutdown/, + "Should have aborted the stringify request at shutdown." + ); + await Assert.rejects( + importPromise, + /shutting down/, + "Ensure imports get aborted during shutdown" + ); + const after = await client.get({ syncIfEmpty: false }); + Assert.equal(after.length, 0); + await TestUtils.waitForCondition(() => !RemoteSettingsWorker.worker); + Assert.ok( + !RemoteSettingsWorker.worker, + "Worker should have been terminated." + ); +}); diff --git a/services/settings/test/unit/xpcshell.ini b/services/settings/test/unit/xpcshell.ini new file mode 100644 index 0000000000..930d6cd85b --- /dev/null +++ b/services/settings/test/unit/xpcshell.ini @@ -0,0 +1,16 @@ +[DEFAULT] +head = ../../../common/tests/unit/head_global.js ../../../common/tests/unit/head_helpers.js +firefox-appdir = browser +tags = remote-settings +support-files = + test_remote_settings_signatures/** +skip-if = appname == "thunderbird" # Bug 1662758 - these tests don't pass with a different default_bucket. + +[test_attachments_downloader.js] +support-files = test_attachments_downloader/** +[test_remote_settings.js] +[test_remote_settings_poll.js] +[test_remote_settings_worker.js] +[test_remote_settings_jexl_filters.js] +[test_remote_settings_signatures.js] +[test_shutdown_handling.js] -- cgit v1.2.3