diff options
Diffstat (limited to 'toolkit/components/kvstore')
16 files changed, 2445 insertions, 0 deletions
diff --git a/toolkit/components/kvstore/Cargo.toml b/toolkit/components/kvstore/Cargo.toml new file mode 100644 index 0000000000..b4db98f683 --- /dev/null +++ b/toolkit/components/kvstore/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "kvstore" +version = "0.1.0" +authors = ["Myk Melez <myk@mykzilla.org>"] + +[dependencies] +atomic_refcell = "0.1" +crossbeam-utils = "0.6.3" +cstr = "0.2" +lazy_static = "1" +libc = "0.2" +log = "0.4" +moz_task = { path = "../../../xpcom/rust/moz_task" } +nserror = { path = "../../../xpcom/rust/nserror" } +nsstring = { path = "../../../xpcom/rust/nsstring" } +rkv = { version = "0.16.1", default-features = false, features=["no-canonicalize-path"] } +storage_variant = { path = "../../../storage/variant" } +xpcom = { path = "../../../xpcom/rust/xpcom" } +tempfile = "3" +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } + +# Get rid of failure's dependency on backtrace. Eventually +# backtrace will move into Rust core, but we don't need it here. +[dependencies.failure] +version = "0.1" +default_features = false +features = ["derive"] diff --git a/toolkit/components/kvstore/kvstore.jsm b/toolkit/components/kvstore/kvstore.jsm new file mode 100644 index 0000000000..9da4bf5cc1 --- /dev/null +++ b/toolkit/components/kvstore/kvstore.jsm @@ -0,0 +1,211 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const gKeyValueService = Cc["@mozilla.org/key-value-service;1"].getService( + Ci.nsIKeyValueService +); + +const EXPORTED_SYMBOLS = ["KeyValueService"]; + +function promisify(fn, ...args) { + return new Promise((resolve, reject) => { + fn({ resolve, reject }, ...args); + }); +} + +/** + * This module wraps the nsIKeyValue* interfaces in a Promise-based API. + * To use it, import it, then call the KeyValueService.getOrCreate() method + * with a database's path and (optionally) its name: + * + * ``` + * ChromeUtils.import("resource://gre/modules/kvstore.jsm"); + * let database = await KeyValueService.getOrCreate(path, name); + * ``` + * + * See the documentation in nsIKeyValue.idl for more information about the API + * for key/value storage. + */ + +class KeyValueService { + static async getOrCreate(dir, name) { + return new KeyValueDatabase( + await promisify(gKeyValueService.getOrCreate, dir, name) + ); + } +} + +/** + * A class that wraps an nsIKeyValueDatabase in a Promise-based API. + * + * This class isn't exported, so you can't instantiate it directly, but you + * can retrieve an instance of this class via KeyValueService.getOrCreate(): + * + * ``` + * const database = await KeyValueService.getOrCreate(path, name); + * ``` + * + * You can then call its put(), get(), has(), and delete() methods to access + * and manipulate key/value pairs: + * + * ``` + * await database.put("foo", 1); + * await database.get("foo") === 1; // true + * await database.has("foo"); // true + * await database.delete("foo"); + * await database.has("foo"); // false + * ``` + * + * You can also call writeMany() to put/delete multiple key/value pairs: + * + * ``` + * await database.writeMany({ + * key1: "value1", + * key2: "value2", + * key3: "value3", + * key4: null, // delete + * }); + * ``` + * + * And you can call its enumerate() method to retrieve a KeyValueEnumerator, + * which is described below. + */ +class KeyValueDatabase { + constructor(database) { + this.database = database; + } + + put(key, value) { + return promisify(this.database.put, key, value); + } + + /** + * Writes multiple key/value pairs to the database. + * + * Note: + * * Each write could be either put or delete. + * * Given multiple values with the same key, only the last value will be stored. + * * If the same key gets put and deleted for multiple times, the final state + * of that key is subject to the ordering of the put(s) and delete(s). + * + * @param pairs Pairs could be any of following types: + * * An Object, all its properties and the corresponding values will + * be used as key value pairs. A property with null or undefined indicating + * a deletion. + * * An Array or an iterable whose elements are key-value pairs. such as + * [["key1", "value1"], ["key2", "value2"]]. Use a pair with value null + * to delete a key-value pair, e.g. ["delete-key", null]. + * * A Map. A key with null or undefined value indicating a deletion. + * @return A promise that is fulfilled when all the key/value pairs are written + * to the database. + */ + writeMany(pairs) { + if (!pairs) { + throw new Error("writeMany(): unexpected argument."); + } + + let entries; + + if ( + pairs instanceof Map || + pairs instanceof Array || + typeof pairs[Symbol.iterator] === "function" + ) { + try { + // Let Map constructor validate the argument. Note that Map remembers + // the original insertion order of the keys, which satisfies the ordering + // premise of this function. + const map = pairs instanceof Map ? pairs : new Map(pairs); + entries = Array.from(map, ([key, value]) => ({ key, value })); + } catch (error) { + throw new Error("writeMany(): unexpected argument."); + } + } else if (typeof pairs === "object") { + entries = Array.from(Object.entries(pairs), ([key, value]) => ({ + key, + value, + })); + } else { + throw new Error("writeMany(): unexpected argument."); + } + + if (entries.length) { + return promisify(this.database.writeMany, entries); + } + return Promise.resolve(); + } + + has(key) { + return promisify(this.database.has, key); + } + + get(key, defaultValue) { + return promisify(this.database.get, key, defaultValue); + } + + delete(key) { + return promisify(this.database.delete, key); + } + + clear() { + return promisify(this.database.clear); + } + + async enumerate(from_key, to_key) { + return new KeyValueEnumerator( + await promisify(this.database.enumerate, from_key, to_key) + ); + } +} + +/** + * A class that wraps an nsIKeyValueEnumerator in a Promise-based API. + * + * This class isn't exported, so you can't instantiate it directly, but you + * can retrieve an instance of this class by calling enumerate() on an instance + * of KeyValueDatabase: + * + * ``` + * const database = await KeyValueService.getOrCreate(path, name); + * const enumerator = await database.enumerate(); + * ``` + * + * And then iterate pairs via its hasMoreElements() and getNext() methods: + * + * ``` + * while (enumerator.hasMoreElements()) { + * const { key, value } = enumerator.getNext(); + * … + * } + * ``` + * + * Or with a `for...of` statement: + * + * ``` + * for (const { key, value } of enumerator) { + * … + * } + * ``` + */ +class KeyValueEnumerator { + constructor(enumerator) { + this.enumerator = enumerator; + } + + hasMoreElements() { + return this.enumerator.hasMoreElements(); + } + + getNext() { + return this.enumerator.getNext(); + } + + *[Symbol.iterator]() { + while (this.enumerator.hasMoreElements()) { + yield this.enumerator.getNext(); + } + } +} diff --git a/toolkit/components/kvstore/moz.build b/toolkit/components/kvstore/moz.build new file mode 100644 index 0000000000..319c1bdc56 --- /dev/null +++ b/toolkit/components/kvstore/moz.build @@ -0,0 +1,26 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Toolkit", "Storage") + +EXTRA_JS_MODULES += [ + "kvstore.jsm", +] + +XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell/xpcshell.ini"] + +XPIDL_SOURCES += [ + "nsIKeyValue.idl", +] + +XPIDL_MODULE = "kvstore" + +UNIFIED_SOURCES += [ + "nsKeyValueModule.cpp", +] + +FINAL_LIBRARY = "xul" diff --git a/toolkit/components/kvstore/nsIKeyValue.idl b/toolkit/components/kvstore/nsIKeyValue.idl new file mode 100644 index 0000000000..9fd4f68c3c --- /dev/null +++ b/toolkit/components/kvstore/nsIKeyValue.idl @@ -0,0 +1,225 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIVariant.idl" + +interface nsIKeyValueDatabaseCallback; +interface nsIKeyValueEnumeratorCallback; +interface nsIKeyValuePairCallback; +interface nsIKeyValueVariantCallback; +interface nsIKeyValueVoidCallback; +interface nsIKeyValuePair; + +/** + * The nsIKeyValue* interfaces provide a simple, asynchronous API to a key/value + * storage engine. Basic put/get/has/delete operations are supported, as is + * enumeration of key/value pairs and the use of multiple named databases within + * a single storage file. Operations have ACID semantics. + * + * This API does not (yet) support transactions, so it will not be appropriate + * for all use cases. Extension of this API to support transactions is tracked + * by bug 1499238. + * + * The kvstore.jsm module wraps this API in a more idiomatic, Promise-based + * JS API that supports async/await. In most cases, you're better off using + * that API from JS rather than using this one directly. Bug 1512319 tracks + * native support for Promise in Rust-implemented XPCOM methods. + */ + +/** + * The key/value service. Enables retrieval of handles to key/value databases. + */ +[scriptable, uuid(46c893dd-4c14-4de0-b33d-a1be18c6d062)] +interface nsIKeyValueService : nsISupports { + /** + * Get a handle to an existing database or a newly-created one + * at the specified path and with the given name. + * + * The service supports multiple named databases at the same path + * (i.e. within the same storage file), so you can call this method + * multiple times with the same path and different names to retrieve + * multiple databases stored in the same location on disk. + */ + void getOrCreate( + in nsIKeyValueDatabaseCallback callback, + in AUTF8String path, + in AUTF8String name); +}; + +/** + * A key/value database. + * + * All methods are asynchronous and take a callback as their first argument. + * The types of the callbacks vary, but they can all be implemented in JS + * via an object literal with the relevant methods. + */ +[scriptable, uuid(c449398e-174c-425b-8195-da6aa0ccd9a5)] +interface nsIKeyValueDatabase : nsISupports { + /** + * Write the specified key/value pair to the database. + */ + void put( + in nsIKeyValueVoidCallback callback, + in AUTF8String key, + in nsIVariant value); + + /** + * Write multiple key/value pairs to the database. + * + * It supports two types of write: + * * Put a key/value pair into the database. It takes a nsIKeyValuePair + * where its key and value follow the same types as the put() method. + * * Delete a key/value pair from database. It takes a nsIkeyValuePair + * where its value property must be null or undefined. + * + * This features the "all-or-nothing" semantics, i.e. if any error occurs + * during the call, it will rollback the previous writes and terminate the + * call. In addition, writeMany should be more efficient than calling "put" + * or "delete" for every single key/value pair since it does all the writes + * in a single transaction. + * + * Note: + * * If there are multiple values with the same key in the specified + * pairs, only the last value will be stored in the database. + * * Deleting a key that is not in the database will be silently ignored. + * * If the same key gets put and deleted for multiple times, the final + * state of that key is subject to the ordering of the put(s) and delete(s). + */ + void writeMany( + in nsIKeyValueVoidCallback callback, + in Array<nsIKeyValuePair> pairs); + + /** + * Retrieve the value of the specified key from the database. + * + * If the key/value pair doesn't exist in the database, and you specify + * a default value, then the default value will be returned. Otherwise, + * the callback's resolve() method will be called with a variant + * of type VTYPE_EMPTY, which translates to the JS `null` value. + */ + void get( + in nsIKeyValueVariantCallback callback, + in AUTF8String key, + [optional] in nsIVariant defaultValue); + + /** + * Determine whether or not the key exists in the database. + */ + void has( + in nsIKeyValueVariantCallback callback, + in AUTF8String key); + + /** + * Remove the key/value pair with the given key from the database. + * + * If the given key doesn't exist in the database, this operation doesn't + * fail; or rather, it fails silently, calling the resolve() method + * of its callback rather than reject(). If you want to know whether + * or not a key exists when deleting it, call the has() method first. + */ + void delete( + in nsIKeyValueVoidCallback callback, + in AUTF8String key); + + /** + * Clear all the key/value pairs from the database. + */ + void clear(in nsIKeyValueVoidCallback callback); + + /** + * Enumerate key/value pairs, starting with the first key equal to + * or greater than the "from" key (inclusive) and ending with the last key + * less than the "to" key (exclusive) sorted lexicographically. + * + * If either key is omitted, the range extends to the first and/or last key + * in the database. + */ + void enumerate( + in nsIKeyValueEnumeratorCallback callback, + [optional] in AUTF8String fromKey, + [optional] in AUTF8String toKey); +}; + +/** + * A key/value pair. Returned by nsIKeyValueEnumerator.getNext(). + */ +[scriptable, uuid(bc37b06a-23b5-4b32-8281-4b8479601c7e)] +interface nsIKeyValuePair : nsISupports { + readonly attribute AUTF8String key; + readonly attribute nsIVariant value; +}; + +/** + * An enumerator of key/value pairs. Although its methods are similar + * to those of nsISimpleEnumerator, this interface's getNext() method returns + * an nsIKeyValuePair rather than an nsISupports, so consumers don't need + * to QI it to that interface; but this interface doesn't implement the JS + * iteration protocol (because the Rust-XPCOM bindings don't yet support it), + * which is another reason why you should use the kvstore.jsm module from JS + * instead of accessing this API directly. + */ +[scriptable, uuid(b9ba7116-b7ff-4717-9a28-a08e6879b199)] +interface nsIKeyValueEnumerator : nsISupports { + bool hasMoreElements(); + nsIKeyValuePair getNext(); +}; + +/** + * A callback for the nsIKeyValueService.getOrCreate() method. + * + * The result is an nsIKeyValueDatabase. + */ +[scriptable, uuid(2becc1f8-2d80-4b63-92a8-24ee8f79ee45)] +interface nsIKeyValueDatabaseCallback : nsISupports { + void resolve(in nsIKeyValueDatabase database); + void reject(in AUTF8String message); +}; + +/** + * A callback for the nsIKeyValueDatabase.enumerate() method. + * + * The result is an nsIKeyValueEnumerator. + */ +[scriptable, uuid(b7ea2183-880b-4424-ab24-5aa1555b775d)] +interface nsIKeyValueEnumeratorCallback : nsISupports { + void resolve(in nsIKeyValueEnumerator enumerator); + void reject(in AUTF8String message); +}; + +/** + * A callback for the nsIKeyValueEnumerator.getNext() method. + * + * The result is the next key/value pair, expressed as separate key and value + * parameters. + */ +[scriptable, uuid(50f65485-ec1e-4307-812b-b8a15e1f382e)] +interface nsIKeyValuePairCallback : nsISupports { + void resolve(in nsIKeyValuePair pair); + void reject(in AUTF8String message); +}; + +/** + * A callback for the nsIKeyValueDatabase.has() and .get() methods. + * + * The result is an nsIVariant, which is always a boolean for the has() method + * and can be any supported data type for the get() method. + */ +[scriptable, uuid(174ebfa1-74ea-42a7-aa90-85bbaf1da4bf)] +interface nsIKeyValueVariantCallback : nsISupports { + void resolve(in nsIVariant result); + void reject(in AUTF8String message); +}; + +/** + * A callback for the nsIKeyValueDatabase.put() and .delete() methods. + * + * There is no result, but the resolve() method is still called when those + * async operations complete, to notify consumers of completion. + */ +[scriptable, uuid(0c17497a-ccf8-451a-838d-9dfa7f846379)] +interface nsIKeyValueVoidCallback : nsISupports { + void resolve(); + void reject(in AUTF8String message); +}; diff --git a/toolkit/components/kvstore/nsKeyValueModule.cpp b/toolkit/components/kvstore/nsKeyValueModule.cpp new file mode 100644 index 0000000000..881baf852e --- /dev/null +++ b/toolkit/components/kvstore/nsKeyValueModule.cpp @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ModuleUtils.h" +#include "nsIClassInfoImpl.h" +#include "nsToolkitCompsCID.h" + +extern "C" { +// Implemented in Rust. +nsresult nsKeyValueServiceConstructor(nsISupports* aOuter, REFNSIID aIID, + void** aResult); +} // extern "C" + +NS_DEFINE_NAMED_CID(NS_KEY_VALUE_SERVICE_CID); + +const mozilla::Module::CIDEntry kKeyValueCIDs[] = { + {&kNS_KEY_VALUE_SERVICE_CID, false, nullptr, nsKeyValueServiceConstructor}, + {nullptr}}; + +const mozilla::Module::ContractIDEntry kKeyValueContracts[] = { + {NS_KEY_VALUE_SERVICE_CONTRACTID, &kNS_KEY_VALUE_SERVICE_CID}, {nullptr}}; + +extern const mozilla::Module kKeyValueModule = { + mozilla::Module::kVersion, kKeyValueCIDs, kKeyValueContracts, nullptr}; diff --git a/toolkit/components/kvstore/src/error.rs b/toolkit/components/kvstore/src/error.rs new file mode 100644 index 0000000000..012237356a --- /dev/null +++ b/toolkit/components/kvstore/src/error.rs @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use nserror::{ + nsresult, NS_ERROR_FAILURE, NS_ERROR_NOT_IMPLEMENTED, NS_ERROR_NO_INTERFACE, + NS_ERROR_NULL_POINTER, NS_ERROR_UNEXPECTED, +}; +use nsstring::nsCString; +use rkv::{MigrateError, StoreError}; +use std::{io::Error as IoError, str::Utf8Error, string::FromUtf16Error, sync::PoisonError}; + +#[derive(Debug, Fail)] +pub enum KeyValueError { + #[fail(display = "error converting string: {:?}", _0)] + ConvertBytes(Utf8Error), + + #[fail(display = "error converting string: {:?}", _0)] + ConvertString(FromUtf16Error), + + #[fail(display = "I/O error: {:?}", _0)] + IoError(IoError), + + #[fail(display = "migrate error: {:?}", _0)] + MigrateError(MigrateError), + + #[fail(display = "no interface '{}'", _0)] + NoInterface(&'static str), + + // NB: We can avoid storing the nsCString error description + // once nsresult is a real type with a Display implementation + // per https://bugzilla.mozilla.org/show_bug.cgi?id=1513350. + #[fail(display = "error result {}", _0)] + Nsresult(nsCString, nsresult), + + #[fail(display = "arg is null")] + NullPointer, + + #[fail(display = "poison error getting read/write lock")] + PoisonError, + + #[fail(display = "error reading key/value pair")] + Read, + + #[fail(display = "store error: {:?}", _0)] + StoreError(StoreError), + + #[fail(display = "unsupported owned value type")] + UnsupportedOwned, + + #[fail(display = "unexpected value")] + UnexpectedValue, + + #[fail(display = "unsupported variant type: {}", _0)] + UnsupportedVariant(u16), +} + +impl From<nsresult> for KeyValueError { + fn from(result: nsresult) -> KeyValueError { + KeyValueError::Nsresult(result.error_name(), result) + } +} + +impl From<KeyValueError> for nsresult { + fn from(err: KeyValueError) -> nsresult { + match err { + KeyValueError::ConvertBytes(_) => NS_ERROR_FAILURE, + KeyValueError::ConvertString(_) => NS_ERROR_FAILURE, + KeyValueError::IoError(_) => NS_ERROR_FAILURE, + KeyValueError::NoInterface(_) => NS_ERROR_NO_INTERFACE, + KeyValueError::Nsresult(_, result) => result, + KeyValueError::NullPointer => NS_ERROR_NULL_POINTER, + KeyValueError::PoisonError => NS_ERROR_UNEXPECTED, + KeyValueError::Read => NS_ERROR_FAILURE, + KeyValueError::StoreError(_) => NS_ERROR_FAILURE, + KeyValueError::MigrateError(_) => NS_ERROR_FAILURE, + KeyValueError::UnsupportedOwned => NS_ERROR_NOT_IMPLEMENTED, + KeyValueError::UnexpectedValue => NS_ERROR_UNEXPECTED, + KeyValueError::UnsupportedVariant(_) => NS_ERROR_NOT_IMPLEMENTED, + } + } +} + +impl From<IoError> for KeyValueError { + fn from(err: IoError) -> KeyValueError { + KeyValueError::IoError(err) + } +} + +impl From<StoreError> for KeyValueError { + fn from(err: StoreError) -> KeyValueError { + KeyValueError::StoreError(err) + } +} + +impl From<MigrateError> for KeyValueError { + fn from(err: MigrateError) -> KeyValueError { + KeyValueError::MigrateError(err) + } +} + +impl From<Utf8Error> for KeyValueError { + fn from(err: Utf8Error) -> KeyValueError { + KeyValueError::ConvertBytes(err) + } +} + +impl From<FromUtf16Error> for KeyValueError { + fn from(err: FromUtf16Error) -> KeyValueError { + KeyValueError::ConvertString(err) + } +} + +impl<T> From<PoisonError<T>> for KeyValueError { + fn from(_err: PoisonError<T>) -> KeyValueError { + KeyValueError::PoisonError + } +} diff --git a/toolkit/components/kvstore/src/lib.rs b/toolkit/components/kvstore/src/lib.rs new file mode 100644 index 0000000000..3723d9e910 --- /dev/null +++ b/toolkit/components/kvstore/src/lib.rs @@ -0,0 +1,383 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +extern crate atomic_refcell; +extern crate crossbeam_utils; +#[macro_use] +extern crate cstr; +#[macro_use] +extern crate failure; +extern crate libc; +extern crate log; +extern crate moz_task; +extern crate nserror; +extern crate nsstring; +extern crate rkv; +extern crate storage_variant; +extern crate tempfile; +extern crate thin_vec; +extern crate xpcom; + +mod error; +mod owned_value; +mod task; + +use atomic_refcell::AtomicRefCell; +use error::KeyValueError; +use libc::c_void; +use moz_task::{ + create_background_task_queue, dispatch_background_task_with_options, DispatchOptions, + TaskRunnable, +}; +use nserror::{nsresult, NS_ERROR_FAILURE, NS_ERROR_NO_AGGREGATION, NS_OK}; +use nsstring::{nsACString, nsCString}; +use owned_value::{owned_to_variant, variant_to_owned}; +use rkv::backend::{SafeModeDatabase, SafeModeEnvironment}; +use rkv::OwnedValue; +use std::{ + ptr, + sync::{Arc, RwLock}, + vec::IntoIter, +}; +use task::{ + ClearTask, DeleteTask, EnumerateTask, GetOrCreateTask, GetTask, HasTask, PutTask, WriteManyTask, +}; +use thin_vec::ThinVec; +use xpcom::{ + getter_addrefs, + interfaces::{ + nsIKeyValueDatabaseCallback, nsIKeyValueEnumeratorCallback, nsIKeyValuePair, + nsIKeyValueVariantCallback, nsIKeyValueVoidCallback, nsISerialEventTarget, nsISupports, + nsIVariant, + }, + nsIID, xpcom, xpcom_method, RefPtr, +}; + +type Rkv = rkv::Rkv<SafeModeEnvironment>; +type SingleStore = rkv::SingleStore<SafeModeDatabase>; +type KeyValuePairResult = Result<(String, OwnedValue), KeyValueError>; + +#[no_mangle] +pub unsafe extern "C" fn nsKeyValueServiceConstructor( + outer: *const nsISupports, + iid: &nsIID, + result: *mut *mut c_void, +) -> nsresult { + *result = ptr::null_mut(); + + if !outer.is_null() { + return NS_ERROR_NO_AGGREGATION; + } + + let service = KeyValueService::new(); + service.QueryInterface(iid, result) +} + +// For each public XPCOM method in the nsIKeyValue* interfaces, we implement +// a pair of Rust methods: +// +// 1. a method named after the XPCOM (as modified by the XPIDL parser, i.e. +// by capitalization of its initial letter) that returns an nsresult; +// +// 2. a method with a Rust-y name that returns a Result<(), KeyValueError>. +// +// XPCOM calls the first method, which is only responsible for calling +// the second one and converting its Result to an nsresult (logging errors +// in the process). The second method is responsible for doing the work. +// +// For example, given an XPCOM method FooBar, we implement a method FooBar +// that calls a method foo_bar. foo_bar returns a Result<(), KeyValueError>, +// and FooBar converts that to an nsresult. +// +// This design allows us to use Rust idioms like the question mark operator +// to simplify the implementation in the second method while returning XPCOM- +// compatible nsresult values to XPCOM callers. +// +// The XPCOM methods are implemented using the xpcom_method! declarative macro +// from the xpcom crate. + +#[derive(xpcom)] +#[xpimplements(nsIKeyValueService)] +#[refcnt = "atomic"] +pub struct InitKeyValueService {} + +impl KeyValueService { + fn new() -> RefPtr<KeyValueService> { + KeyValueService::allocate(InitKeyValueService {}) + } + + xpcom_method!( + get_or_create => GetOrCreate( + callback: *const nsIKeyValueDatabaseCallback, + path: *const nsACString, + name: *const nsACString + ) + ); + + fn get_or_create( + &self, + callback: &nsIKeyValueDatabaseCallback, + path: &nsACString, + name: &nsACString, + ) -> Result<(), nsresult> { + let task = Box::new(GetOrCreateTask::new( + RefPtr::new(callback), + nsCString::from(path), + nsCString::from(name), + )); + + dispatch_background_task_with_options( + RefPtr::new(TaskRunnable::new("KVService::GetOrCreate", task)?.coerce()), + DispatchOptions::default().may_block(true), + ) + } +} + +#[derive(xpcom)] +#[xpimplements(nsIKeyValueDatabase)] +#[refcnt = "atomic"] +pub struct InitKeyValueDatabase { + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + queue: RefPtr<nsISerialEventTarget>, +} + +impl KeyValueDatabase { + fn new( + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + ) -> Result<RefPtr<KeyValueDatabase>, KeyValueError> { + let queue = create_background_task_queue(cstr!("KeyValueDatabase"))?; + Ok(KeyValueDatabase::allocate(InitKeyValueDatabase { + rkv, + store, + queue, + })) + } + + xpcom_method!( + put => Put( + callback: *const nsIKeyValueVoidCallback, + key: *const nsACString, + value: *const nsIVariant + ) + ); + + fn put( + &self, + callback: &nsIKeyValueVoidCallback, + key: &nsACString, + value: &nsIVariant, + ) -> Result<(), nsresult> { + let value = variant_to_owned(value)?.ok_or(KeyValueError::UnexpectedValue)?; + + let task = Box::new(PutTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + nsCString::from(key), + value, + )); + + TaskRunnable::dispatch(TaskRunnable::new("KVDatabase::Put", task)?, &self.queue) + } + + xpcom_method!( + write_many => WriteMany( + callback: *const nsIKeyValueVoidCallback, + pairs: *const ThinVec<RefPtr<nsIKeyValuePair>> + ) + ); + + fn write_many( + &self, + callback: &nsIKeyValueVoidCallback, + pairs: &ThinVec<RefPtr<nsIKeyValuePair>>, + ) -> Result<(), nsresult> { + let mut entries = Vec::with_capacity(pairs.len()); + + for pair in pairs { + let mut key = nsCString::new(); + unsafe { pair.GetKey(&mut *key) }.to_result()?; + if key.is_empty() { + return Err(nsresult::from(KeyValueError::UnexpectedValue)); + } + + let val: RefPtr<nsIVariant> = getter_addrefs(|p| unsafe { pair.GetValue(p) })?; + let value = variant_to_owned(&val)?; + entries.push((key, value)); + } + + let task = Box::new(WriteManyTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + entries, + )); + + TaskRunnable::dispatch( + TaskRunnable::new("KVDatabase::WriteMany", task)?, + &self.queue, + ) + } + + xpcom_method!( + get => Get( + callback: *const nsIKeyValueVariantCallback, + key: *const nsACString, + default_value: *const nsIVariant + ) + ); + + fn get( + &self, + callback: &nsIKeyValueVariantCallback, + key: &nsACString, + default_value: &nsIVariant, + ) -> Result<(), nsresult> { + let task = Box::new(GetTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + nsCString::from(key), + variant_to_owned(default_value)?, + )); + + TaskRunnable::dispatch(TaskRunnable::new("KVDatabase::Get", task)?, &self.queue) + } + + xpcom_method!( + has => Has(callback: *const nsIKeyValueVariantCallback, key: *const nsACString) + ); + + fn has(&self, callback: &nsIKeyValueVariantCallback, key: &nsACString) -> Result<(), nsresult> { + let task = Box::new(HasTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + nsCString::from(key), + )); + + TaskRunnable::dispatch(TaskRunnable::new("KVDatabase::Has", task)?, &self.queue) + } + + xpcom_method!( + delete => Delete(callback: *const nsIKeyValueVoidCallback, key: *const nsACString) + ); + + fn delete(&self, callback: &nsIKeyValueVoidCallback, key: &nsACString) -> Result<(), nsresult> { + let task = Box::new(DeleteTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + nsCString::from(key), + )); + + TaskRunnable::dispatch(TaskRunnable::new("KVDatabase::Delete", task)?, &self.queue) + } + + xpcom_method!( + clear => Clear(callback: *const nsIKeyValueVoidCallback) + ); + + fn clear(&self, callback: &nsIKeyValueVoidCallback) -> Result<(), nsresult> { + let task = Box::new(ClearTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + )); + + TaskRunnable::dispatch(TaskRunnable::new("KVDatabase::Clear", task)?, &self.queue) + } + + xpcom_method!( + enumerate => Enumerate( + callback: *const nsIKeyValueEnumeratorCallback, + from_key: *const nsACString, + to_key: *const nsACString + ) + ); + + fn enumerate( + &self, + callback: &nsIKeyValueEnumeratorCallback, + from_key: &nsACString, + to_key: &nsACString, + ) -> Result<(), nsresult> { + let task = Box::new(EnumerateTask::new( + RefPtr::new(callback), + Arc::clone(&self.rkv), + self.store, + nsCString::from(from_key), + nsCString::from(to_key), + )); + + TaskRunnable::dispatch( + TaskRunnable::new("KVDatabase::Enumerate", task)?, + &self.queue, + ) + } +} + +#[derive(xpcom)] +#[xpimplements(nsIKeyValueEnumerator)] +#[refcnt = "atomic"] +pub struct InitKeyValueEnumerator { + iter: AtomicRefCell<IntoIter<KeyValuePairResult>>, +} + +impl KeyValueEnumerator { + fn new(pairs: Vec<KeyValuePairResult>) -> RefPtr<KeyValueEnumerator> { + KeyValueEnumerator::allocate(InitKeyValueEnumerator { + iter: AtomicRefCell::new(pairs.into_iter()), + }) + } + + xpcom_method!(has_more_elements => HasMoreElements() -> bool); + + fn has_more_elements(&self) -> Result<bool, KeyValueError> { + Ok(!self.iter.borrow().as_slice().is_empty()) + } + + xpcom_method!(get_next => GetNext() -> *const nsIKeyValuePair); + + fn get_next(&self) -> Result<RefPtr<nsIKeyValuePair>, KeyValueError> { + let mut iter = self.iter.borrow_mut(); + let (key, value) = iter + .next() + .ok_or_else(|| KeyValueError::from(NS_ERROR_FAILURE))??; + + // We fail on retrieval of the key/value pair if the key isn't valid + // UTF-*, if the value is unexpected, or if we encountered a store error + // while retrieving the pair. + Ok(RefPtr::new( + KeyValuePair::new(key, value).coerce::<nsIKeyValuePair>(), + )) + } +} + +#[derive(xpcom)] +#[xpimplements(nsIKeyValuePair)] +#[refcnt = "atomic"] +pub struct InitKeyValuePair { + key: String, + value: OwnedValue, +} + +impl KeyValuePair { + fn new(key: String, value: OwnedValue) -> RefPtr<KeyValuePair> { + KeyValuePair::allocate(InitKeyValuePair { key, value }) + } + + xpcom_method!(get_key => GetKey() -> nsACString); + xpcom_method!(get_value => GetValue() -> *const nsIVariant); + + fn get_key(&self) -> Result<nsCString, KeyValueError> { + Ok(nsCString::from(&self.key)) + } + + fn get_value(&self) -> Result<RefPtr<nsIVariant>, KeyValueError> { + Ok(owned_to_variant(self.value.clone())?) + } +} diff --git a/toolkit/components/kvstore/src/owned_value.rs b/toolkit/components/kvstore/src/owned_value.rs new file mode 100644 index 0000000000..a22abcb8da --- /dev/null +++ b/toolkit/components/kvstore/src/owned_value.rs @@ -0,0 +1,72 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use error::KeyValueError; +use nsstring::{nsCString, nsString}; +use rkv::OwnedValue; +use std::convert::TryInto; +use storage_variant::{DataType, NsIVariantExt, VariantType}; +use xpcom::{interfaces::nsIVariant, RefPtr}; + +pub fn owned_to_variant(owned: OwnedValue) -> Result<RefPtr<nsIVariant>, KeyValueError> { + match owned { + OwnedValue::Bool(val) => Ok(val.into_variant()), + OwnedValue::I64(val) => Ok(val.into_variant()), + OwnedValue::F64(val) => Ok(val.into_variant()), + OwnedValue::Str(ref val) => Ok(nsString::from(val).into_variant()), + + // kvstore doesn't (yet?) support these types of OwnedValue, + // and we should never encounter them, but we need to exhaust + // all possible variants of the OwnedValue enum. + OwnedValue::Instant(_) => Err(KeyValueError::UnsupportedOwned), + OwnedValue::Json(_) => Err(KeyValueError::UnsupportedOwned), + OwnedValue::U64(_) => Err(KeyValueError::UnsupportedOwned), + OwnedValue::Uuid(_) => Err(KeyValueError::UnsupportedOwned), + OwnedValue::Blob(_) => Err(KeyValueError::UnsupportedOwned), + } +} + +pub fn variant_to_owned(variant: &nsIVariant) -> Result<Option<OwnedValue>, KeyValueError> { + let data_type = variant.get_data_type(); + + match data_type.try_into() { + Ok(DataType::Int32) => { + let mut val: i32 = 0; + unsafe { variant.GetAsInt32(&mut val) }.to_result()?; + Ok(Some(OwnedValue::I64(val.into()))) + } + Ok(DataType::Int64) => { + let mut val: i64 = 0; + unsafe { variant.GetAsInt64(&mut val) }.to_result()?; + Ok(Some(OwnedValue::I64(val))) + } + Ok(DataType::Double) => { + let mut val: f64 = 0.0; + unsafe { variant.GetAsDouble(&mut val) }.to_result()?; + Ok(Some(OwnedValue::F64(val))) + } + Ok(DataType::CString) + | Ok(DataType::CharStr) + | Ok(DataType::StringSizeIs) + | Ok(DataType::Utf8String) => { + let mut val: nsCString = nsCString::new(); + unsafe { variant.GetAsAUTF8String(&mut *val) }.to_result()?; + let s = std::str::from_utf8(&*val)?; + Ok(Some(OwnedValue::Str(s.into()))) + } + Ok(DataType::AString) | Ok(DataType::WCharStr) | Ok(DataType::WStringSizeIs) => { + let mut val: nsString = nsString::new(); + unsafe { variant.GetAsAString(&mut *val) }.to_result()?; + let str = String::from_utf16(&val)?; + Ok(Some(OwnedValue::Str(str))) + } + Ok(DataType::Bool) => { + let mut val: bool = false; + unsafe { variant.GetAsBool(&mut val) }.to_result()?; + Ok(Some(OwnedValue::Bool(val))) + } + Ok(DataType::Void) | Ok(DataType::EmptyArray) | Ok(DataType::Empty) => Ok(None), + Err(_) => Err(KeyValueError::UnsupportedVariant(data_type)), + } +} diff --git a/toolkit/components/kvstore/src/task.rs b/toolkit/components/kvstore/src/task.rs new file mode 100644 index 0000000000..4d5de31e3d --- /dev/null +++ b/toolkit/components/kvstore/src/task.rs @@ -0,0 +1,696 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +extern crate xpcom; + +use crossbeam_utils::atomic::AtomicCell; +use error::KeyValueError; +use moz_task::Task; +use nserror::{nsresult, NS_ERROR_FAILURE}; +use nsstring::nsCString; +use owned_value::owned_to_variant; +use rkv::backend::{BackendInfo, SafeMode, SafeModeDatabase, SafeModeEnvironment}; +use rkv::{Migrator, OwnedValue, StoreError, StoreOptions, Value}; +use std::{ + path::Path, + str, + sync::{Arc, RwLock}, +}; +use storage_variant::VariantType; +use xpcom::{ + interfaces::{ + nsIKeyValueDatabaseCallback, nsIKeyValueEnumeratorCallback, nsIKeyValueVariantCallback, + nsIKeyValueVoidCallback, nsIVariant, + }, + RefPtr, ThreadBoundRefPtr, +}; +use KeyValueDatabase; +use KeyValueEnumerator; +use KeyValuePairResult; + +type Manager = rkv::Manager<SafeModeEnvironment>; +type Rkv = rkv::Rkv<SafeModeEnvironment>; +type SingleStore = rkv::SingleStore<SafeModeDatabase>; + +/// A macro to generate a done() implementation for a Task. +/// Takes one argument that specifies the type of the Task's callback function: +/// value: a callback function that takes a value +/// void: the callback function doesn't take a value +/// +/// The "value" variant calls self.convert() to convert a successful result +/// into the value to pass to the callback function. So if you generate done() +/// for a callback that takes a value, ensure you also implement convert()! +macro_rules! task_done { + (value) => { + fn done(&self) -> Result<(), nsresult> { + // If TaskRunnable.run() calls Task.done() to return a result + // on the main thread before TaskRunnable.run() returns on the database + // thread, then the Task will get dropped on the database thread. + // + // But the callback is an nsXPCWrappedJS that isn't safe to release + // on the database thread. So we move it out of the Task here to ensure + // it gets released on the main thread. + let threadbound = self.callback.swap(None).ok_or(NS_ERROR_FAILURE)?; + let callback = threadbound.get_ref().ok_or(NS_ERROR_FAILURE)?; + + match self.result.swap(None) { + Some(Ok(value)) => unsafe { callback.Resolve(self.convert(value)?.coerce()) }, + Some(Err(err)) => unsafe { callback.Reject(&*nsCString::from(err.to_string())) }, + None => unsafe { callback.Reject(&*nsCString::from("unexpected")) }, + } + .to_result() + } + }; + + (void) => { + fn done(&self) -> Result<(), nsresult> { + // If TaskRunnable.run() calls Task.done() to return a result + // on the main thread before TaskRunnable.run() returns on the database + // thread, then the Task will get dropped on the database thread. + // + // But the callback is an nsXPCWrappedJS that isn't safe to release + // on the database thread. So we move it out of the Task here to ensure + // it gets released on the main thread. + let threadbound = self.callback.swap(None).ok_or(NS_ERROR_FAILURE)?; + let callback = threadbound.get_ref().ok_or(NS_ERROR_FAILURE)?; + + match self.result.swap(None) { + Some(Ok(())) => unsafe { callback.Resolve() }, + Some(Err(err)) => unsafe { callback.Reject(&*nsCString::from(err.to_string())) }, + None => unsafe { callback.Reject(&*nsCString::from("unexpected")) }, + } + .to_result() + } + }; +} + +/// A tuple comprising an Arc<RwLock<Rkv>> and a SingleStore, which is +/// the result of GetOrCreateTask. We declare this type because otherwise +/// Clippy complains "error: very complex type used. Consider factoring +/// parts into `type` definitions" (i.e. clippy::type-complexity) when we +/// declare the type of `GetOrCreateTask::result`. +type RkvStoreTuple = (Arc<RwLock<Rkv>>, SingleStore); + +// The threshold for active resizing. +const RESIZE_RATIO: f32 = 0.85; + +/// The threshold (50 MB) to switch the resizing policy from the double size to +/// the constant increment for active resizing. +const INCREMENTAL_RESIZE_THRESHOLD: usize = 52_428_800; + +/// The incremental resize step (5 MB) +const INCREMENTAL_RESIZE_STEP: usize = 5_242_880; + +/// The RKV disk page size and mask. +const PAGE_SIZE: usize = 4096; +const PAGE_SIZE_MASK: usize = 0b_1111_1111_1111; + +/// Round the non-zero size to the multiple of page size greater or equal. +/// +/// It does not handle the special cases such as size zero and overflow, +/// because even if that happens (extremely unlikely though), RKV will +/// ignore the new size if it's smaller than the current size. +/// +/// E.g: +/// [ 1 - 4096] -> 4096, +/// [4097 - 8192] -> 8192, +/// [8193 - 12286] -> 12286, +fn round_to_pagesize(size: usize) -> usize { + if size & PAGE_SIZE_MASK == 0 { + size + } else { + (size & !PAGE_SIZE_MASK) + PAGE_SIZE + } +} + +/// Kvstore employes two auto resizing strategies: active and passive resizing. +/// They work together to liberate consumers from having to guess the "proper" +/// size of the store upfront. See more detail about this in Bug 1543861. +/// +/// Active resizing that is performed at the store startup. +/// +/// It either increases the size in double, or by a constant size if its size +/// reaches INCREMENTAL_RESIZE_THRESHOLD. +/// +/// Note that on Linux / MAC OSX, the increased size would only take effect if +/// there is a write transaction committed afterwards. +fn active_resize(env: &Rkv) -> Result<(), StoreError> { + let info = env.info()?; + let current_size = info.map_size(); + + let size = if current_size < INCREMENTAL_RESIZE_THRESHOLD { + current_size << 1 + } else { + current_size + INCREMENTAL_RESIZE_STEP + }; + + env.set_map_size(size)?; + Ok(()) +} + +/// Passive resizing that is performed when the MAP_FULL error occurs. It +/// increases the store with a `wanted` size. +/// +/// Note that the `wanted` size must be rounded to a multiple of page size +/// by using `round_to_pagesize`. +fn passive_resize(env: &Rkv, wanted: usize) -> Result<(), StoreError> { + let info = env.info()?; + let current_size = info.map_size(); + env.set_map_size(current_size + wanted)?; + Ok(()) +} + +pub struct GetOrCreateTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueDatabaseCallback>>>, + path: nsCString, + name: nsCString, + result: AtomicCell<Option<Result<RkvStoreTuple, KeyValueError>>>, +} + +impl GetOrCreateTask { + pub fn new( + callback: RefPtr<nsIKeyValueDatabaseCallback>, + path: nsCString, + name: nsCString, + ) -> GetOrCreateTask { + GetOrCreateTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + path, + name, + result: AtomicCell::default(), + } + } + + fn convert(&self, result: RkvStoreTuple) -> Result<RefPtr<KeyValueDatabase>, KeyValueError> { + Ok(KeyValueDatabase::new(result.0, result.1)?) + } +} + +impl Task for GetOrCreateTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result + .store(Some(|| -> Result<RkvStoreTuple, KeyValueError> { + let store; + let mut manager = Manager::singleton().write()?; + // Note that path canonicalization is diabled to work around crashes on Fennec: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1531887 + let path = Path::new(str::from_utf8(&self.path)?); + let rkv = manager.get_or_create(path, Rkv::new::<SafeMode>)?; + Migrator::easy_migrate_lmdb_to_safe_mode(path, rkv.read()?)?; + { + let env = rkv.read()?; + let load_ratio = env.load_ratio()?.unwrap_or(0.0); + if load_ratio > RESIZE_RATIO { + active_resize(&env)?; + } + store = env.open_single(str::from_utf8(&self.name)?, StoreOptions::create())?; + } + Ok((rkv, store)) + }())); + } + + task_done!(value); +} + +pub struct PutTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVoidCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + value: OwnedValue, + result: AtomicCell<Option<Result<(), KeyValueError>>>, +} + +impl PutTask { + pub fn new( + callback: RefPtr<nsIKeyValueVoidCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + value: OwnedValue, + ) -> PutTask { + PutTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + key, + value, + result: AtomicCell::default(), + } + } +} + +impl Task for PutTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some(|| -> Result<(), KeyValueError> { + let env = self.rkv.read()?; + let key = str::from_utf8(&self.key)?; + let v = Value::from(&self.value); + let mut resized = false; + + // Use a loop here in case we want to retry from a recoverable + // error such as `StoreError::MapFull`. + loop { + let mut writer = env.write()?; + + match self.store.put(&mut writer, key, &v) { + Ok(_) => (), + + // Only handle the first MapFull error via passive resizing. + // Propogate the subsequent MapFull error. + Err(StoreError::MapFull) if !resized => { + // abort the failed transaction for resizing. + writer.abort(); + + // calculate the size of pairs and resize the store accordingly. + let pair_size = + key.len() + v.serialized_size().map_err(StoreError::from)? as usize; + let wanted = round_to_pagesize(pair_size); + passive_resize(&env, wanted)?; + resized = true; + continue; + } + + Err(err) => return Err(KeyValueError::StoreError(err)), + } + + writer.commit()?; + break; + } + + Ok(()) + }())); + } + + task_done!(void); +} + +pub struct WriteManyTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVoidCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + pairs: Vec<(nsCString, Option<OwnedValue>)>, + result: AtomicCell<Option<Result<(), KeyValueError>>>, +} + +impl WriteManyTask { + pub fn new( + callback: RefPtr<nsIKeyValueVoidCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + pairs: Vec<(nsCString, Option<OwnedValue>)>, + ) -> WriteManyTask { + WriteManyTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + pairs, + result: AtomicCell::default(), + } + } + + fn calc_pair_size(&self) -> Result<usize, StoreError> { + let mut total = 0; + + for (key, value) in self.pairs.iter() { + if let Some(val) = value { + total += key.len(); + total += Value::from(val) + .serialized_size() + .map_err(StoreError::from)? as usize; + } + } + + Ok(total) + } +} + +impl Task for WriteManyTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some(|| -> Result<(), KeyValueError> { + let env = self.rkv.read()?; + let mut resized = false; + + // Use a loop here in case we want to retry from a recoverable + // error such as `StoreError::MapFull`. + 'outer: loop { + let mut writer = env.write()?; + + for (key, value) in self.pairs.iter() { + let key = str::from_utf8(key)?; + match value { + // To put. + Some(val) => { + match self.store.put(&mut writer, key, &Value::from(val)) { + Ok(_) => (), + + // Only handle the first MapFull error via passive resizing. + // Propogate the subsequent MapFull error. + Err(StoreError::MapFull) if !resized => { + // Abort the failed transaction for resizing. + writer.abort(); + + // Calculate the size of pairs and resize accordingly. + let pair_size = self.calc_pair_size()?; + let wanted = round_to_pagesize(pair_size); + passive_resize(&env, wanted)?; + resized = true; + continue 'outer; + } + + Err(err) => return Err(KeyValueError::StoreError(err)), + } + } + // To delete. + None => { + match self.store.delete(&mut writer, key) { + Ok(_) => (), + + // RKV fails with an error if the key to delete wasn't found, + // and Rkv returns that error, but we ignore it, as we expect most + // of our consumers to want this behavior. + Err(StoreError::KeyValuePairNotFound) => (), + + Err(err) => return Err(KeyValueError::StoreError(err)), + }; + } + } + } + + writer.commit()?; + break; // 'outer: loop + } + + Ok(()) + }())); + } + + task_done!(void); +} + +pub struct GetTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVariantCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + default_value: Option<OwnedValue>, + result: AtomicCell<Option<Result<Option<OwnedValue>, KeyValueError>>>, +} + +impl GetTask { + pub fn new( + callback: RefPtr<nsIKeyValueVariantCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + default_value: Option<OwnedValue>, + ) -> GetTask { + GetTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + key, + default_value, + result: AtomicCell::default(), + } + } + + fn convert(&self, result: Option<OwnedValue>) -> Result<RefPtr<nsIVariant>, KeyValueError> { + Ok(match result { + Some(val) => owned_to_variant(val)?, + None => ().into_variant(), + }) + } +} + +impl Task for GetTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result + .store(Some(|| -> Result<Option<OwnedValue>, KeyValueError> { + let key = str::from_utf8(&self.key)?; + let env = self.rkv.read()?; + let reader = env.read()?; + let value = self.store.get(&reader, key)?; + + Ok(match value { + Some(value) => Some(OwnedValue::from(&value)), + None => match self.default_value { + Some(ref val) => Some(val.clone()), + None => None, + }, + }) + }())); + } + + task_done!(value); +} + +pub struct HasTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVariantCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + result: AtomicCell<Option<Result<bool, KeyValueError>>>, +} + +impl HasTask { + pub fn new( + callback: RefPtr<nsIKeyValueVariantCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + ) -> HasTask { + HasTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + key, + result: AtomicCell::default(), + } + } + + fn convert(&self, result: bool) -> Result<RefPtr<nsIVariant>, KeyValueError> { + Ok(result.into_variant()) + } +} + +impl Task for HasTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some(|| -> Result<bool, KeyValueError> { + let key = str::from_utf8(&self.key)?; + let env = self.rkv.read()?; + let reader = env.read()?; + let value = self.store.get(&reader, key)?; + Ok(value.is_some()) + }())); + } + + task_done!(value); +} + +pub struct DeleteTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVoidCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + result: AtomicCell<Option<Result<(), KeyValueError>>>, +} + +impl DeleteTask { + pub fn new( + callback: RefPtr<nsIKeyValueVoidCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + key: nsCString, + ) -> DeleteTask { + DeleteTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + key, + result: AtomicCell::default(), + } + } +} + +impl Task for DeleteTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some(|| -> Result<(), KeyValueError> { + let key = str::from_utf8(&self.key)?; + let env = self.rkv.read()?; + let mut writer = env.write()?; + + match self.store.delete(&mut writer, key) { + Ok(_) => (), + + // RKV fails with an error if the key to delete wasn't found, + // and Rkv returns that error, but we ignore it, as we expect most + // of our consumers to want this behavior. + Err(StoreError::KeyValuePairNotFound) => (), + + Err(err) => return Err(KeyValueError::StoreError(err)), + }; + + writer.commit()?; + + Ok(()) + }())); + } + + task_done!(void); +} + +pub struct ClearTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueVoidCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + result: AtomicCell<Option<Result<(), KeyValueError>>>, +} + +impl ClearTask { + pub fn new( + callback: RefPtr<nsIKeyValueVoidCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + ) -> ClearTask { + ClearTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + result: AtomicCell::default(), + } + } +} + +impl Task for ClearTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some(|| -> Result<(), KeyValueError> { + let env = self.rkv.read()?; + let mut writer = env.write()?; + self.store.clear(&mut writer)?; + writer.commit()?; + + Ok(()) + }())); + } + + task_done!(void); +} + +pub struct EnumerateTask { + callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueEnumeratorCallback>>>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + from_key: nsCString, + to_key: nsCString, + result: AtomicCell<Option<Result<Vec<KeyValuePairResult>, KeyValueError>>>, +} + +impl EnumerateTask { + pub fn new( + callback: RefPtr<nsIKeyValueEnumeratorCallback>, + rkv: Arc<RwLock<Rkv>>, + store: SingleStore, + from_key: nsCString, + to_key: nsCString, + ) -> EnumerateTask { + EnumerateTask { + callback: AtomicCell::new(Some(ThreadBoundRefPtr::new(callback))), + rkv, + store, + from_key, + to_key, + result: AtomicCell::default(), + } + } + + fn convert( + &self, + result: Vec<KeyValuePairResult>, + ) -> Result<RefPtr<KeyValueEnumerator>, KeyValueError> { + Ok(KeyValueEnumerator::new(result)) + } +} + +impl Task for EnumerateTask { + fn run(&self) { + // We do the work within a closure that returns a Result so we can + // use the ? operator to simplify the implementation. + self.result.store(Some( + || -> Result<Vec<KeyValuePairResult>, KeyValueError> { + let env = self.rkv.read()?; + let reader = env.read()?; + let from_key = str::from_utf8(&self.from_key)?; + let to_key = str::from_utf8(&self.to_key)?; + + let iterator = if from_key.is_empty() { + self.store.iter_start(&reader)? + } else { + self.store.iter_from(&reader, &from_key)? + }; + + // Ideally, we'd enumerate pairs lazily, as the consumer calls + // nsIKeyValueEnumerator.getNext(), which calls our + // KeyValueEnumerator.get_next() implementation. But KeyValueEnumerator + // can't reference the Iter because Rust "cannot #[derive(xpcom)] + // on a generic type," and the Iter requires a lifetime parameter, + // which would make KeyValueEnumerator generic. + // + // Our fallback approach is to eagerly collect the iterator + // into a collection that KeyValueEnumerator owns. Fixing this so we + // enumerate pairs lazily is bug 1499252. + let pairs: Vec<KeyValuePairResult> = iterator + // Convert the key to a string so we can compare it to the "to" key. + // For forward compatibility, we don't fail here if we can't convert + // a key to UTF-8. Instead, we store the Err in the collection + // and fail lazily in KeyValueEnumerator.get_next(). + .map(|result| match result { + Ok((key, val)) => Ok((str::from_utf8(&key), val)), + Err(err) => Err(err), + }) + // Stop iterating once we reach the to_key, if any. + .take_while(|result| match result { + Ok((key, _val)) => { + if to_key.is_empty() { + true + } else { + match *key { + Ok(key) => key < to_key, + Err(_err) => true, + } + } + } + Err(_) => true, + }) + // Convert the key/value pair to owned. + .map(|result| match result { + Ok((key, val)) => match (key, val) { + (Ok(key), val) => Ok((key.to_owned(), OwnedValue::from(&val))), + (Err(err), _) => Err(err.into()), + }, + Err(err) => Err(KeyValueError::StoreError(err)), + }) + .collect(); + + Ok(pairs) + }(), + )); + } + + task_done!(value); +} diff --git a/toolkit/components/kvstore/test/xpcshell/data/test-env-32/data.mdb b/toolkit/components/kvstore/test/xpcshell/data/test-env-32/data.mdb Binary files differnew file mode 100644 index 0000000000..75e555baab --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/data/test-env-32/data.mdb diff --git a/toolkit/components/kvstore/test/xpcshell/data/test-env-32/lock.mdb b/toolkit/components/kvstore/test/xpcshell/data/test-env-32/lock.mdb Binary files differnew file mode 100644 index 0000000000..76bed60632 --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/data/test-env-32/lock.mdb diff --git a/toolkit/components/kvstore/test/xpcshell/data/test-env-64/data.mdb b/toolkit/components/kvstore/test/xpcshell/data/test-env-64/data.mdb Binary files differnew file mode 100644 index 0000000000..d6400c69e7 --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/data/test-env-64/data.mdb diff --git a/toolkit/components/kvstore/test/xpcshell/data/test-env-64/lock.mdb b/toolkit/components/kvstore/test/xpcshell/data/test-env-64/lock.mdb Binary files differnew file mode 100644 index 0000000000..e083a773e9 --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/data/test-env-64/lock.mdb diff --git a/toolkit/components/kvstore/test/xpcshell/make-test-env.js b/toolkit/components/kvstore/test/xpcshell/make-test-env.js new file mode 100644 index 0000000000..041677b9a6 --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/make-test-env.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// An xpcshell script to create a test database. Useful for creating +// the test-env-32 and test-env-64 databases that we use to test migration +// of databases across architecture changes. +// +// To create a test database, simply run this script using xpcshell: +// +// path/to/xpcshell path/to/make-test-env.js +// +// The script will create the test-env-32 or test-env-64 directory +// (depending on the current architecture) in the current working directory, +// create a database called "db" within it, and populate the database +// with sample data. +// +// Note: you don't necessarily need to run this script on every architecture +// for which you'd like to create a database. Once you have a database for one +// architecture, you can use the mdb_dump and mdb_load utilities (if available +// for your systems) to create them for others. To do so, first dump the data +// on the original architecture: +// +// mdb_dump -s db path/to/original/test-env-dir > path/to/dump.txt +// +// Then load the data on the new architecture: +// +// mkdir path/to/new/test-env-dir +// mdb_load -s db path/to/dump.txt + +"use strict"; + +const { KeyValueService } = ChromeUtils.import( + "resource://gre/modules/kvstore.jsm" +); +const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm"); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +(async function() { + const currentDir = await OS.File.getCurrentDirectory(); + const testEnvDir = Services.appinfo.is64Bit ? "test-env-64" : "test-env-32"; + const testEnvPath = OS.Path.join(currentDir, testEnvDir); + await OS.File.makeDir(testEnvPath, { from: currentDir }); + + const database = await KeyValueService.getOrCreate(testEnvPath, "db"); + await database.put("int-key", 1234); + await database.put("double-key", 56.78); + await database.put("string-key", "Héllo, wőrld!"); + await database.put("bool-key", true); + + scriptDone = true; +})(); + +// Do async processing until the async function call completes. +// From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO> +let scriptDone = false; +const mainThread = Services.tm.currentThread; +while (!scriptDone) { + mainThread.processNextEvent(true); +} +while (mainThread.hasPendingEvents()) { + mainThread.processNextEvent(true); +} diff --git a/toolkit/components/kvstore/test/xpcshell/test_kvstore.js b/toolkit/components/kvstore/test/xpcshell/test_kvstore.js new file mode 100644 index 0000000000..c7defd2294 --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/test_kvstore.js @@ -0,0 +1,594 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { KeyValueService } = ChromeUtils.import( + "resource://gre/modules/kvstore.jsm" +); +const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm"); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +function run_test() { + do_get_profile(); + run_next_test(); +} + +async function makeDatabaseDir(name) { + const databaseDir = OS.Path.join(OS.Constants.Path.profileDir, name); + await OS.File.makeDir(databaseDir, { from: OS.Constants.Path.profileDir }); + return databaseDir; +} + +const gKeyValueService = Cc["@mozilla.org/key-value-service;1"].getService( + Ci.nsIKeyValueService +); + +add_task(async function getService() { + Assert.ok(gKeyValueService); +}); + +add_task(async function getOrCreate() { + const databaseDir = await makeDatabaseDir("getOrCreate"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + Assert.ok(database); + + // Test creating a database with a nonexistent path. + const nonexistentDir = OS.Path.join( + OS.Constants.Path.profileDir, + "nonexistent" + ); + await Assert.rejects( + KeyValueService.getOrCreate(nonexistentDir, "db"), + /UnsuitableEnvironmentPath/ + ); + + // Test creating a database with a non-normalized but fully-qualified path. + let nonNormalizedDir = await makeDatabaseDir("non-normalized"); + nonNormalizedDir = OS.Path.join( + nonNormalizedDir, + "..", + ".", + "non-normalized" + ); + Assert.ok(await KeyValueService.getOrCreate(nonNormalizedDir, "db")); +}); + +add_task(async function putGetHasDelete() { + const databaseDir = await makeDatabaseDir("putGetHasDelete"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // Getting key/value pairs that don't exist (yet) returns default values + // or null, depending on whether you specify a default value. + Assert.strictEqual(await database.get("int-key", 1), 1); + Assert.strictEqual(await database.get("double-key", 1.1), 1.1); + Assert.strictEqual(await database.get("string-key", ""), ""); + Assert.strictEqual(await database.get("bool-key", false), false); + Assert.strictEqual(await database.get("int-key"), null); + Assert.strictEqual(await database.get("double-key"), null); + Assert.strictEqual(await database.get("string-key"), null); + Assert.strictEqual(await database.get("bool-key"), null); + + // The put method succeeds without returning a value. + Assert.strictEqual(await database.put("int-key", 1234), undefined); + Assert.strictEqual(await database.put("double-key", 56.78), undefined); + Assert.strictEqual( + await database.put("string-key", "Héllo, wőrld!"), + undefined + ); + Assert.strictEqual(await database.put("bool-key", true), undefined); + + // Getting key/value pairs that exist returns the expected values. + Assert.strictEqual(await database.get("int-key", 1), 1234); + Assert.strictEqual(await database.get("double-key", 1.1), 56.78); + Assert.strictEqual(await database.get("string-key", ""), "Héllo, wőrld!"); + Assert.strictEqual(await database.get("bool-key", false), true); + Assert.strictEqual(await database.get("int-key"), 1234); + Assert.strictEqual(await database.get("double-key"), 56.78); + Assert.strictEqual(await database.get("string-key"), "Héllo, wőrld!"); + Assert.strictEqual(await database.get("bool-key"), true); + + // The has() method works as expected for both existing and non-existent keys. + Assert.strictEqual(await database.has("int-key"), true); + Assert.strictEqual(await database.has("double-key"), true); + Assert.strictEqual(await database.has("string-key"), true); + Assert.strictEqual(await database.has("bool-key"), true); + Assert.strictEqual(await database.has("nonexistent-key"), false); + + // The delete() method succeeds without returning a value. + Assert.strictEqual(await database.delete("int-key"), undefined); + Assert.strictEqual(await database.delete("double-key"), undefined); + Assert.strictEqual(await database.delete("string-key"), undefined); + Assert.strictEqual(await database.delete("bool-key"), undefined); + + // The has() method works as expected for a deleted key. + Assert.strictEqual(await database.has("int-key"), false); + Assert.strictEqual(await database.has("double-key"), false); + Assert.strictEqual(await database.has("string-key"), false); + Assert.strictEqual(await database.has("bool-key"), false); + + // Getting key/value pairs that were deleted returns default values. + Assert.strictEqual(await database.get("int-key", 1), 1); + Assert.strictEqual(await database.get("double-key", 1.1), 1.1); + Assert.strictEqual(await database.get("string-key", ""), ""); + Assert.strictEqual(await database.get("bool-key", false), false); + Assert.strictEqual(await database.get("int-key"), null); + Assert.strictEqual(await database.get("double-key"), null); + Assert.strictEqual(await database.get("string-key"), null); + Assert.strictEqual(await database.get("bool-key"), null); +}); + +add_task(async function putWithResizing() { + const databaseDir = await makeDatabaseDir("putWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1M = base.repeat(32768); + const val2M = val1M.repeat(2); + Assert.strictEqual(await database.put("A-1M-value", val1M), undefined); + Assert.strictEqual(await database.put("A-2M-value", val2M), undefined); + Assert.strictEqual(await database.put("A-32B-value", base), undefined); + + Assert.strictEqual(await database.get("A-1M-value"), val1M); + Assert.strictEqual(await database.get("A-2M-value"), val2M); + Assert.strictEqual(await database.get("A-32B-value"), base); +}); + +add_task(async function largeNumbers() { + const databaseDir = await makeDatabaseDir("largeNumbers"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + const MAX_INT_VARIANT = Math.pow(2, 31) - 1; + const MIN_DOUBLE_VARIANT = Math.pow(2, 31); + + await database.put("max-int-variant", MAX_INT_VARIANT); + await database.put("min-double-variant", MIN_DOUBLE_VARIANT); + await database.put("max-safe-integer", Number.MAX_SAFE_INTEGER); + await database.put("min-safe-integer", Number.MIN_SAFE_INTEGER); + await database.put("max-value", Number.MAX_VALUE); + await database.put("min-value", Number.MIN_VALUE); + + Assert.strictEqual(await database.get("max-int-variant"), MAX_INT_VARIANT); + Assert.strictEqual( + await database.get("min-double-variant"), + MIN_DOUBLE_VARIANT + ); + Assert.strictEqual( + await database.get("max-safe-integer"), + Number.MAX_SAFE_INTEGER + ); + Assert.strictEqual( + await database.get("min-safe-integer"), + Number.MIN_SAFE_INTEGER + ); + Assert.strictEqual(await database.get("max-value"), Number.MAX_VALUE); + Assert.strictEqual(await database.get("min-value"), Number.MIN_VALUE); +}); + +add_task(async function extendedCharacterKey() { + const databaseDir = await makeDatabaseDir("extendedCharacterKey"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // Ensure that we can use extended character (i.e. non-ASCII) strings as keys. + + await database.put("Héllo, wőrld!", 1); + Assert.strictEqual(await database.has("Héllo, wőrld!"), true); + Assert.strictEqual(await database.get("Héllo, wőrld!"), 1); + + const enumerator = await database.enumerate(); + const { key } = enumerator.getNext(); + Assert.strictEqual(key, "Héllo, wőrld!"); + + await database.delete("Héllo, wőrld!"); +}); + +add_task(async function clear() { + const databaseDir = await makeDatabaseDir("clear"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + await database.put("int-key", 1234); + await database.put("double-key", 56.78); + await database.put("string-key", "Héllo, wőrld!"); + await database.put("bool-key", true); + + Assert.strictEqual(await database.clear(), undefined); + Assert.strictEqual(await database.has("int-key"), false); + Assert.strictEqual(await database.has("double-key"), false); + Assert.strictEqual(await database.has("string-key"), false); + Assert.strictEqual(await database.has("bool-key"), false); +}); + +add_task(async function writeManyFailureCases() { + const databaseDir = await makeDatabaseDir("writeManyFailureCases"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + Assert.throws(() => database.writeMany(), /unexpected argument/); + Assert.throws(() => database.writeMany("foo"), /unexpected argument/); + Assert.throws(() => database.writeMany(["foo"]), /unexpected argument/); +}); + +add_task(async function writeManyPutOnly() { + const databaseDir = await makeDatabaseDir("writeMany"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + async function test_helper(pairs) { + Assert.strictEqual(await database.writeMany(pairs), undefined); + Assert.strictEqual(await database.get("int-key"), 1234); + Assert.strictEqual(await database.get("double-key"), 56.78); + Assert.strictEqual(await database.get("string-key"), "Héllo, wőrld!"); + Assert.strictEqual(await database.get("bool-key"), true); + await database.clear(); + } + + // writeMany with an empty object is OK + Assert.strictEqual(await database.writeMany({}), undefined); + + // writeMany with an object + const pairs = { + "int-key": 1234, + "double-key": 56.78, + "string-key": "Héllo, wőrld!", + "bool-key": true, + }; + await test_helper(pairs); + + // writeMany with an array of pairs + const arrayPairs = [ + ["int-key", 1234], + ["double-key", 56.78], + ["string-key", "Héllo, wőrld!"], + ["bool-key", true], + ]; + await test_helper(arrayPairs); + + // writeMany with a key/value generator + function* pairMaker() { + yield ["int-key", 1234]; + yield ["double-key", 56.78]; + yield ["string-key", "Héllo, wőrld!"]; + yield ["bool-key", true]; + } + await test_helper(pairMaker()); + + // writeMany with a map + const mapPairs = new Map(arrayPairs); + await test_helper(mapPairs); +}); + +add_task(async function writeManyLargePairsWithResizing() { + const databaseDir = await makeDatabaseDir("writeManyWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1M = base.repeat(32768); + const val2M = val1M.repeat(2); + + // writeMany with an object + const pairs = { + "A-1M-value": val1M, + "A-32B-value": base, + "A-2M-value": val2M, + }; + + Assert.strictEqual(await database.writeMany(pairs), undefined); + + Assert.strictEqual(await database.get("A-1M-value"), val1M); + Assert.strictEqual(await database.get("A-2M-value"), val2M); + Assert.strictEqual(await database.get("A-32B-value"), base); +}); + +add_task(async function writeManySmallPairsWithResizing() { + const databaseDir = await makeDatabaseDir("writeManyWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1K = base.repeat(32); + // writeMany with a key/value generator + function* pairMaker() { + for (let i = 0; i < 1024; i++) { + yield [`key-${i}`, val1K]; + } + } + + Assert.strictEqual(await database.writeMany(pairMaker()), undefined); + for (let i = 0; i < 1024; i++) { + Assert.ok(await database.has(`key-${i}`)); + } +}); + +add_task(async function writeManyDeleteOnly() { + const databaseDir = await makeDatabaseDir("writeManyDeletesOnly"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // writeMany with an object + const pairs = { + "int-key": 1234, + "double-key": 56.78, + "string-key": "Héllo, wőrld!", + "bool-key": true, + }; + + async function test_helper(deletes) { + Assert.strictEqual(await database.writeMany(pairs), undefined); + Assert.strictEqual(await database.writeMany(deletes), undefined); + Assert.strictEqual(await database.get("int-key"), null); + Assert.strictEqual(await database.get("double-key"), null); + Assert.strictEqual(await database.get("string-key"), null); + Assert.strictEqual(await database.get("bool-key"), null); + } + + // writeMany with an empty object is OK + Assert.strictEqual(await database.writeMany({}), undefined); + + // writeMany with an object + await test_helper({ + "int-key": null, + "double-key": null, + "string-key": null, + "bool-key": null, + }); + + // writeMany with an array of pairs + const arrayPairs = [ + ["int-key", null], + ["double-key", null], + ["string-key", null], + ["bool-key", null], + ]; + await test_helper(arrayPairs); + + // writeMany with a key/value generator + function* pairMaker() { + yield ["int-key", null]; + yield ["double-key", null]; + yield ["string-key", null]; + yield ["bool-key", null]; + } + await test_helper(pairMaker()); + + // writeMany with a map + const mapPairs = new Map(arrayPairs); + await test_helper(mapPairs); +}); + +add_task(async function writeManyPutDelete() { + const databaseDir = await makeDatabaseDir("writeManyPutDelete"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + await database.writeMany([ + ["key1", "val1"], + ["key3", "val3"], + ["key4", "val4"], + ["key5", "val5"], + ]); + + await database.writeMany([ + ["key2", "val2"], + ["key4", null], + ["key5", null], + ]); + + Assert.strictEqual(await database.get("key1"), "val1"); + Assert.strictEqual(await database.get("key2"), "val2"); + Assert.strictEqual(await database.get("key3"), "val3"); + Assert.strictEqual(await database.get("key4"), null); + Assert.strictEqual(await database.get("key5"), null); + + await database.clear(); + + await database.writeMany([ + ["key1", "val1"], + ["key1", null], + ["key1", "val11"], + ["key1", null], + ["key2", null], + ["key2", "val2"], + ]); + + Assert.strictEqual(await database.get("key1"), null); + Assert.strictEqual(await database.get("key2"), "val2"); +}); + +add_task(async function getOrCreateNamedDatabases() { + const databaseDir = await makeDatabaseDir("getOrCreateNamedDatabases"); + + let fooDB = await KeyValueService.getOrCreate(databaseDir, "foo"); + Assert.ok(fooDB, "retrieval of first named database works"); + + let barDB = await KeyValueService.getOrCreate(databaseDir, "bar"); + Assert.ok(barDB, "retrieval of second named database works"); + + let bazDB = await KeyValueService.getOrCreate(databaseDir, "baz"); + Assert.ok(bazDB, "retrieval of third named database works"); + + // Key/value pairs that are put into a database don't exist in others. + await bazDB.put("key", 1); + Assert.ok(!(await fooDB.has("key")), "the foo DB still doesn't have the key"); + await fooDB.put("key", 2); + Assert.ok(!(await barDB.has("key")), "the bar DB still doesn't have the key"); + await barDB.put("key", 3); + Assert.strictEqual( + await bazDB.get("key", 0), + 1, + "the baz DB has its KV pair" + ); + Assert.strictEqual( + await fooDB.get("key", 0), + 2, + "the foo DB has its KV pair" + ); + Assert.strictEqual( + await barDB.get("key", 0), + 3, + "the bar DB has its KV pair" + ); + + // Key/value pairs that are deleted from a database still exist in other DBs. + await bazDB.delete("key"); + Assert.strictEqual( + await fooDB.get("key", 0), + 2, + "the foo DB still has its KV pair" + ); + await fooDB.delete("key"); + Assert.strictEqual( + await barDB.get("key", 0), + 3, + "the bar DB still has its KV pair" + ); + await barDB.delete("key"); +}); + +add_task(async function enumeration() { + const databaseDir = await makeDatabaseDir("enumeration"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + await database.put("int-key", 1234); + await database.put("double-key", 56.78); + await database.put("string-key", "Héllo, wőrld!"); + await database.put("bool-key", true); + + async function test(fromKey, toKey, pairs) { + const enumerator = await database.enumerate(fromKey, toKey); + + for (const pair of pairs) { + Assert.strictEqual(enumerator.hasMoreElements(), true); + const element = enumerator.getNext(); + Assert.ok(element); + Assert.strictEqual(element.key, pair[0]); + Assert.strictEqual(element.value, pair[1]); + } + + Assert.strictEqual(enumerator.hasMoreElements(), false); + Assert.throws(() => enumerator.getNext(), /NS_ERROR_FAILURE/); + } + + // Test enumeration without specifying "from" and "to" keys, which should + // enumerate all of the pairs in the database. This test does so explicitly + // by passing "null", "undefined" or "" (empty string) arguments + // for those parameters. The iterator test below also tests this implicitly + // by not specifying arguments for those parameters. + await test(null, null, [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + await test(undefined, undefined, [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // The implementation doesn't distinguish between a null/undefined value + // and an empty string, so enumerating pairs from "" to "" has the same effect + // as enumerating pairs without specifying from/to keys: it enumerates + // all of the pairs in the database. + await test("", "", [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // Test enumeration from a key that doesn't exist and is lexicographically + // less than the least key in the database, which should enumerate + // all of the pairs in the database. + await test("aaaaa", null, [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // Test enumeration from a key that doesn't exist and is lexicographically + // greater than the first key in the database, which should enumerate pairs + // whose key is greater than or equal to the specified key. + await test("ccccc", null, [ + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // Test enumeration from a key that does exist, which should enumerate pairs + // whose key is greater than or equal to that key. + await test("int-key", null, [ + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // Test enumeration from a key that doesn't exist and is lexicographically + // greater than the greatest test key in the database, which should enumerate + // none of the pairs in the database. + await test("zzzzz", null, []); + + // Test enumeration to a key that doesn't exist and is lexicographically + // greater than the greatest test key in the database, which should enumerate + // all of the pairs in the database. + await test(null, "zzzzz", [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ["string-key", "Héllo, wőrld!"], + ]); + + // Test enumeration to a key that doesn't exist and is lexicographically + // less than the greatest test key in the database, which should enumerate + // pairs whose key is less than the specified key. + await test(null, "ppppp", [ + ["bool-key", true], + ["double-key", 56.78], + ["int-key", 1234], + ]); + + // Test enumeration to a key that does exist, which should enumerate pairs + // whose key is less than that key. + await test(null, "int-key", [ + ["bool-key", true], + ["double-key", 56.78], + ]); + + // Test enumeration to a key that doesn't exist and is lexicographically + // less than the least key in the database, which should enumerate + // none of the pairs in the database. + await test(null, "aaaaa", []); + + // Test enumeration between intermediate keys that don't exist, which should + // enumerate the pairs whose keys lie in between them. + await test("ggggg", "ppppp", [["int-key", 1234]]); + + // Test enumeration from a key that exists to the same key, which shouldn't + // enumerate any pairs, because the "to" key is exclusive. + await test("int-key", "int-key", []); + + // Test enumeration from a greater key to a lesser one, which should + // enumerate none of the pairs in the database, even if the reverse ordering + // would enumerate some pairs. Consumers are responsible for ordering + // the "from" and "to" keys such that "from" is less than or equal to "to". + await test("ppppp", "ccccc", []); + await test("int-key", "ccccc", []); + await test("ppppp", "int-key", []); + + const actual = {}; + for (const { key, value } of await database.enumerate()) { + actual[key] = value; + } + Assert.deepEqual(actual, { + "bool-key": true, + "double-key": 56.78, + "int-key": 1234, + "string-key": "Héllo, wőrld!", + }); + + await database.delete("int-key"); + await database.delete("double-key"); + await database.delete("string-key"); + await database.delete("bool-key"); +}); diff --git a/toolkit/components/kvstore/test/xpcshell/xpcshell.ini b/toolkit/components/kvstore/test/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..1dad3a27ba --- /dev/null +++ b/toolkit/components/kvstore/test/xpcshell/xpcshell.ini @@ -0,0 +1,6 @@ +[DEFAULT] +skip-if = toolkit == 'android' +support-files = + data/** + +[test_kvstore.js] |