From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- third_party/rust/rkv/tests/env-all.rs | 328 ++++ third_party/rust/rkv/tests/env-lmdb.rs | 1647 +++++++++++++++++++++ third_party/rust/rkv/tests/env-migration.rs | 607 ++++++++ third_party/rust/rkv/tests/env-safe.rs | 1370 +++++++++++++++++ third_party/rust/rkv/tests/integer-store.rs | 88 ++ third_party/rust/rkv/tests/manager.rs | 471 ++++++ third_party/rust/rkv/tests/multi-integer-store.rs | 122 ++ third_party/rust/rkv/tests/test_txn.rs | 131 ++ 8 files changed, 4764 insertions(+) create mode 100644 third_party/rust/rkv/tests/env-all.rs create mode 100644 third_party/rust/rkv/tests/env-lmdb.rs create mode 100644 third_party/rust/rkv/tests/env-migration.rs create mode 100644 third_party/rust/rkv/tests/env-safe.rs create mode 100644 third_party/rust/rkv/tests/integer-store.rs create mode 100644 third_party/rust/rkv/tests/manager.rs create mode 100644 third_party/rust/rkv/tests/multi-integer-store.rs create mode 100644 third_party/rust/rkv/tests/test_txn.rs (limited to 'third_party/rust/rkv/tests') diff --git a/third_party/rust/rkv/tests/env-all.rs b/third_party/rust/rkv/tests/env-all.rs new file mode 100644 index 0000000000..a8d93c786e --- /dev/null +++ b/third_party/rust/rkv/tests/env-all.rs @@ -0,0 +1,328 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::fs; + +use tempfile::Builder; + +use rkv::{ + backend::{Lmdb, SafeMode}, + Rkv, StoreOptions, Value, +}; + +#[test] +fn test_open_safe_same_dir_as_lmdb() { + let root = Builder::new() + .prefix("test_open_safe_same_dir_as_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create database of type A and save to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type A was written to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and verify that it is empty. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let _ = k + .open_single("sk", StoreOptions::default()) + .expect_err("not opened"); + } + // Verify that database of type A wasn't changed. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and save to disk (type A exists at the same path). + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo1", &Value::I64(5678)) + .expect("wrote"); + sk.put(&mut writer, "bar1", &Value::Bool(false)) + .expect("wrote"); + sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&writer, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&writer, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type B was written to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&reader, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&reader, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + } + // Verify that database of type A still wasn't changed. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } +} + +#[test] +fn test_open_lmdb_same_dir_as_safe() { + let root = Builder::new() + .prefix("test_open_lmdb_same_dir_as_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create database of type A and save to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type A was written to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and verify that it is empty. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let _ = k + .open_single("sk", StoreOptions::default()) + .expect_err("not opened"); + } + // Verify that database of type A wasn't changed. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and save to disk (type A exists at the same path). + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo1", &Value::I64(5678)) + .expect("wrote"); + sk.put(&mut writer, "bar1", &Value::Bool(false)) + .expect("wrote"); + sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&writer, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&writer, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type B was written to disk. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&reader, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&reader, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + } + // Verify that database of type A still wasn't changed. + { + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } +} diff --git a/third_party/rust/rkv/tests/env-lmdb.rs b/third_party/rust/rkv/tests/env-lmdb.rs new file mode 100644 index 0000000000..e3aca394ab --- /dev/null +++ b/third_party/rust/rkv/tests/env-lmdb.rs @@ -0,0 +1,1647 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable +// deprecates `clippy::cyclomatic_complexity`. +#![allow(clippy::complexity)] + +use std::{ + fs, + path::Path, + str, + sync::{Arc, RwLock}, + thread, +}; + +use byteorder::{ByteOrder, LittleEndian}; +use tempfile::Builder; + +use rkv::{ + backend::{ + BackendEnvironmentBuilder, BackendInfo, BackendStat, Lmdb, LmdbDatabase, LmdbEnvironment, + LmdbRwTransaction, + }, + EnvironmentFlags, Rkv, SingleStore, StoreError, StoreOptions, Value, Writer, +}; + +fn check_rkv(k: &Rkv) { + let _ = k + .open_single(None, StoreOptions::create()) + .expect("created default"); + + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + let reader = k.read().expect("reader"); + + let result = s.get(&reader, "foo"); + assert_eq!(None, result.expect("success but no value")); +} + +// The default size is 1MB. +const DEFAULT_SIZE: usize = 1024 * 1024; + +/// We can't open a directory that doesn't exist. +#[test] +fn test_open_fails() { + let root = Builder::new() + .prefix("test_open_fails") + .tempdir() + .expect("tempdir"); + assert!(root.path().exists()); + + let nope = root.path().join("nope/"); + assert!(!nope.exists()); + + let pb = nope.to_path_buf(); + match Rkv::new::(nope.as_path()).err() { + Some(StoreError::UnsuitableEnvironmentPath(p)) => { + assert_eq!(pb, p); + } + _ => panic!("expected error"), + }; +} + +#[test] +fn test_open() { + let root = Builder::new() + .prefix("test_open") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_with_no_subdir_1() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + { + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); + } + { + let mut builder = Rkv::environment_builder::(); + builder.set_flags(EnvironmentFlags::NO_SUB_DIR); + builder.set_max_dbs(2); + + let mut datamdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + + let k = Rkv::from_builder(&datamdb, builder).expect("rkv"); + check_rkv(&k); + } +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath")] +fn test_open_from_builder_with_no_subdir_2() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + { + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); + } + { + let mut builder = Rkv::environment_builder::(); + builder.set_flags(EnvironmentFlags::NO_SUB_DIR); + builder.set_max_dbs(2); + + let mut datamdb = root.path().to_path_buf(); + datamdb.push("bogus.mdb"); + + let k = Rkv::from_builder(&datamdb, builder).expect("rkv"); + check_rkv(&k); + } +} + +#[test] +fn test_open_from_builder_with_dir_1() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + builder.set_make_dir_if_needed(true); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")] +fn test_open_from_builder_with_dir_2() { + let root = Path::new("bogus"); + println!("Root path: {root:?}"); + assert!(!root.is_dir()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root, builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_create_with_capacity_1() { + let root = Builder::new() + .prefix("test_create_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This errors with "opened: DbsFull" because we specified a capacity of one (database), + // and check_rkv already opened one (plus the default database, which doesn't count + // against the limit). + let _zzz = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); +} + +#[test] +fn test_create_with_capacity_2() { + let root = Builder::new() + .prefix("test_create_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This doesn't error with "opened: DbsFull" with because even though we specified a + // capacity of one (database), and check_rkv already opened one, the default database + // doesn't count against the limit. + let _zzz = k.open_single(None, StoreOptions::create()).expect("opened"); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_open_with_capacity_1() { + let root = Builder::new() + .prefix("test_open_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single("zzz", StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_open_with_capacity_2() { + let root = Builder::new() + .prefix("test_open_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single(None, StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_list_dbs_1() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![Some("s".to_owned())]); +} + +#[test] +fn test_list_dbs_2() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 2).expect("rkv"); + check_rkv(&k); + + let _ = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![Some("s".to_owned()), Some("zzz".to_owned())]); +} + +#[test] +fn test_list_dbs_3() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 0).expect("rkv"); + + let _ = k.open_single(None, StoreOptions::create()).expect("opened"); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![None]); +} + +fn get_larger_than_default_map_size_value() -> usize { + // The LMDB C library and lmdb Rust crate docs for setting the map size + // + // + // both say that the default map size is 10,485,760 bytes, i.e. 10MiB. + // + // But the DEFAULT_MAPSIZE define in the LMDB code + // https://github.com/LMDB/lmdb/blob/26c7df88e44e31623d0802a564f24781acdefde3/libraries/liblmdb/mdb.c#L729 + // sets the default map size to 1,048,576 bytes, i.e. 1MiB. + // + DEFAULT_SIZE + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */ +} + +#[test] +#[should_panic(expected = "wrote: MapFull")] +fn test_exceed_map_size() { + let root = Builder::new() + .prefix("test_exceed_map_size") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + + // Writing a large enough value should cause LMDB to fail on MapFull. + // We write a string that is larger than the default map size. + let val = "x".repeat(get_larger_than_default_map_size_value()); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str(&val)) + .expect("wrote"); +} + +#[test] +#[should_panic(expected = "wrote: KeyValuePairBadSize")] +fn test_exceed_key_size_limit() { + let root = Builder::new() + .prefix("test_exceed_key_size_limit") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let key = "k".repeat(512); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, key, &Value::Str("val")).expect("wrote"); +} + +#[test] +fn test_increase_map_size() { + let root = Builder::new() + .prefix("test_open_with_map_size") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::(); + // Set the map size to the size of the value we'll store in it + 100KiB, + // which ensures that there's enough space for the value and metadata. + builder.set_map_size( + get_larger_than_default_map_size_value() + 100 * 1024, /* 100KiB */ + ); + builder.set_max_dbs(2); + let k = Rkv::from_builder(root.path(), builder).unwrap(); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + let val = "x".repeat(get_larger_than_default_map_size_value()); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str(&val)) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::Str(&val)) + ); +} + +#[test] +fn test_round_trip_and_transactions() { + let root = Builder::new() + .prefix("test_round_trip_and_transactions") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "noo").expect("read"), + Some(Value::F64(1234.0.into())) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + // Isolation. Reads won't return values. + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + // Dropped: tx rollback. Reads will still return nothing. + + { + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + // Isolation. Reads still return values. + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + // Dropped: tx rollback. Reads will still return values. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed but return None to indicate a missing value. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), None); + assert_eq!(sk.get(&r, "bar").expect("read"), None); + assert_eq!(sk.get(&r, "baz").expect("read"), None); + } +} + +#[test] +fn test_single_store_clear() { + let root = Builder::new() + .prefix("test_single_store_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + let iter = sk.iter_start(&r).expect("iter"); + assert_eq!(iter.count(), 0); + } +} + +#[test] +#[should_panic(expected = "KeyValuePairNotFound")] +fn test_single_store_delete_nonexistent() { + let root = Builder::new() + .prefix("test_single_store_delete_nonexistent") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "bogus").unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multi_put_get_del() { + let root = Builder::new() + .prefix("test_multi_put_get_del") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().unwrap(); + + let writer = k.write().unwrap(); + { + let mut iter = multistore.get(&writer, "str1").unwrap(); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar"))); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo"))); + } + writer.commit().unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .delete(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str1").unwrap(), + Some(Value::Str("str1 bar")) + ); + multistore + .delete(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str2").unwrap(), + Some(Value::Str("str2 foo")) + ); + multistore.delete_all(&mut writer, "str3").unwrap(); + assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None); + writer.commit().unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multiple_store_clear() { + let root = Builder::new() + .prefix("test_multiple_store_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let multistore = k + .open_multi("multistore", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = k.write().expect("writer"); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + multistore.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + assert_eq!(multistore.get_first(&r, "str1").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str2").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str3").expect("read"), None); + } +} + +#[test] +fn test_open_store_for_read() { + let root = Builder::new() + .prefix("test_open_store_for_read") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + + // First create the store, and start a write transaction on it. + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("write"); + + // Open the same store for read, note that the write transaction is still in progress, + // it should not block the reader though. + let sk_readonly = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + writer.commit().expect("commit"); + + // Now the write transaction is committed, any followed reads should see its change. + let reader = k.read().expect("reader"); + assert_eq!( + sk_readonly.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); +} + +#[test] +#[should_panic(expected = "open a missing store")] +fn test_open_a_missing_store() { + let root = Builder::new() + .prefix("test_open_a_missing_store") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let _sk = k + .open_single("sk", StoreOptions::default()) + .expect("open a missing store"); +} + +#[test] +#[should_panic(expected = "new failed: FileInvalid")] +fn test_open_a_broken_store() { + let root = Builder::new() + .prefix("test_open_a_missing_store") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + let _ = Rkv::new::(root.path()).expect("new failed"); +} + +#[test] +fn test_open_fail_with_badrslot() { + let root = Builder::new() + .prefix("test_open_fail_with_badrslot") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::default()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_read_before_write_num() { + let root = Builder::new() + .prefix("test_read_before_write_num") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a number, modifying it, and then writing it back. + // We have to be done with the Value::I64 before calling Writer::put, + // as the Value::I64 borrows an immutable reference to the Writer. + // So we extract and copy its primitive value. + + fn get_existing_foo( + store: SingleStore, + writer: &Writer, + ) -> Option { + match store.get(writer, "foo").expect("read") { + Some(Value::I64(val)) => Some(val), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or(99); + existing += 1; + sk.put(&mut writer, "foo", &Value::I64(existing)) + .expect("success"); + + let updated = get_existing_foo(sk, &writer).unwrap_or(99); + assert_eq!(updated, 100); + writer.commit().expect("commit"); +} + +#[test] +fn test_read_before_write_str() { + let root = Builder::new() + .prefix("test_read_before_write_str") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a string, modifying it, and then writing it back. + // We have to be done with the Value::Str before calling Writer::put, + // as the Value::Str (and its underlying &str) borrows an immutable + // reference to the Writer. So we copy it to a String. + + fn get_existing_foo( + store: SingleStore, + writer: &Writer, + ) -> Option { + match store.get(writer, "foo").expect("read") { + Some(Value::Str(val)) => Some(val.to_string()), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or_default(); + existing.push('…'); + sk.put(&mut writer, "foo", &Value::Str(&existing)) + .expect("write"); + + let updated = get_existing_foo(sk, &writer).unwrap_or_default(); + assert_eq!(updated, "…"); + writer.commit().expect("commit"); +} + +#[test] +fn test_concurrent_read_transactions_prohibited() { + let root = Builder::new() + .prefix("test_concurrent_reads_prohibited") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let _first = k.read().expect("reader"); + let second = k.read(); + + match second { + Err(StoreError::ReadTransactionAlreadyExists(t)) => { + println!("Thread was {t:?}"); + } + Err(e) => { + println!("Got error {e:?}"); + } + _ => { + panic!("Expected error."); + } + } +} + +#[test] +fn test_isolation() { + let root = Builder::new() + .prefix("test_isolation") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + + // Add one field. + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote"); + writer.commit().expect("committed"); + } + + { + let reader = k.read().unwrap(); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + } + + // Establish a long-lived reader that outlasts a writer. + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // Start a write transaction. + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote"); + + // The reader and writer are isolated. + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999))); + + // If we commit the writer, we still have isolation. + writer.commit().expect("committed"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // A new reader sees the committed value. Note that LMDB doesn't allow two + // read transactions to exist in the same thread, so we abort the previous one. + reader.abort(); + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999))); +} + +#[test] +fn test_blob() { + let root = Builder::new() + .prefix("test_round_trip_blob") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4])) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::Blob(&[1, 2, 3, 4])) + ); + + fn u16_to_u8(src: &[u16]) -> Vec { + let mut dst = vec![0; 2 * src.len()]; + LittleEndian::write_u16_into(src, &mut dst); + dst + } + + fn u8_to_u16(src: &[u8]) -> Vec { + let mut dst = vec![0; src.len() / 2]; + LittleEndian::read_u16_into(src, &mut dst); + dst + } + + // When storing UTF-16 strings as blobs, we'll need to convert + // their [u16] backing storage to [u8]. Test that converting, writing, + // reading, and converting back works as expected. + let u16_array = [1000, 10000, 54321, 65535]; + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array))) + .expect("wrote"); + let u8_array = match sk.get(&writer, "bar").expect("read") { + Some(Value::Blob(val)) => val, + _ => &[], + }; + assert_eq!(u8_to_u16(u8_array), u16_array); +} + +#[test] +fn test_sync() { + let root = Builder::new() + .prefix("test_sync") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(1); + builder.set_flags(EnvironmentFlags::NO_SYNC); + { + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + } + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); +} + +#[test] +#[cfg(feature = "db-int-key")] +fn test_stat() { + let root = Builder::new() + .prefix("test_stat") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + for i in 0..5 { + let sk = k + .open_integer(&format!("sk{i}")[..], StoreOptions::create()) + .expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, i, &Value::I64(i64::from(i))) + .expect("wrote"); + writer.commit().expect("committed"); + } + } + assert_eq!(k.stat().expect("stat").depth(), 1); + assert_eq!(k.stat().expect("stat").entries(), 5); + assert_eq!(k.stat().expect("stat").branch_pages(), 0); + assert_eq!(k.stat().expect("stat").leaf_pages(), 1); +} + +#[test] +fn test_info() { + let root = Builder::new() + .prefix("test_info") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + + let info = k.info().expect("info"); + + // The default size is 1MB. + assert_eq!(info.map_size(), DEFAULT_SIZE); + // Should greater than 0 after the write txn. + assert!(info.last_pgno() > 0); + // A txn to open_single + a txn to write. + assert_eq!(info.last_txnid(), 2); + // The default max readers is 126. + assert_eq!(info.max_readers(), 126); + assert_eq!(info.num_readers(), 0); + + // A new reader should increment the reader counter. + let _reader = k.read().expect("reader"); + let info = k.info().expect("info"); + + assert_eq!(info.num_readers(), 1); +} + +#[test] +fn test_load_ratio() { + let root = Builder::new() + .prefix("test_load_ratio") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + let ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(ratio > 0.0_f32 && ratio < 1.0_f32); + + // Put data to database should increase the load ratio. + let mut writer = k.write().expect("writer"); + sk.put( + &mut writer, + "bar", + &Value::Str(&"more-than-4KB".repeat(1000)), + ) + .expect("wrote"); + writer.commit().expect("commited"); + let new_ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(new_ratio > ratio); + + // Clear the database so that all the used pages should go to freelist, hence the ratio + // should decrease. + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("clear"); + writer.commit().expect("commited"); + let after_clear_ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(after_clear_ratio < new_ratio); +} + +#[test] +fn test_set_map_size() { + let root = Builder::new() + .prefix("test_size_map_size") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + assert_eq!(k.info().expect("info").map_size(), DEFAULT_SIZE); + + k.set_map_size(2 * DEFAULT_SIZE).expect("resized"); + + // Should be able to write. + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + + assert_eq!(k.info().expect("info").map_size(), 2 * DEFAULT_SIZE); +} + +#[test] +fn test_iter() { + let root = Builder::new() + .prefix("test_iter") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // An iterator over an empty store returns no values. + { + let reader = k.read().unwrap(); + let mut iter = sk.iter_start(&reader).unwrap(); + assert!(iter.next().is_none()); + } + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Reader.iter() returns (key, value) tuples ordered by key. + let mut iter = sk.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterators don't loop. Once one returns None, additional calls + // to its next() method will always return None. + assert!(iter.next().is_none()); + + // Reader.iter_from() begins iteration at the first key equal to + // or greater than the given key. + let mut iter = sk.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Reader.iter_from() works as expected when the given key is a prefix + // of a key in the store. + let mut iter = sk.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_iter_from_key_greater_than_existing() { + let root = Builder::new() + .prefix("test_iter_from_key_greater_than_existing") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + let mut iter = sk.iter_from(&reader, "nuu").unwrap(); + assert!(iter.next().is_none()); +} + +#[test] +fn test_multiple_store_read_write() { + let root = Builder::new() + .prefix("test_multiple_store_read_write") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + let s3 = k + .open_single("store_3", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + s1.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote"); + s3.put(&mut writer, "foo", &Value::Bool(true)) + .expect("wrote"); + + assert_eq!( + s1.get(&writer, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&writer, "foo").expect("read"), + Some(Value::Bool(true)) + ); + + writer.commit().expect("committed"); + + let reader = k.read().expect("unbound_reader"); + assert_eq!( + s1.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&reader, "foo").expect("read"), + Some(Value::Bool(true)) + ); + reader.abort(); + + // test delete across multiple stores + let mut writer = k.write().expect("writer"); + s1.delete(&mut writer, "foo").expect("deleted"); + s2.delete(&mut writer, "foo").expect("deleted"); + s3.delete(&mut writer, "foo").expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s1.get(&reader, "key").expect("value"), None); + assert_eq!(s2.get(&reader, "key").expect("value"), None); + assert_eq!(s3.get(&reader, "key").expect("value"), None); +} + +#[test] +fn test_multiple_store_iter() { + let root = Builder::new() + .prefix("test_multiple_store_iter") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + // Write to "s1" + s1.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s1.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s1.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s1.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + // &mut writer to "s2" + s2.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s2.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s2.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s2.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Iterate through the whole store in "s1" + let mut iter = s1.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate through the whole store in "s2" + let mut iter = s2.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s1" + let mut iter = s1.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s2" + let mut iter = s2.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s1" + let mut iter = s1.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s2" + let mut iter = s2.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_store_multiple_thread() { + let root = Builder::new() + .prefix("test_multiple_thread") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let rkv_arc = Arc::new(RwLock::new( + Rkv::new::(root.path()).expect("new succeeded"), + )); + let store = rkv_arc + .read() + .unwrap() + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let num_threads = 10; + let mut write_handles = Vec::with_capacity(num_threads as usize); + let mut read_handles = Vec::with_capacity(num_threads as usize); + + // Note that this isn't intended to demonstrate a good use of threads. + // For this shape of data, it would be more performant to write/read + // all values using one transaction in a single thread. The point here + // is just to confirm that a store can be shared by multiple threads. + + // For each KV pair, spawn a thread that writes it to the store. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + write_handles.push(thread::spawn(move || { + let rkv = rkv_arc.write().expect("rkv"); + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, i.to_string(), &Value::U64(i)) + .expect("written"); + writer.commit().unwrap(); + })); + } + for handle in write_handles { + handle.join().expect("joined"); + } + + // For each KV pair, spawn a thread that reads it from the store + // and returns its value. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + read_handles.push(thread::spawn(move || { + let rkv = rkv_arc.read().expect("rkv"); + let reader = rkv.read().expect("reader"); + let value = match store.get(&reader, i.to_string()) { + Ok(Some(Value::U64(value))) => value, + Ok(Some(_)) => panic!("value type unexpected"), + Ok(None) => panic!("value not found"), + Err(err) => panic!("{}", err), + }; + assert_eq!(value, i); + value + })); + } + + // Sum the values returned from the threads and confirm that they're + // equal to the sum of values written to the threads. + let thread_sum: u64 = read_handles + .into_iter() + .map(|handle| handle.join().expect("value")) + .sum(); + assert_eq!(thread_sum, (0..num_threads).sum()); +} + +#[test] +fn test_use_value_as_key() { + let root = Builder::new() + .prefix("test_use_value_as_key") + .tempdir() + .expect("tempdir"); + let rkv = Rkv::new::(root.path()).expect("new succeeded"); + let store = rkv + .open_single("store", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // It's possible to retrieve a value with a Reader and then use it + // as a key with a Writer. + { + let reader = &rkv.read().unwrap(); + if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") { + let mut writer = rkv.write().expect("writer"); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also retrieve a Value with a Writer and then use it as a key + // with the same Writer if you copy the value to an owned type + // so the Writer isn't still being borrowed by the retrieved value + // when you try to borrow the Writer again to modify that value. + { + let mut writer = rkv.write().expect("writer"); + if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") { + let key = value.to_owned(); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let name1 = rkv + .open_single("name1", StoreOptions::create()) + .expect("opened"); + let name2 = rkv + .open_single("name2", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv.write().expect("writer"); + name1 + .put(&mut writer, "key1", &Value::Str("bar")) + .expect("wrote"); + name1 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + name2 + .put(&mut writer, "key2", &Value::Str("bar")) + .expect("wrote"); + name2 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also iterate (store, key) pairs to retrieve foreign keys, + // then iterate those foreign keys to modify/delete them. + // + // You need to open the stores in advance, since opening a store + // uses a write transaction internally, so opening them while a writer + // is extant will hang. + // + // And you need to copy the values to an owned type so the Writer isn't + // still being borrowed by a retrieved value when you try to borrow + // the Writer again to modify another value. + let fields = vec![ + ( + rkv.open_single("name1", StoreOptions::create()) + .expect("opened"), + "key1", + ), + ( + rkv.open_single("name2", StoreOptions::create()) + .expect("opened"), + "key2", + ), + ]; + { + let mut foreignkeys = Vec::new(); + let mut writer = rkv.write().expect("writer"); + for (store, key) in fields.iter() { + if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") { + foreignkeys.push((store, value.to_owned())); + } + } + for (store, key) in foreignkeys.iter() { + store.delete(&mut writer, key).expect("deleted"); + } + writer.commit().expect("committed"); + } +} diff --git a/third_party/rust/rkv/tests/env-migration.rs b/third_party/rust/rkv/tests/env-migration.rs new file mode 100644 index 0000000000..7026c25de8 --- /dev/null +++ b/third_party/rust/rkv/tests/env-migration.rs @@ -0,0 +1,607 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fs, path::Path}; + +use tempfile::Builder; + +use rkv::{ + backend::{Lmdb, LmdbEnvironment, SafeMode, SafeModeEnvironment}, + Manager, Migrator, Rkv, StoreOptions, Value, +}; + +macro_rules! populate_store { + ($env:expr) => { + let store = $env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = $env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + }; +} + +#[test] +fn test_open_migrator_lmdb_to_safe() { + let root = Builder::new() + .prefix("test_open_migrator_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Check if the files were written to disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + } + // Verify that database was written to disk. + { + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + let store = src_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = src_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Open and migrate. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the old files were deleted from disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + } +} + +#[test] +fn test_open_migrator_safe_to_lmdb() { + let root = Builder::new() + .prefix("test_open_migrator_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Check if the files were written to disk. + { + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + } + // Verify that database was written to disk. + { + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + let store = src_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = src_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Open and migrate. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the old files were deleted from disk. + { + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(!safebin.exists()); + } +} + +#[test] +fn test_open_migrator_round_trip() { + let root = Builder::new() + .prefix("test_open_migrator_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Open and migrate. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Open and migrate back. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated twice. + { + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the right files are finally present on disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + assert!(!safebin.exists()); + } +} + +#[test] +fn test_easy_migrator_no_dir_1() { + let root = Builder::new() + .prefix("test_easy_migrator_no_dir") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // This won't fail with IoError even though the path is a bogus path, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(Path::new("bogus"), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + assert!(!safebin.exists()); // safe mode doesn't write an empty db to disk +} + +#[test] +fn test_easy_migrator_no_dir_2() { + let root = Builder::new() + .prefix("test_easy_migrator_no_dir") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // This won't fail with IoError even though the path is a bogus path, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(Path::new("bogus"), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // lmdb writes an empty db to disk + assert!(lockmdb.exists()); + assert!(!safebin.exists()); +} + +#[test] +fn test_easy_migrator_invalid_1() { + let root = Builder::new() + .prefix("test_easy_migrator_invalid") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // corrupted db isn't deleted + assert!(lockmdb.exists()); + assert!(!safebin.exists()); +} + +#[test] +fn test_easy_migrator_invalid_2() { + let root = Builder::new() + .prefix("test_easy_migrator_invalid") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // lmdb writes an empty db to disk + assert!(lockmdb.exists()); + assert!(safebin.exists()); // corrupted db isn't deleted +} + +#[test] +#[should_panic(expected = "migrated: SourceEmpty")] +fn test_migrator_lmdb_to_safe_1() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); +} + +#[test] +#[should_panic(expected = "migrated: DestinationNotEmpty")] +fn test_migrator_lmdb_to_safe_2() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&dst_env); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); +} + +#[test] +fn test_migrator_lmdb_to_safe_3() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); + + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); +} + +#[test] +#[should_panic(expected = "migrated: SourceEmpty")] +fn test_migrator_safe_to_lmdb_1() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); +} + +#[test] +#[should_panic(expected = "migrated: DestinationNotEmpty")] +fn test_migrator_safe_to_lmdb_2() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&dst_env); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); +} + +#[test] +fn test_migrator_safe_to_lmdb_3() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); + + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); +} + +#[test] +fn test_easy_migrator_failed_migration_1() { + let root = Builder::new() + .prefix("test_easy_migrator_failed_migration_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(&dbfile, "bogus").expect("bogus dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); + + // Populate destination environment and persist to disk. + populate_store!(&dst_env); + dst_env.sync(true).expect("synced"); + + // Delete bogus file and create a valid source environment in its place. + fs::remove_file(&dbfile).expect("bogus dbfile removed"); + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + + // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty. + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); +} + +#[test] +fn test_easy_migrator_failed_migration_2() { + let root = Builder::new() + .prefix("test_easy_migrator_failed_migration_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(&dbfile, "bogus").expect("bogus dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); + + // Populate destination environment and persist to disk. + populate_store!(&dst_env); + dst_env.sync(true).expect("synced"); + + // Delete bogus file and create a valid source environment in its place. + fs::remove_file(&dbfile).expect("bogus dbfile removed"); + let src_env = Rkv::new::(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + + // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty. + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); +} + +fn test_easy_migrator_from_manager_failed_migration_1() { + let root = Builder::new() + .prefix("test_easy_migrator_from_manager_failed_migration_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let mut src_manager = Manager::::singleton().write().unwrap(); + let created_src_arc = src_manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); + let src_env = created_src_arc.read().unwrap(); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + { + let mut dst_manager = Manager::::singleton().write().unwrap(); + let created_dst_arc_1 = dst_manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); + let dst_env_1 = created_dst_arc_1.read().unwrap(); + populate_store!(&dst_env_1); + dst_env_1.sync(true).expect("synced"); + } + + // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty. + let dst_manager = Manager::::singleton().read().unwrap(); + let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap(); + let dst_env_2 = created_dst_arc_2.read().unwrap(); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), dst_env_2).expect("migrated"); +} + +fn test_easy_migrator_from_manager_failed_migration_2() { + let root = Builder::new() + .prefix("test_easy_migrator_from_manager_failed_migration_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let mut src_manager = Manager::::singleton().write().unwrap(); + let created_src_arc = src_manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); + let src_env = created_src_arc.read().unwrap(); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + { + let mut dst_manager = Manager::::singleton().write().unwrap(); + let created_dst_arc_1 = dst_manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); + let dst_env_1 = created_dst_arc_1.read().unwrap(); + populate_store!(&dst_env_1); + dst_env_1.sync(true).expect("synced"); + } + + // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty. + let dst_manager = Manager::::singleton().read().unwrap(); + let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap(); + let dst_env_2 = created_dst_arc_2.read().unwrap(); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), dst_env_2).expect("migrated"); +} + +#[test] +fn test_easy_migrator_from_manager_failed_migration() { + test_easy_migrator_from_manager_failed_migration_1(); + test_easy_migrator_from_manager_failed_migration_2(); +} diff --git a/third_party/rust/rkv/tests/env-safe.rs b/third_party/rust/rkv/tests/env-safe.rs new file mode 100644 index 0000000000..9c20e39507 --- /dev/null +++ b/third_party/rust/rkv/tests/env-safe.rs @@ -0,0 +1,1370 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable +// deprecates `clippy::cyclomatic_complexity`. +#![allow(clippy::complexity)] + +use std::{ + fs, + path::Path, + str, + sync::{Arc, RwLock}, + thread, +}; + +use byteorder::{ByteOrder, LittleEndian}; +use tempfile::Builder; + +use rkv::{ + backend::{ + BackendEnvironmentBuilder, SafeMode, SafeModeDatabase, SafeModeEnvironment, + SafeModeRwTransaction, + }, + Rkv, SingleStore, StoreError, StoreOptions, Value, Writer, +}; + +fn check_rkv(k: &Rkv) { + let _ = k + .open_single(None, StoreOptions::create()) + .expect("created default"); + + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + let reader = k.read().expect("reader"); + + let result = s.get(&reader, "foo"); + assert_eq!(None, result.expect("success but no value")); +} + +/// We can't open a directory that doesn't exist. +#[test] +fn test_open_fails_safe() { + let root = Builder::new() + .prefix("test_open_fails_safe") + .tempdir() + .expect("tempdir"); + assert!(root.path().exists()); + + let nope = root.path().join("nope/"); + assert!(!nope.exists()); + + let pb = nope.to_path_buf(); + match Rkv::new::(nope.as_path()).err() { + Some(StoreError::UnsuitableEnvironmentPath(p)) => { + assert_eq!(pb, p); + } + _ => panic!("expected error"), + }; +} + +#[test] +fn test_open_safe() { + let root = Builder::new() + .prefix("test_open_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_safe() { + let root = Builder::new() + .prefix("test_open_from_builder_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_with_dir_safe_1() { + let root = Builder::new() + .prefix("test_open_from_builder_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + builder.set_make_dir_if_needed(true); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")] +fn test_open_from_builder_with_dir_safe_2() { + let root = Path::new("bogus"); + println!("Root path: {root:?}"); + assert!(!root.is_dir()); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root, builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_create_with_capacity_safe_1() { + let root = Builder::new() + .prefix("test_create_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This errors with "opened: DbsFull" because we specified a capacity of one (database), + // and check_rkv already opened one (plus the default database, which doesn't count + // against the limit). + let _zzz = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); +} + +#[test] +fn test_create_with_capacity_safe_2() { + let root = Builder::new() + .prefix("test_create_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This doesn't error with "opened: DbsFull" because even though we specified a capacity + // of one (database), and check_rkv already opened one, the default database doesn't + // count against the limit). + let _zzz = k.open_single(None, StoreOptions::create()).expect("opened"); +} + +#[test] +#[should_panic(expected = "opened: SafeModeError(DbNotFoundError)")] +fn test_open_with_capacity_safe_1() { + let root = Builder::new() + .prefix("test_open_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single("zzz", StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_open_with_capacity_safe_2() { + let root = Builder::new() + .prefix("test_open_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single(None, StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_list_dbs_safe_1() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!(dbs, vec![None, Some("s".to_owned())]); +} + +#[test] +fn test_list_dbs_safe_2() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 2).expect("rkv"); + check_rkv(&k); + + let _ = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!( + dbs, + vec![None, Some("s".to_owned()), Some("zzz".to_owned())] + ); +} + +#[test] +fn test_list_dbs_safe_3() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::(root.path(), 0).expect("rkv"); + + let _ = k.open_single(None, StoreOptions::create()).expect("opened"); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!(dbs, vec![None]); +} + +#[test] +fn test_round_trip_and_transactions_safe() { + let root = Builder::new() + .prefix("test_round_trip_and_transactions_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "noo").expect("read"), + Some(Value::F64(1234.0.into())) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + // Isolation. Reads won't return values. + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + // Dropped: tx rollback. Reads will still return nothing. + + { + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + // Isolation. Reads still return values. + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + // Dropped: tx rollback. Reads will still return values. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed but return None to indicate a missing value. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), None); + assert_eq!(sk.get(&r, "bar").expect("read"), None); + assert_eq!(sk.get(&r, "baz").expect("read"), None); + } +} + +#[test] +fn test_single_store_clear_safe() { + let root = Builder::new() + .prefix("test_single_store_clear_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + let iter = sk.iter_start(&r).expect("iter"); + assert_eq!(iter.count(), 0); + } +} + +#[test] +#[should_panic(expected = "KeyValuePairNotFound")] +fn test_single_store_delete_nonexistent_safe() { + let root = Builder::new() + .prefix("test_single_store_delete_nonexistent_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "bogus").unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multi_put_get_del_safe() { + let root = Builder::new() + .prefix("test_multi_put_get_del_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().unwrap(); + + let writer = k.write().unwrap(); + { + let mut iter = multistore.get(&writer, "str1").unwrap(); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar"))); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo"))); + } + writer.commit().unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .delete(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str1").unwrap(), + Some(Value::Str("str1 bar")) + ); + multistore + .delete(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str2").unwrap(), + Some(Value::Str("str2 foo")) + ); + multistore.delete_all(&mut writer, "str3").unwrap(); + assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None); + writer.commit().unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multiple_store_clear_safe() { + let root = Builder::new() + .prefix("test_multiple_store_clear_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let multistore = k + .open_multi("multistore", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = k.write().expect("writer"); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + multistore.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + assert_eq!(multistore.get_first(&r, "str1").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str2").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str3").expect("read"), None); + } +} + +#[test] +fn test_open_store_for_read_safe() { + let root = Builder::new() + .prefix("test_open_store_for_read_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + + // First create the store, and start a write transaction on it. + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("write"); + + // Open the same store for read, note that the write transaction is still in progress, + // it should not block the reader though. + let sk_readonly = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + writer.commit().expect("commit"); + + // Now the write transaction is committed, any followed reads should see its change. + let reader = k.read().expect("reader"); + assert_eq!( + sk_readonly.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); +} + +#[test] +#[should_panic(expected = "open a missing store")] +fn test_open_a_missing_store_safe() { + let root = Builder::new() + .prefix("test_open_a_missing_store_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let _sk = k + .open_single("sk", StoreOptions::default()) + .expect("open a missing store"); +} + +#[test] +#[should_panic(expected = "new failed: FileInvalid")] +fn test_open_a_broken_store_safe() { + let root = Builder::new() + .prefix("test_open_a_missing_store_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + let _ = Rkv::new::(root.path()).expect("new failed"); +} + +#[test] +fn test_open_fail_with_badrslot_safe() { + let root = Builder::new() + .prefix("test_open_fail_with_badrslot_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::default()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_create_fail_with_badrslot_safe() { + let root = Builder::new() + .prefix("test_create_fail_with_badrslot_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::create()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_read_before_write_num_safe() { + let root = Builder::new() + .prefix("test_read_before_write_num_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a number, modifying it, and then writing it back. + // We have to be done with the Value::I64 before calling Writer::put, + // as the Value::I64 borrows an immutable reference to the Writer. + // So we extract and copy its primitive value. + + fn get_existing_foo( + store: SingleStore, + writer: &Writer, + ) -> Option { + match store.get(writer, "foo").expect("read") { + Some(Value::I64(val)) => Some(val), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or(99); + existing += 1; + sk.put(&mut writer, "foo", &Value::I64(existing)) + .expect("success"); + + let updated = get_existing_foo(sk, &writer).unwrap_or(99); + assert_eq!(updated, 100); + writer.commit().expect("commit"); +} + +#[test] +fn test_read_before_write_str_safe() { + let root = Builder::new() + .prefix("test_read_before_write_str_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a string, modifying it, and then writing it back. + // We have to be done with the Value::Str before calling Writer::put, + // as the Value::Str (and its underlying &str) borrows an immutable + // reference to the Writer. So we copy it to a String. + + fn get_existing_foo( + store: SingleStore, + writer: &Writer, + ) -> Option { + match store.get(writer, "foo").expect("read") { + Some(Value::Str(val)) => Some(val.to_string()), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or_default(); + existing.push('…'); + sk.put(&mut writer, "foo", &Value::Str(&existing)) + .expect("write"); + + let updated = get_existing_foo(sk, &writer).unwrap_or_default(); + assert_eq!(updated, "…"); + writer.commit().expect("commit"); +} + +#[test] +fn test_isolation_safe() { + let root = Builder::new() + .prefix("test_isolation_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + + // Add one field. + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote"); + writer.commit().expect("committed"); + } + + { + let reader = k.read().unwrap(); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + } + + // Establish a long-lived reader that outlasts a writer. + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // Start a write transaction. + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote"); + + // The reader and writer are isolated. + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999))); + + // If we commit the writer, we still have isolation. + writer.commit().expect("committed"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // A new reader sees the committed value. Note that LMDB doesn't allow two + // read transactions to exist in the same thread, so we abort the previous one. + reader.abort(); + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999))); +} + +#[test] +fn test_blob_safe() { + let root = Builder::new() + .prefix("test_round_trip_blob_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4])) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::Blob(&[1, 2, 3, 4])) + ); + + fn u16_to_u8(src: &[u16]) -> Vec { + let mut dst = vec![0; 2 * src.len()]; + LittleEndian::write_u16_into(src, &mut dst); + dst + } + + fn u8_to_u16(src: &[u8]) -> Vec { + let mut dst = vec![0; src.len() / 2]; + LittleEndian::read_u16_into(src, &mut dst); + dst + } + + // When storing UTF-16 strings as blobs, we'll need to convert + // their [u16] backing storage to [u8]. Test that converting, writing, + // reading, and converting back works as expected. + let u16_array = [1000, 10000, 54321, 65535]; + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array))) + .expect("wrote"); + let u8_array = match sk.get(&writer, "bar").expect("read") { + Some(Value::Blob(val)) => val, + _ => &[], + }; + assert_eq!(u8_to_u16(u8_array), u16_array); +} + +#[test] +fn test_sync_safe() { + let root = Builder::new() + .prefix("test_sync_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut builder = Rkv::environment_builder::(); + builder.set_max_dbs(1); + { + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + } + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); +} + +#[test] +fn test_iter_safe() { + let root = Builder::new() + .prefix("test_iter_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // An iterator over an empty store returns no values. + { + let reader = k.read().unwrap(); + let mut iter = sk.iter_start(&reader).unwrap(); + assert!(iter.next().is_none()); + } + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Reader.iter() returns (key, value) tuples ordered by key. + let mut iter = sk.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterators don't loop. Once one returns None, additional calls + // to its next() method will always return None. + assert!(iter.next().is_none()); + + // Reader.iter_from() begins iteration at the first key equal to + // or greater than the given key. + let mut iter = sk.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Reader.iter_from() works as expected when the given key is a prefix + // of a key in the store. + let mut iter = sk.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_iter_from_key_greater_than_existing_safe() { + let root = Builder::new() + .prefix("test_iter_from_key_greater_than_existing_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + let mut iter = sk.iter_from(&reader, "nuu").unwrap(); + assert!(iter.next().is_none()); +} + +#[test] +fn test_multiple_store_read_write_safe() { + let root = Builder::new() + .prefix("test_multiple_store_read_write_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + let s3 = k + .open_single("store_3", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + s1.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote"); + s3.put(&mut writer, "foo", &Value::Bool(true)) + .expect("wrote"); + + assert_eq!( + s1.get(&writer, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&writer, "foo").expect("read"), + Some(Value::Bool(true)) + ); + + writer.commit().expect("committed"); + + let reader = k.read().expect("unbound_reader"); + assert_eq!( + s1.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&reader, "foo").expect("read"), + Some(Value::Bool(true)) + ); + reader.abort(); + + // test delete across multiple stores + let mut writer = k.write().expect("writer"); + s1.delete(&mut writer, "foo").expect("deleted"); + s2.delete(&mut writer, "foo").expect("deleted"); + s3.delete(&mut writer, "foo").expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s1.get(&reader, "key").expect("value"), None); + assert_eq!(s2.get(&reader, "key").expect("value"), None); + assert_eq!(s3.get(&reader, "key").expect("value"), None); +} + +#[test] +fn test_multiple_store_iter_safe() { + let root = Builder::new() + .prefix("test_multiple_store_iter_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + // Write to "s1" + s1.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s1.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s1.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s1.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + // &mut writer to "s2" + s2.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s2.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s2.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s2.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Iterate through the whole store in "s1" + let mut iter = s1.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate through the whole store in "s2" + let mut iter = s2.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s1" + let mut iter = s1.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s2" + let mut iter = s2.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s1" + let mut iter = s1.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s2" + let mut iter = s2.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_store_multiple_thread_safe() { + let root = Builder::new() + .prefix("test_multiple_thread_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let rkv_arc = Arc::new(RwLock::new( + Rkv::new::(root.path()).expect("new succeeded"), + )); + let store = rkv_arc + .read() + .unwrap() + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let num_threads = 10; + let mut write_handles = Vec::with_capacity(num_threads as usize); + let mut read_handles = Vec::with_capacity(num_threads as usize); + + // Note that this isn't intended to demonstrate a good use of threads. + // For this shape of data, it would be more performant to write/read + // all values using one transaction in a single thread. The point here + // is just to confirm that a store can be shared by multiple threads. + + // For each KV pair, spawn a thread that writes it to the store. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + write_handles.push(thread::spawn(move || { + let rkv = rkv_arc.write().expect("rkv"); + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, i.to_string(), &Value::U64(i)) + .expect("written"); + writer.commit().unwrap(); + })); + } + for handle in write_handles { + handle.join().expect("joined"); + } + + // For each KV pair, spawn a thread that reads it from the store + // and returns its value. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + read_handles.push(thread::spawn(move || { + let rkv = rkv_arc.read().expect("rkv"); + let reader = rkv.read().expect("reader"); + let value = match store.get(&reader, i.to_string()) { + Ok(Some(Value::U64(value))) => value, + Ok(Some(_)) => panic!("value type unexpected"), + Ok(None) => panic!("value not found"), + Err(err) => panic!("{}", err), + }; + assert_eq!(value, i); + value + })); + } + + // Sum the values returned from the threads and confirm that they're + // equal to the sum of values written to the threads. + let thread_sum: u64 = read_handles + .into_iter() + .map(|handle| handle.join().expect("value")) + .sum(); + assert_eq!(thread_sum, (0..num_threads).sum()); +} + +#[test] +fn test_use_value_as_key_safe() { + let root = Builder::new() + .prefix("test_use_value_as_key_safe") + .tempdir() + .expect("tempdir"); + let rkv = Rkv::new::(root.path()).expect("new succeeded"); + let store = rkv + .open_single("store", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // It's possible to retrieve a value with a Reader and then use it + // as a key with a Writer. + { + let reader = &rkv.read().unwrap(); + if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") { + let mut writer = rkv.write().expect("writer"); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also retrieve a Value with a Writer and then use it as a key + // with the same Writer if you copy the value to an owned type + // so the Writer isn't still being borrowed by the retrieved value + // when you try to borrow the Writer again to modify that value. + { + let mut writer = rkv.write().expect("writer"); + if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") { + let key = value.to_owned(); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let name1 = rkv + .open_single("name1", StoreOptions::create()) + .expect("opened"); + let name2 = rkv + .open_single("name2", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv.write().expect("writer"); + name1 + .put(&mut writer, "key1", &Value::Str("bar")) + .expect("wrote"); + name1 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + name2 + .put(&mut writer, "key2", &Value::Str("bar")) + .expect("wrote"); + name2 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also iterate (store, key) pairs to retrieve foreign keys, + // then iterate those foreign keys to modify/delete them. + // + // You need to open the stores in advance, since opening a store + // uses a write transaction internally, so opening them while a writer + // is extant will hang. + // + // And you need to copy the values to an owned type so the Writer isn't + // still being borrowed by a retrieved value when you try to borrow + // the Writer again to modify another value. + let fields = vec![ + ( + rkv.open_single("name1", StoreOptions::create()) + .expect("opened"), + "key1", + ), + ( + rkv.open_single("name2", StoreOptions::create()) + .expect("opened"), + "key2", + ), + ]; + { + let mut foreignkeys = Vec::new(); + let mut writer = rkv.write().expect("writer"); + for (store, key) in fields.iter() { + if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") { + foreignkeys.push((store, value.to_owned())); + } + } + for (store, key) in foreignkeys.iter() { + store.delete(&mut writer, key).expect("deleted"); + } + writer.commit().expect("committed"); + } +} diff --git a/third_party/rust/rkv/tests/integer-store.rs b/third_party/rust/rkv/tests/integer-store.rs new file mode 100644 index 0000000000..4f27776e79 --- /dev/null +++ b/third_party/rust/rkv/tests/integer-store.rs @@ -0,0 +1,88 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(feature = "db-int-key")] +#![allow(clippy::many_single_char_names)] + +use std::fs; + +use serde_derive::Serialize; +use tempfile::Builder; + +use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value}; + +#[test] +fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + macro_rules! test_integer_keys { + ($store:expr, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + $store + .put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + $store.get(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + $store.get(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + // The integer module provides only the u32 integer key variant + // of IntegerStore, so we can use it without further ado. + test_integer_keys!(s, std::u32::MIN); + test_integer_keys!(s, std::u32::MAX); + + // If you want to use another integer key variant, you need to implement + // a newtype, implement PrimitiveInt, and implement or derive Serialize + // for it. Here we do so for the i32 type. + + // DANGER! Doing this enables you to open a store with multiple, + // different integer key types, which may result in unexpected behavior. + // Make sure you know what you're doing! + + let t = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct I32(i32); + impl PrimitiveInt for I32 {} + test_integer_keys!(t, I32(std::i32::MIN)); + test_integer_keys!(t, I32(std::i32::MAX)); + + let u = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct U16(u16); + impl PrimitiveInt for U16 {} + test_integer_keys!(u, U16(std::u16::MIN)); + test_integer_keys!(u, U16(std::u16::MAX)); + + let v = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct U64(u64); + impl PrimitiveInt for U64 {} + test_integer_keys!(v, U64(std::u64::MIN)); + test_integer_keys!(v, U64(std::u64::MAX)); +} diff --git a/third_party/rust/rkv/tests/manager.rs b/third_party/rust/rkv/tests/manager.rs new file mode 100644 index 0000000000..e4785e2461 --- /dev/null +++ b/third_party/rust/rkv/tests/manager.rs @@ -0,0 +1,471 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fs, sync::Arc}; + +use tempfile::Builder; + +#[cfg(feature = "lmdb")] +use rkv::backend::{Lmdb, LmdbEnvironment}; +use rkv::{ + backend::{BackendEnvironmentBuilder, RecoveryStrategy, SafeMode, SafeModeEnvironment}, + CloseOptions, Rkv, StoreOptions, Value, +}; + +/// Test that a manager can be created with simple type inference. +#[cfg(feature = "lmdb")] +#[test] +#[allow(clippy::let_underscore_lock)] +fn test_simple() { + type Manager = rkv::Manager; + + let _unused = Manager::singleton().write().unwrap(); +} + +/// Test that a manager can be created with simple type inference. +#[test] +#[allow(clippy::let_underscore_lock)] +fn test_simple_safe() { + type Manager = rkv::Manager; + + let _unused = Manager::singleton().write().unwrap(); +} + +/// Test that a shared Rkv instance can be created with simple type inference. +#[cfg(feature = "lmdb")] +#[test] +fn test_simple_2() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_simple_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + let _ = manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); +} + +/// Test that a shared Rkv instance can be created with simple type inference. +#[test] +fn test_simple_safe_2() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_simple_safe_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + let _ = manager + .get_or_create(root.path(), Rkv::new::) + .unwrap(); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[cfg(feature = "lmdb")] +#[test] +fn test_same() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_same") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let p = root.path(); + assert!(Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .is_none()); + + let created_arc = Manager::singleton() + .write() + .unwrap() + .get_or_create(p, Rkv::new::) + .expect("created"); + let fetched_arc = Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[test] +fn test_same_safe() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_same_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let p = root.path(); + assert!(Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .is_none()); + + let created_arc = Manager::singleton() + .write() + .unwrap() + .get_or_create(p, Rkv::new::) + .expect("created"); + let fetched_arc = Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[cfg(feature = "lmdb")] +#[test] +fn test_same_with_capacity() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_same_with_capacity") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + + let p = root.path(); + assert!(manager.get(p).expect("success").is_none()); + + let created_arc = manager + .get_or_create_with_capacity(p, 10, Rkv::with_capacity::) + .expect("created"); + let fetched_arc = manager.get(p).expect("success").expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[test] +fn test_same_with_capacity_safe() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_same_with_capacity_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + + let p = root.path(); + assert!(manager.get(p).expect("success").is_none()); + + let created_arc = manager + .get_or_create_with_capacity(p, 10, Rkv::with_capacity::) + .expect("created"); + let fetched_arc = manager.get(p).expect("success").expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Some storage drivers are able to discard when the database is corrupted at runtime. +/// Test how these managers can discard corrupted databases and re-open. +#[test] +fn test_safe_mode_corrupt_while_open_1() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + let shared_env = manager + .get_or_create(root.path(), Rkv::new::) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Write some data. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Verify it was flushed to disk. + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreating environment fails. + manager + .get_or_create(root.path(), Rkv::new::) + .expect_err("not created"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // But we can use a builder and pass `discard_if_corrupted` to deal with it. + let mut builder = Rkv::environment_builder::(); + builder.set_corruption_recovery_strategy(RecoveryStrategy::Discard); + manager + .get_or_create_from_builder(root.path(), builder, Rkv::from_builder::) + .expect("created"); + assert!(manager.get(root.path()).expect("success").is_some()); +} + +/// Some storage drivers are able to recover when the database is corrupted at runtime. +/// Test how these managers can recover corrupted databases while open. +#[test] +fn test_safe_mode_corrupt_while_open_2() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + let shared_env = manager + .get_or_create(root.path(), Rkv::new::) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Write some data. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Verify it was flushed to disk. + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + + // Reading still works. Magic. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + reader.abort(); + + // Writing still works, dbfile will be un-corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo2", &Value::I64(5678)) + .expect("wrote"); + store + .put(&mut writer, "bar2", &Value::Bool(false)) + .expect("wrote"); + store + .put(&mut writer, "baz2", &Value::Str("byé, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreate environment. + let shared_env = manager + .get_or_create(root.path(), Rkv::new::) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Verify that the dbfile is not corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + assert_eq!( + store.get(&reader, "foo2").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + store.get(&reader, "bar2").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + store.get(&reader, "baz2").expect("read"), + Some(Value::Str("byé, yöu")) + ); +} + +/// Test how the manager can discard corrupted databases, while moving the corrupted one aside for +/// later inspection. +#[test] +fn test_safe_mode_corrupt_while_open_3() { + type Manager = rkv::Manager; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_3") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + assert!(safebin.exists(), "Corrupted database file was written to"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + + // Recreating environment fails. + manager + .get_or_create(root.path(), Rkv::new::) + .expect_err("not created"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // But we can use a builder and pass `RecoveryStrategy::Rename` to deal with it. + let mut builder = Rkv::environment_builder::(); + builder.set_corruption_recovery_strategy(RecoveryStrategy::Rename); + manager + .get_or_create_from_builder(root.path(), builder, Rkv::from_builder::) + .expect("created"); + assert!(manager.get(root.path()).expect("success").is_some()); + + assert!(!safebin.exists(), "Database file was moved out of the way"); + + let mut corruptbin = root.path().to_path_buf(); + corruptbin.push("data.safe.bin.corrupt"); + assert!(corruptbin.exists(), "Corrupted database file exists"); + + let shared_env = manager + .get_or_create(root.path(), Rkv::new::) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Writing still works. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + + let reader = env.read().expect("reader"); + assert_eq!(store.get(&reader, "foo").expect("read"), None, "Nothing to be read"); + + // We can write. + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(5678)) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + assert!(safebin.exists(), "Database file exists"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreate environment. + let shared_env = manager + .get_or_create(root.path(), Rkv::new::) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Verify that the dbfile is not corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(5678)), + "Database contains expected value" + ); +} diff --git a/third_party/rust/rkv/tests/multi-integer-store.rs b/third_party/rust/rkv/tests/multi-integer-store.rs new file mode 100644 index 0000000000..bce0060288 --- /dev/null +++ b/third_party/rust/rkv/tests/multi-integer-store.rs @@ -0,0 +1,122 @@ +// Copyright 2018 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] +#![allow(clippy::many_single_char_names)] + +use std::fs; + +use serde_derive::Serialize; +use tempfile::Builder; + +use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value}; + +#[test] +fn test_multi_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + macro_rules! test_integer_keys { + ($store:expr, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + $store + .put(&mut writer, $key, &Value::Str("hello1")) + .expect("write"); + $store + .put(&mut writer, $key, &Value::Str("hello2")) + .expect("write"); + $store + .put(&mut writer, $key, &Value::Str("hello3")) + .expect("write"); + let vals = $store + .get(&writer, $key) + .expect("read") + .map(|result| result.expect("ok")) + .map(|(_, v)| v) + .collect::>(); + assert_eq!( + vals, + vec![ + Value::Str("hello1"), + Value::Str("hello2"), + Value::Str("hello3") + ] + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let vals = $store + .get(&reader, $key) + .expect("read") + .map(|result| result.expect("ok")) + .map(|(_, v)| v) + .collect::>(); + assert_eq!( + vals, + vec![ + Value::Str("hello1"), + Value::Str("hello2"), + Value::Str("hello3") + ] + ); + }}; + } + + // The integer module provides only the u32 integer key variant + // of IntegerStore, so we can use it without further ado. + test_integer_keys!(s, std::u32::MIN); + test_integer_keys!(s, std::u32::MAX); + + // If you want to use another integer key variant, you need to implement + // a newtype, implement PrimitiveInt, and implement or derive Serialize + // for it. Here we do so for the i32 type. + + // DANGER! Doing this enables you to open a store with multiple, + // different integer key types, which may result in unexpected behavior. + // Make sure you know what you're doing! + + let t = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct I32(i32); + impl PrimitiveInt for I32 {} + test_integer_keys!(t, I32(std::i32::MIN)); + test_integer_keys!(t, I32(std::i32::MAX)); + + let u = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct U16(u16); + impl PrimitiveInt for U16 {} + test_integer_keys!(u, U16(std::u16::MIN)); + test_integer_keys!(u, U16(std::u16::MAX)); + + let v = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct U64(u64); + impl PrimitiveInt for U64 {} + test_integer_keys!(v, U64(std::u64::MIN)); + test_integer_keys!(v, U64(std::u64::MAX)); +} diff --git a/third_party/rust/rkv/tests/test_txn.rs b/third_party/rust/rkv/tests/test_txn.rs new file mode 100644 index 0000000000..1e8fdfb416 --- /dev/null +++ b/third_party/rust/rkv/tests/test_txn.rs @@ -0,0 +1,131 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(feature = "db-dup-sort")] + +use std::fs; + +use tempfile::Builder; + +use rkv::{ + backend::{SafeMode, SafeModeDatabase, SafeModeRoCursor, SafeModeRwTransaction}, + Readable, Rkv, StoreOptions, Value, Writer, +}; + +/// Consider a struct like this: +/// struct Sample { +/// id: u64, +/// value: String, +/// date: String, +/// } +/// We would like to index all of the fields so that we can search for the struct not only +/// by ID but also by value and date. When we index the fields individually in their own +/// tables, it is important that we run all operations within a single transaction to +/// ensure coherence of the indices. +/// This test features helper functions for reading and writing the parts of the struct. +/// Note that the reader functions take `Readable` because they might run within a Read +/// Transaction or a Write Transaction. The test demonstrates fetching values via both. + +type SingleStore = rkv::SingleStore; +type MultiStore = rkv::MultiStore; + +#[test] +fn read_many() { + let root = Builder::new() + .prefix("test_txns") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new::(root.path()).expect("new succeeded"); + let samplestore = k.open_single("s", StoreOptions::create()).expect("open"); + let datestore = k.open_multi("m", StoreOptions::create()).expect("open"); + let valuestore = k.open_multi("m", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("env write lock"); + + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + put_id_field(&mut writer, datestore, &date, id); + put_id_field(&mut writer, valuestore, &value, id); + put_sample(&mut writer, samplestore, id, &value); + } + + // now we read in the same transaction + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + let ids = get_ids_by_field(&writer, datestore, &date); + let ids2 = get_ids_by_field(&writer, valuestore, &value); + let samples = get_samples(&writer, samplestore, &ids); + let samples2 = get_samples(&writer, samplestore, &ids2); + println!("{samples:?}, {samples2:?}"); + } + } + + { + let reader = k.read().expect("env read lock"); + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + let ids = get_ids_by_field(&reader, datestore, &date); + let ids2 = get_ids_by_field(&reader, valuestore, &value); + let samples = get_samples(&reader, samplestore, &ids); + let samples2 = get_samples(&reader, samplestore, &ids2); + println!("{samples:?}, {samples2:?}"); + } + } +} + +fn get_ids_by_field<'t, T>(txn: &'t T, store: MultiStore, field: &'t str) -> Vec +where + T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>, +{ + store + .get(txn, field) + .expect("get iterator") + .map(|id| match id.expect("field") { + (_, Value::U64(id)) => id, + _ => panic!("getting value in iter"), + }) + .collect::>() +} + +fn get_samples<'t, T>(txn: &'t T, samplestore: SingleStore, ids: &[u64]) -> Vec +where + T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>, +{ + ids.iter() + .map(|id| { + let bytes = id.to_be_bytes(); + match samplestore.get(txn, &bytes).expect("fetch sample") { + Some(Value::Str(sample)) => String::from(sample), + Some(_) => panic!("wrong type"), + None => panic!("no sample for this id!"), + } + }) + .collect::>() +} + +fn put_sample( + txn: &mut Writer, + samplestore: SingleStore, + id: u64, + value: &str, +) { + let idbytes = id.to_be_bytes(); + samplestore + .put(txn, &idbytes, &Value::Str(value)) + .expect("put id"); +} + +fn put_id_field(txn: &mut Writer, store: MultiStore, field: &str, id: u64) { + store.put(txn, field, &Value::U64(id)).expect("put id"); +} -- cgit v1.2.3